@ultimat3/realtime 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +180 -0
- package/package.json +36 -0
- package/src/change-buffer.ts +69 -0
- package/src/changefeed-env.ts +146 -0
- package/src/changefeed.ts +191 -0
- package/src/channel.ts +181 -0
- package/src/client.ts +439 -0
- package/src/cursor.ts +188 -0
- package/src/errors.ts +253 -0
- package/src/fanout.ts +159 -0
- package/src/hooks.ts +230 -0
- package/src/index.ts +328 -0
- package/src/json.ts +76 -0
- package/src/live-definition.ts +144 -0
- package/src/live-query.ts +449 -0
- package/src/local-store.ts +188 -0
- package/src/matcher-bridge.ts +169 -0
- package/src/nats-commands.ts +97 -0
- package/src/nats-connection-fixture.ts +105 -0
- package/src/nats-connection.ts +464 -0
- package/src/nats-fake.ts +431 -0
- package/src/nats-jetstream.ts +226 -0
- package/src/nats-kv.ts +157 -0
- package/src/nats-protocol.ts +222 -0
- package/src/nats-socket.ts +236 -0
- package/src/nats-transport.ts +257 -0
- package/src/offline-queue.ts +206 -0
- package/src/pg-advisory-lock.ts +98 -0
- package/src/pg-auth.ts +300 -0
- package/src/pg-bytes.ts +185 -0
- package/src/pg-connection-fixture.ts +215 -0
- package/src/pg-connection.ts +337 -0
- package/src/pg-entity-row.ts +130 -0
- package/src/pg-replication-fixture.ts +261 -0
- package/src/pg-replication.ts +396 -0
- package/src/pg-socket.ts +265 -0
- package/src/pg-wire.ts +192 -0
- package/src/pgoutput.ts +297 -0
- package/src/policy-gate.ts +56 -0
- package/src/presence.ts +219 -0
- package/src/rebase.ts +198 -0
- package/src/replicator.ts +185 -0
- package/src/socket.ts +208 -0
- package/src/sync-node.ts +400 -0
- package/src/sync-protocol.ts +376 -0
- package/src/thundering-herd.ts +141 -0
- package/src/transport-env.ts +104 -0
package/src/cursor.ts
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
// Reconnect. The highest-risk surface in the framework: a deploy drops N sockets at once and
|
|
2
|
+
// every one of them asks "what changed since X?". The cursor exists to make that answer cheap,
|
|
3
|
+
// and the budget exists to make the expensive answer (a snapshot) the *chosen* one, not the
|
|
4
|
+
// accidental one. See README "Reconnect is the hard part".
|
|
5
|
+
|
|
6
|
+
import { type Clock, systemClock } from '@ultimat3/core';
|
|
7
|
+
import { CursorStaleError } from './errors';
|
|
8
|
+
import { canonicalJson, fnv1a, type Row, type RowPatch } from './json';
|
|
9
|
+
|
|
10
|
+
/** Ids are bounded so a cursor stays small enough to ship in a `hello` frame. */
|
|
11
|
+
export const CURSOR_ID_LIMIT = 512;
|
|
12
|
+
|
|
13
|
+
/** A digest of `''` means "not verified at this lsn" — set on every delta resume. */
|
|
14
|
+
export const DIGEST_UNVERIFIED = '';
|
|
15
|
+
|
|
16
|
+
export interface LiveCursor {
|
|
17
|
+
readonly qid: string;
|
|
18
|
+
/** Last lsn the subscriber has applied. Lexicographically comparable (see `formatLsn`). */
|
|
19
|
+
readonly lsn: string;
|
|
20
|
+
/** Result-set digest at the last snapshot, or `DIGEST_UNVERIFIED` after a delta resume. */
|
|
21
|
+
readonly digest: string;
|
|
22
|
+
/** Last-seen row ids, truncated at `CURSOR_ID_LIMIT`. */
|
|
23
|
+
readonly ids: readonly string[];
|
|
24
|
+
/** Number of rows in the result set — survives id truncation. */
|
|
25
|
+
readonly count: number;
|
|
26
|
+
readonly at: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface ReconnectBudget {
|
|
30
|
+
/** Hard ceiling on replayed patches, independent of the cost model. */
|
|
31
|
+
readonly maxPatches: number;
|
|
32
|
+
/** A cursor older than this is presumed to have drifted; re-snapshot instead of trusting it. */
|
|
33
|
+
readonly maxLagMs: number;
|
|
34
|
+
/** Cost of one snapshot, expressed in replayed-patch equivalents. This is the crossover point. */
|
|
35
|
+
readonly snapshotCost: number;
|
|
36
|
+
readonly patchCost: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 250 patch-equivalents per snapshot is the measured-intent default: a bounded `orderBy + limit`
|
|
41
|
+
* snapshot is one indexed query, a patch is one buffer read plus one frame. Replaying more than
|
|
42
|
+
* ~250 patches costs more wall-clock *and* more client work than starting over.
|
|
43
|
+
*/
|
|
44
|
+
export const defaultReconnectBudget: ReconnectBudget = {
|
|
45
|
+
maxPatches: 500,
|
|
46
|
+
maxLagMs: 5 * 60_000,
|
|
47
|
+
snapshotCost: 250,
|
|
48
|
+
patchCost: 1,
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
export type ResumeReason =
|
|
52
|
+
| 'in-window'
|
|
53
|
+
| 'unknown-query'
|
|
54
|
+
| 'out-of-window'
|
|
55
|
+
| 'lag-exceeded'
|
|
56
|
+
| 'budget-exceeded'
|
|
57
|
+
| 'digest-unverified';
|
|
58
|
+
|
|
59
|
+
export interface ResumeDecision {
|
|
60
|
+
readonly resnapshot: boolean;
|
|
61
|
+
readonly reason: ResumeReason;
|
|
62
|
+
/** Replay cost in patch-equivalents, for logs and the reconnect benchmark. */
|
|
63
|
+
readonly cost: number;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Retained-change source behind a resume. Implemented by `RingChangeBuffer` in the replicator. */
|
|
67
|
+
export interface ResumeSource {
|
|
68
|
+
append(qid: string, patch: RowPatch): void;
|
|
69
|
+
/** Patches strictly after `lsn`, or `null` when the gap is not covered by the retained window. */
|
|
70
|
+
since(qid: string, lsn: string): RowPatch[] | null;
|
|
71
|
+
headLsn(qid: string): string | null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export type ResumeResult<R extends Row = Row> =
|
|
75
|
+
| { readonly kind: 'delta'; readonly patches: readonly RowPatch[]; readonly cursor: LiveCursor }
|
|
76
|
+
| { readonly kind: 'snapshot'; readonly rows: readonly R[]; readonly cursor: LiveCursor };
|
|
77
|
+
|
|
78
|
+
export interface ResumeDeps<R extends Row = Row> {
|
|
79
|
+
readonly source: ResumeSource;
|
|
80
|
+
/** Bounded re-read of the query at a current lsn. Omit only if a stale cursor should throw. */
|
|
81
|
+
readonly snapshot?: (qid: string) => Promise<{ rows: readonly R[]; lsn: string }>;
|
|
82
|
+
readonly budget?: ReconnectBudget;
|
|
83
|
+
readonly clock?: Clock;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function makeCursor(
|
|
87
|
+
qid: string,
|
|
88
|
+
lsn: string,
|
|
89
|
+
rows: readonly Row[],
|
|
90
|
+
now: number,
|
|
91
|
+
): LiveCursor {
|
|
92
|
+
return {
|
|
93
|
+
qid,
|
|
94
|
+
lsn,
|
|
95
|
+
digest: digestOf(rows),
|
|
96
|
+
ids: rows.slice(0, CURSOR_ID_LIMIT).map((r) => r.id),
|
|
97
|
+
count: rows.length,
|
|
98
|
+
at: now,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** FNV-1a over `id:row` pairs in result order — order-sensitive, so a re-sort is detected. */
|
|
103
|
+
export function digestOf(rows: readonly Row[]): string {
|
|
104
|
+
return fnv1a(rows.map((row) => `${row.id}:${canonicalJson(row)}`).join(';'));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Client-side drift check: a mismatch after delta resumes is a request for a fresh snapshot. */
|
|
108
|
+
export function verifyDigest(cursor: LiveCursor, rows: readonly Row[]): boolean {
|
|
109
|
+
if (cursor.digest === DIGEST_UNVERIFIED) return false;
|
|
110
|
+
return cursor.digest === digestOf(rows);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function shouldResnapshot(
|
|
114
|
+
cursor: LiveCursor,
|
|
115
|
+
available: readonly RowPatch[] | null,
|
|
116
|
+
now: number,
|
|
117
|
+
budget: ReconnectBudget = defaultReconnectBudget,
|
|
118
|
+
): ResumeDecision {
|
|
119
|
+
if (available === null) {
|
|
120
|
+
return { resnapshot: true, reason: 'out-of-window', cost: budget.snapshotCost };
|
|
121
|
+
}
|
|
122
|
+
if (now - cursor.at > budget.maxLagMs) {
|
|
123
|
+
return { resnapshot: true, reason: 'lag-exceeded', cost: budget.snapshotCost };
|
|
124
|
+
}
|
|
125
|
+
const cost = available.length * budget.patchCost;
|
|
126
|
+
if (available.length > budget.maxPatches || cost > budget.snapshotCost) {
|
|
127
|
+
return { resnapshot: true, reason: 'budget-exceeded', cost };
|
|
128
|
+
}
|
|
129
|
+
return { resnapshot: false, reason: 'in-window', cost };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The reconnect entry point. Cheap path = replay the retained window. Fallback = one bounded
|
|
134
|
+
* snapshot. There is deliberately no third path: WAL history traversal is what turns a rolling
|
|
135
|
+
* restart into a self-inflicted outage.
|
|
136
|
+
*/
|
|
137
|
+
export async function resumeFrom<R extends Row = Row>(
|
|
138
|
+
cursor: LiveCursor,
|
|
139
|
+
deps: ResumeDeps<R>,
|
|
140
|
+
): Promise<ResumeResult<R>> {
|
|
141
|
+
const clock = deps.clock ?? systemClock;
|
|
142
|
+
const budget = deps.budget ?? defaultReconnectBudget;
|
|
143
|
+
const now = clock.now().getTime();
|
|
144
|
+
const available = deps.source.since(cursor.qid, cursor.lsn);
|
|
145
|
+
const decision = shouldResnapshot(cursor, available, now, budget);
|
|
146
|
+
|
|
147
|
+
if (!decision.resnapshot && available !== null) {
|
|
148
|
+
const head = deps.source.headLsn(cursor.qid) ?? cursor.lsn;
|
|
149
|
+
return { kind: 'delta', patches: available, cursor: advance(cursor, available, head, now) };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (!deps.snapshot) {
|
|
153
|
+
throw new CursorStaleError({ qid: cursor.qid, lsn: cursor.lsn, reason: decision.reason });
|
|
154
|
+
}
|
|
155
|
+
const fresh = await deps.snapshot(cursor.qid);
|
|
156
|
+
return {
|
|
157
|
+
kind: 'snapshot',
|
|
158
|
+
rows: fresh.rows,
|
|
159
|
+
cursor: makeCursor(cursor.qid, fresh.lsn, fresh.rows, now),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Advance a cursor across a delta without re-reading rows: ids are patchable, the digest is not. */
|
|
164
|
+
export function advance(
|
|
165
|
+
cursor: LiveCursor,
|
|
166
|
+
patches: readonly RowPatch[],
|
|
167
|
+
lsn: string,
|
|
168
|
+
now: number,
|
|
169
|
+
): LiveCursor {
|
|
170
|
+
const ids = new Set(cursor.ids);
|
|
171
|
+
let count = cursor.count;
|
|
172
|
+
for (const patch of patches) {
|
|
173
|
+
if (patch.op === 'delete') {
|
|
174
|
+
if (ids.delete(patch.id)) count -= 1;
|
|
175
|
+
} else if (patch.op === 'insert' && !ids.has(patch.id)) {
|
|
176
|
+
ids.add(patch.id);
|
|
177
|
+
count += 1;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
qid: cursor.qid,
|
|
182
|
+
lsn,
|
|
183
|
+
digest: DIGEST_UNVERIFIED,
|
|
184
|
+
ids: [...ids].slice(0, CURSOR_ID_LIMIT),
|
|
185
|
+
count,
|
|
186
|
+
at: now,
|
|
187
|
+
};
|
|
188
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
// Realtime's X_* codes. Every throw in this package goes through one of these classes so
|
|
2
|
+
// the same string renders in the terminal, the browser overlay, and `--json`.
|
|
3
|
+
|
|
4
|
+
import { registerErrorCodes, UltimateError } from '@ultimat3/core';
|
|
5
|
+
|
|
6
|
+
/** Codes this package declares and owns. */
|
|
7
|
+
export const REALTIME_OWNED_ERROR_CODES = [
|
|
8
|
+
'X_TOPIC_FORBIDDEN',
|
|
9
|
+
'X_SUBSCRIPTION_LIMIT',
|
|
10
|
+
'X_PROTOCOL_VERSION',
|
|
11
|
+
'X_CURSOR_STALE',
|
|
12
|
+
'X_REBASE_CONFLICT',
|
|
13
|
+
'X_TRANSPORT_UNAVAILABLE',
|
|
14
|
+
'X_TRANSPORT_PROTOCOL',
|
|
15
|
+
'X_REPLICATION_PROTOCOL',
|
|
16
|
+
'X_REPLICATION_FAILED',
|
|
17
|
+
'X_REPLICATOR_SLOT_HELD',
|
|
18
|
+
'X_LIVE_CLIENT_MISSING',
|
|
19
|
+
'X_LIVE_ROW_UNIDENTIFIED',
|
|
20
|
+
] as const;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* `X_NOT_IMPLEMENTED` is `@ultimat3/core`'s, and `X_FORBIDDEN` — thrown by the surface denials this
|
|
24
|
+
* package renders — is `@ultimat3/policy`'s. Neither is titled here: the owner writes the one title
|
|
25
|
+
* every surface renders, and a copy kept alongside it is a copy that goes stale unnoticed.
|
|
26
|
+
*/
|
|
27
|
+
export const REALTIME_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED'] as const;
|
|
28
|
+
|
|
29
|
+
/** Every code realtime can throw through `RealtimeError`: the ones it owns plus the borrowed one. */
|
|
30
|
+
export const REALTIME_ERROR_CODES = [
|
|
31
|
+
...REALTIME_OWNED_ERROR_CODES,
|
|
32
|
+
...REALTIME_BORROWED_ERROR_CODES,
|
|
33
|
+
] as const;
|
|
34
|
+
|
|
35
|
+
export type RealtimeOwnedErrorCode = (typeof REALTIME_OWNED_ERROR_CODES)[number];
|
|
36
|
+
export type RealtimeErrorCode = (typeof REALTIME_ERROR_CODES)[number];
|
|
37
|
+
|
|
38
|
+
export const REALTIME_ERROR_TITLES: Readonly<Record<RealtimeOwnedErrorCode, string>> = {
|
|
39
|
+
X_TOPIC_FORBIDDEN: 'the actor may not subscribe to this topic',
|
|
40
|
+
X_SUBSCRIPTION_LIMIT: 'socket or tenant hit its subscription cap',
|
|
41
|
+
X_PROTOCOL_VERSION: 'client and sync node disagree on the wire protocol',
|
|
42
|
+
X_CURSOR_STALE: 'the resume LSN is outside the change buffer',
|
|
43
|
+
X_REBASE_CONFLICT: 'a local mutation could not be rebased',
|
|
44
|
+
X_TRANSPORT_UNAVAILABLE: 'the fanout bus is unreachable',
|
|
45
|
+
X_TRANSPORT_PROTOCOL: 'the bus does not speak the protocol this build speaks',
|
|
46
|
+
X_REPLICATION_PROTOCOL: 'the WAL stream cannot be decoded',
|
|
47
|
+
X_REPLICATION_FAILED: 'the replication connection was refused',
|
|
48
|
+
X_REPLICATOR_SLOT_HELD: 'another replicator already owns this database',
|
|
49
|
+
X_LIVE_CLIENT_MISSING: 'a realtime hook ran with no LiveClient registered',
|
|
50
|
+
X_LIVE_ROW_UNIDENTIFIED: 'a live query returned a row with no id',
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
// One unconditional call, so a second package claiming one of realtime's codes throws
|
|
54
|
+
// X_ERROR_CODE_DUPLICATE instead of losing silently to whichever module imported first.
|
|
55
|
+
registerErrorCodes(
|
|
56
|
+
Object.fromEntries(
|
|
57
|
+
Object.entries(REALTIME_ERROR_TITLES).map(([code, title]) => [code, { title }]),
|
|
58
|
+
),
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
const DOCS_BASE = 'https://ultimate.dev/errors/';
|
|
62
|
+
|
|
63
|
+
/** Base for every realtime error: fills `docs` from the code so no call site can forget it. */
|
|
64
|
+
export class RealtimeError extends UltimateError {
|
|
65
|
+
constructor(opts: { code: RealtimeErrorCode; cause: string; fix: string }) {
|
|
66
|
+
super({
|
|
67
|
+
code: opts.code,
|
|
68
|
+
cause: opts.cause,
|
|
69
|
+
fix: opts.fix,
|
|
70
|
+
docs: `${DOCS_BASE}${opts.code}`,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Subscribe (or an actor change) denied by the topic's policy. Never leaks the topic's data. */
|
|
76
|
+
export class TopicForbiddenError extends RealtimeError {
|
|
77
|
+
constructor(args: { topic: string; actorId: string | null; reason: string }) {
|
|
78
|
+
super({
|
|
79
|
+
code: 'X_TOPIC_FORBIDDEN',
|
|
80
|
+
cause: `actor ${args.actorId ?? '<anonymous>'} may not subscribe to "${args.topic}": ${args.reason}`,
|
|
81
|
+
fix: `declare a guard for this topic: hub.guard('${args.topic}', ({ actor }) => ...)`,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Load shedding, not a crash: a socket or tenant asked for more subscriptions than the cap. */
|
|
87
|
+
export class SubscriptionLimitError extends RealtimeError {
|
|
88
|
+
constructor(args: { scope: 'socket' | 'tenant'; id: string; limit: number }) {
|
|
89
|
+
super({
|
|
90
|
+
code: 'X_SUBSCRIPTION_LIMIT',
|
|
91
|
+
cause: `${args.scope} ${args.id} reached the subscription cap of ${args.limit}`,
|
|
92
|
+
fix: `raise realtime.limits.${args.scope === 'socket' ? 'perSocket' : 'perTenant'} in app.config.ts, or unsubscribe unused live queries`,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Client and server disagree on the wire format — a version mismatch or a malformed frame.
|
|
99
|
+
* Both are the same class of bug (a peer speaking a shape we do not have), so both get one code.
|
|
100
|
+
*/
|
|
101
|
+
export class ProtocolVersionError extends RealtimeError {
|
|
102
|
+
constructor(args: { got: unknown; expected: number; detail?: string }) {
|
|
103
|
+
super({
|
|
104
|
+
code: 'X_PROTOCOL_VERSION',
|
|
105
|
+
cause:
|
|
106
|
+
args.detail ??
|
|
107
|
+
`frame protocol version ${String(args.got)} is not the server version ${args.expected}`,
|
|
108
|
+
fix: 'x build && redeploy the client; the sync node sends `update-available` before it drains',
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** A resume cursor cannot be honoured and no snapshot path was supplied. */
|
|
114
|
+
export class CursorStaleError extends RealtimeError {
|
|
115
|
+
constructor(args: { qid: string; lsn: string; reason: string }) {
|
|
116
|
+
super({
|
|
117
|
+
code: 'X_CURSOR_STALE',
|
|
118
|
+
cause: `cursor for query ${args.qid} at lsn ${args.lsn} cannot be resumed: ${args.reason}`,
|
|
119
|
+
fix: 'pass `snapshot` to resumeFrom() so the fallback path can re-snapshot instead of failing',
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** A rebase could not be resolved: `custom(merge)` returned nothing, or the base row vanished. */
|
|
125
|
+
export class RebaseConflictError extends RealtimeError {
|
|
126
|
+
constructor(args: { key: string; entity: string; reason: string }) {
|
|
127
|
+
super({
|
|
128
|
+
code: 'X_REBASE_CONFLICT',
|
|
129
|
+
cause: `mutation ${args.key} on ${args.entity} could not be rebased: ${args.reason}`,
|
|
130
|
+
fix: "set conflict: 'server-wins' on the mutator, or return a row from custom(merge)",
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The fanout bus is down. `sync` nodes are stateless, so this is always recoverable. */
|
|
136
|
+
export class TransportUnavailableError extends RealtimeError {
|
|
137
|
+
constructor(args: { transport: string; reason: string; fix?: string }) {
|
|
138
|
+
super({
|
|
139
|
+
code: 'X_TRANSPORT_UNAVAILABLE',
|
|
140
|
+
cause: `transport "${args.transport}" is unavailable: ${args.reason}`,
|
|
141
|
+
// Names the key `selectTransport` actually reads, and a command that actually exists.
|
|
142
|
+
fix: args.fix ?? 'x doctor — then check NATS_URL points at a reachable nats-server',
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The bytes on the bus socket are not the protocol we speak: an unknown NATS verb, a header block
|
|
149
|
+
* that is not `NATS/1.0`, a JetStream reply in a shape the API never produces. Always a version or
|
|
150
|
+
* configuration mismatch rather than a transient fault, so reconnecting to the same server cannot
|
|
151
|
+
* help — which is exactly why it is a different code from `X_TRANSPORT_UNAVAILABLE`.
|
|
152
|
+
*/
|
|
153
|
+
export class TransportProtocolError extends RealtimeError {
|
|
154
|
+
constructor(args: { transport: string; stage: string; detail: string; fix?: string }) {
|
|
155
|
+
super({
|
|
156
|
+
code: 'X_TRANSPORT_PROTOCOL',
|
|
157
|
+
cause: `transport "${args.transport}" ${args.stage}: ${args.detail}`,
|
|
158
|
+
fix:
|
|
159
|
+
args.fix ??
|
|
160
|
+
'x doctor transport — the bus must be nats-server >= 2.11 with JetStream enabled (`nats-server -js`)',
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The bytes on the replication socket are not the bytes the protocol allows: a truncated message,
|
|
167
|
+
* an unknown pgoutput tag, an auth method we do not speak. Always a version or configuration
|
|
168
|
+
* mismatch rather than a transient fault, so retrying the same connection cannot help.
|
|
169
|
+
*/
|
|
170
|
+
export class ReplicationProtocolError extends RealtimeError {
|
|
171
|
+
constructor(args: { stage: string; detail: string; fix?: string }) {
|
|
172
|
+
super({
|
|
173
|
+
code: 'X_REPLICATION_PROTOCOL',
|
|
174
|
+
cause: `postgres replication ${args.stage}: ${args.detail}`,
|
|
175
|
+
fix:
|
|
176
|
+
args.fix ??
|
|
177
|
+
'x doctor db — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical',
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* The replication connection itself failed — refused credentials, a slot another process holds,
|
|
184
|
+
* an `ErrorResponse` from the server. The server's own message is passed through verbatim
|
|
185
|
+
* because it names the object that has to change.
|
|
186
|
+
*/
|
|
187
|
+
export class ReplicationFailedError extends RealtimeError {
|
|
188
|
+
constructor(args: { stage: string; detail: string; fix: string }) {
|
|
189
|
+
super({
|
|
190
|
+
code: 'X_REPLICATION_FAILED',
|
|
191
|
+
cause: `postgres replication ${args.stage} failed: ${args.detail}`,
|
|
192
|
+
fix: args.fix,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
|
|
199
|
+
* nothing is wrong with this process: the database already has its one replicator, and a second
|
|
200
|
+
* one that started anyway would publish every change twice. Terminal for a container whose whole
|
|
201
|
+
* job is that role — the scheduler is the thing that has to change, not the connection.
|
|
202
|
+
*/
|
|
203
|
+
export class ReplicatorSlotHeldError extends RealtimeError {
|
|
204
|
+
constructor(args: { key: string; holder?: string | undefined }) {
|
|
205
|
+
super({
|
|
206
|
+
code: 'X_REPLICATOR_SLOT_HELD',
|
|
207
|
+
cause:
|
|
208
|
+
`advisory lock ${args.key} is held${args.holder === undefined ? '' : ` by ${args.holder}`}` +
|
|
209
|
+
' — one database has exactly one replicator',
|
|
210
|
+
fix: 'scale the replicator to 1 per database: kubectl scale deploy/replicator --replicas=1',
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* A hook was called before the app entry registered its client. Never a transient fault: the
|
|
217
|
+
* registration is a single call in the entry, so the fix is the call itself rather than a retry.
|
|
218
|
+
*/
|
|
219
|
+
export class LiveClientMissingError extends RealtimeError {
|
|
220
|
+
constructor(args: { hook: string }) {
|
|
221
|
+
super({
|
|
222
|
+
code: 'X_LIVE_CLIENT_MISSING',
|
|
223
|
+
cause: `${args.hook}() ran before any LiveClient was registered`,
|
|
224
|
+
fix: 'setLiveClient(new LiveClient({ signal: createSignal, connect, buildId })) in the app entry, above the first render',
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* A subscribable read projected a row with no `id`. Patches, cursors and the local store all
|
|
231
|
+
* address a row by `id`, so such a row cannot be delivered — and delivering it anyway produces a
|
|
232
|
+
* subscription that looks correct until the first update nobody can apply.
|
|
233
|
+
*/
|
|
234
|
+
export class LiveRowUnidentifiedError extends RealtimeError {
|
|
235
|
+
constructor(args: { query: string; keys: readonly string[] }) {
|
|
236
|
+
super({
|
|
237
|
+
code: 'X_LIVE_ROW_UNIDENTIFIED',
|
|
238
|
+
cause: `live query "${args.query}" returned a row with no id (columns: ${args.keys.join(', ') || 'none'})`,
|
|
239
|
+
fix: `select the primary key in ${args.query}'s sql(), or drop live: true from it`,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Deep infrastructure that is interface-complete but not wired. Carries the exact next step. */
|
|
245
|
+
export class NotImplementedError extends RealtimeError {
|
|
246
|
+
constructor(args: { what: string; fix: string }) {
|
|
247
|
+
super({
|
|
248
|
+
code: 'X_NOT_IMPLEMENTED',
|
|
249
|
+
cause: `${args.what} is interface-complete but not implemented in this build`,
|
|
250
|
+
fix: args.fix,
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
}
|
package/src/fanout.ts
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// Cross-node fanout. This interface is the reason `sync` nodes are stateless: every piece of state
|
|
2
|
+
// that must survive one node's death (subscription routing, presence) lives behind `Transport`,
|
|
3
|
+
// never in a node's heap. Subjects are NATS-shaped (`x.change.posts.org-1`) so the in-process
|
|
4
|
+
// default and the real bus route identically.
|
|
5
|
+
|
|
6
|
+
import { type Clock, systemClock } from '@ultimat3/core';
|
|
7
|
+
import { TransportUnavailableError } from './errors';
|
|
8
|
+
|
|
9
|
+
export type TransportHandler = (payload: string, subject: string) => void;
|
|
10
|
+
|
|
11
|
+
export interface TransportSubscription {
|
|
12
|
+
readonly subject: string;
|
|
13
|
+
unsubscribe(): void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface TransportSetEntry {
|
|
17
|
+
readonly member: string;
|
|
18
|
+
readonly value: string;
|
|
19
|
+
readonly expiresAt: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* TTL'd keyed sets on the bus (NATS KV / Redis hashes). Presence uses this so a lost node loses
|
|
24
|
+
* nothing: members simply stop heartbeating and expire on their own.
|
|
25
|
+
*/
|
|
26
|
+
export interface TransportSet {
|
|
27
|
+
put(key: string, member: string, value: string, ttlMs: number): Promise<void>;
|
|
28
|
+
/** `false` when the member had already expired — the caller must re-`put` (that is a re-join). */
|
|
29
|
+
touch(key: string, member: string, ttlMs: number): Promise<boolean>;
|
|
30
|
+
drop(key: string, member: string): Promise<void>;
|
|
31
|
+
entries(key: string): Promise<readonly TransportSetEntry[]>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface Transport {
|
|
35
|
+
readonly name: string;
|
|
36
|
+
publish(subject: string, payload: string): Promise<void>;
|
|
37
|
+
subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription>;
|
|
38
|
+
readonly shared: TransportSet;
|
|
39
|
+
close(): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** NATS subject semantics: `*` matches one token, `>` matches one-or-more trailing tokens. */
|
|
43
|
+
export function subjectMatches(pattern: string, subject: string): boolean {
|
|
44
|
+
const p = pattern.split('.');
|
|
45
|
+
const s = subject.split('.');
|
|
46
|
+
for (let i = 0; i < p.length; i += 1) {
|
|
47
|
+
const token = p[i];
|
|
48
|
+
if (token === '>') return s.length > i;
|
|
49
|
+
if (i >= s.length) return false;
|
|
50
|
+
if (token !== '*' && token !== s[i]) return false;
|
|
51
|
+
}
|
|
52
|
+
return p.length === s.length;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface InProcessTransportOptions {
|
|
56
|
+
readonly clock?: Clock;
|
|
57
|
+
/** A failing subscriber must not break fanout for the others. */
|
|
58
|
+
readonly onError?: (error: unknown, subject: string) => void;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
class InProcessSet implements TransportSet {
|
|
62
|
+
readonly #keys = new Map<string, Map<string, TransportSetEntry>>();
|
|
63
|
+
readonly #clock: Clock;
|
|
64
|
+
|
|
65
|
+
constructor(clock: Clock) {
|
|
66
|
+
this.#clock = clock;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async put(key: string, member: string, value: string, ttlMs: number): Promise<void> {
|
|
70
|
+
const bucket = this.#keys.get(key) ?? new Map<string, TransportSetEntry>();
|
|
71
|
+
bucket.set(member, { member, value, expiresAt: this.#clock.now().getTime() + ttlMs });
|
|
72
|
+
this.#keys.set(key, bucket);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async touch(key: string, member: string, ttlMs: number): Promise<boolean> {
|
|
76
|
+
const entry = this.#keys.get(key)?.get(member);
|
|
77
|
+
if (!entry || entry.expiresAt <= this.#clock.now().getTime()) return false;
|
|
78
|
+
await this.put(key, member, entry.value, ttlMs);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async drop(key: string, member: string): Promise<void> {
|
|
83
|
+
const bucket = this.#keys.get(key);
|
|
84
|
+
bucket?.delete(member);
|
|
85
|
+
if (bucket && bucket.size === 0) this.#keys.delete(key);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async entries(key: string): Promise<readonly TransportSetEntry[]> {
|
|
89
|
+
const bucket = this.#keys.get(key);
|
|
90
|
+
if (!bucket) return [];
|
|
91
|
+
const now = this.#clock.now().getTime();
|
|
92
|
+
const live: TransportSetEntry[] = [];
|
|
93
|
+
for (const entry of bucket.values()) {
|
|
94
|
+
if (entry.expiresAt > now) live.push(entry);
|
|
95
|
+
else bucket.delete(entry.member);
|
|
96
|
+
}
|
|
97
|
+
return live;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Single-node default: `x dev`, tests, and small deployments that have not earned a bus yet. */
|
|
102
|
+
export class InProcessTransport implements Transport {
|
|
103
|
+
readonly name = 'in-process';
|
|
104
|
+
readonly shared: TransportSet;
|
|
105
|
+
readonly #handlers = new Map<string, Set<TransportHandler>>();
|
|
106
|
+
readonly #onError: (error: unknown, subject: string) => void;
|
|
107
|
+
#closed = false;
|
|
108
|
+
|
|
109
|
+
constructor(options: InProcessTransportOptions = {}) {
|
|
110
|
+
this.shared = new InProcessSet(options.clock ?? systemClock);
|
|
111
|
+
this.#onError = options.onError ?? (() => undefined);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async publish(subject: string, payload: string): Promise<void> {
|
|
115
|
+
this.#assertOpen();
|
|
116
|
+
for (const [pattern, handlers] of this.#handlers) {
|
|
117
|
+
if (!subjectMatches(pattern, subject)) continue;
|
|
118
|
+
for (const handler of handlers) {
|
|
119
|
+
try {
|
|
120
|
+
handler(payload, subject);
|
|
121
|
+
} catch (error) {
|
|
122
|
+
this.#onError(error, subject);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription> {
|
|
129
|
+
this.#assertOpen();
|
|
130
|
+
const handlers = this.#handlers.get(subject) ?? new Set<TransportHandler>();
|
|
131
|
+
handlers.add(handler);
|
|
132
|
+
this.#handlers.set(subject, handlers);
|
|
133
|
+
return {
|
|
134
|
+
subject,
|
|
135
|
+
unsubscribe: () => {
|
|
136
|
+
handlers.delete(handler);
|
|
137
|
+
if (handlers.size === 0) this.#handlers.delete(subject);
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async close(): Promise<void> {
|
|
143
|
+
this.#closed = true;
|
|
144
|
+
this.#handlers.clear();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
get subjectCount(): number {
|
|
148
|
+
return this.#handlers.size;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
#assertOpen(): void {
|
|
152
|
+
if (this.#closed) {
|
|
153
|
+
throw new TransportUnavailableError({ transport: this.name, reason: 'transport is closed' });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// The production bus lives in `nats-transport.ts`: it needs a socket, a codec and a JetStream
|
|
159
|
+
// client, none of which belong in the file that defines what a transport *is*.
|