@ultimat3/realtime 23.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +75 -81
- package/README.md +107 -17
- package/package.json +4 -4
- package/src/advisory-lock.ts +57 -0
- package/src/boot.ts +12 -7
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +2 -2
- package/src/changefeed.ts +39 -1
- package/src/channel-authz.ts +43 -12
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +34 -4
- package/src/channel-records.ts +33 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +143 -44
- package/src/client-channels.ts +10 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +7 -1
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +1 -0
- package/src/cursor.ts +11 -0
- package/src/errors.ts +70 -19
- package/src/fanout.ts +13 -0
- package/src/index.ts +1 -1
- package/src/live-definition.ts +22 -7
- package/src/live-fanout.ts +43 -7
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +38 -0
- package/src/live-query.ts +36 -37
- package/src/nats-client.ts +11 -2
- package/src/nats-transport.ts +22 -0
- package/src/offline-queue.ts +48 -5
- package/src/page-errors.ts +17 -0
- package/src/page-outbox.ts +121 -50
- package/src/pg-advisory-lock.ts +45 -1
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +119 -21
- package/src/pg-entity-row.ts +16 -0
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +44 -45
- package/src/pg-socket.ts +6 -4
- package/src/pg-tls.ts +18 -0
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/presence.ts +85 -8
- package/src/query-window.ts +16 -4
- package/src/record-persister.ts +7 -2
- package/src/replication-errors.ts +8 -4
- package/src/replicator-envelope.ts +110 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +319 -186
- package/src/server.ts +10 -6
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/subscriber-gate.ts +11 -1
- package/src/sync-auth.ts +47 -2
- package/src/sync-frames.ts +73 -20
- package/src/sync-node-bounds.ts +15 -0
- package/src/sync-node-contract.ts +20 -2
- package/src/sync-node.ts +92 -17
- package/src/sync-origin.ts +26 -11
- package/src/sync-protocol.ts +13 -3
- package/src/sync-upgrade.ts +76 -8
- package/src/use-mutation.ts +33 -5
- package/src/wire-channel.ts +13 -4
package/src/live-definition.ts
CHANGED
|
@@ -27,8 +27,10 @@ export interface LiveDefinitionOptions {
|
|
|
27
27
|
*/
|
|
28
28
|
readonly ctx: Ctx;
|
|
29
29
|
/**
|
|
30
|
-
* Where the shared window sits in the change stream, asked at snapshot time
|
|
31
|
-
*
|
|
30
|
+
* Where the shared window sits in the change stream, asked at snapshot time — the node's newest
|
|
31
|
+
* received change (`registry.lastLsn`). Omitted, or answering `''` before the first change, the
|
|
32
|
+
* snapshot claims no position and the retained ring is floored by its first patch instead: a
|
|
33
|
+
* cursor minted before that patch re-snapshots, one that received it resumes as a delta.
|
|
32
34
|
*/
|
|
33
35
|
readonly lsn?: () => string;
|
|
34
36
|
/** Pins the reconnect epoch in tests; the server derives it from the build. */
|
|
@@ -165,6 +167,9 @@ export function liveQueryDefinition(
|
|
|
165
167
|
// The position is taken BEFORE the rows: the rows are then at least that new, so the claim
|
|
166
168
|
// is true. Taken after, a commit landing mid-read was claimed and missing — and every change
|
|
167
169
|
// up to it is dropped downstream as already folded.
|
|
170
|
+
// `''` when the node holds no position — no `lsn` option, or no change received yet. Never
|
|
171
|
+
// a made-up one: a position the node never held becomes a ring floor every foreign cursor
|
|
172
|
+
// is "inside", and the client that resumed there kept rows this node never patched.
|
|
168
173
|
const lsn = options.lsn?.() ?? '';
|
|
169
174
|
return { rows: await readFor(options.ctx, tenant, () => window.read()), lsn };
|
|
170
175
|
},
|
|
@@ -191,15 +196,25 @@ function evictOldest(windows: Map<string, SharedWindow>, max: number): void {
|
|
|
191
196
|
}
|
|
192
197
|
|
|
193
198
|
/**
|
|
194
|
-
* Rows crossing the wire are addressed by `id` — patches, cursors and the local store all
|
|
195
|
-
* it. A
|
|
196
|
-
*
|
|
199
|
+
* Rows crossing the wire are addressed by a TEXT `id` — patches, cursors and the local store all
|
|
200
|
+
* key on it. A safe-integer id is stringified, the one normalisation the change feed applies to the
|
|
201
|
+
* same row (`pg-replication.ts`'s `toRow`), so a snapshot row and its later patch name one identity.
|
|
202
|
+
* Anything else is refused here rather than delivered as a row nobody can patch.
|
|
197
203
|
*/
|
|
198
204
|
function rowsOf(query: string, rows: readonly object[]): readonly Row[] {
|
|
199
205
|
const out: Row[] = [];
|
|
200
206
|
for (const row of rows) {
|
|
201
|
-
|
|
202
|
-
|
|
207
|
+
const id: unknown = (row as { id?: unknown }).id;
|
|
208
|
+
const candidate =
|
|
209
|
+
typeof id === 'number' && Number.isSafeInteger(id) ? { ...row, id: String(id) } : row;
|
|
210
|
+
if (!isRow(candidate)) {
|
|
211
|
+
throw new LiveRowUnidentifiedError({
|
|
212
|
+
query,
|
|
213
|
+
keys: Object.keys(row),
|
|
214
|
+
...(id === undefined || id === null ? {} : { idType: typeof id }),
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
out.push(candidate);
|
|
203
218
|
}
|
|
204
219
|
return out;
|
|
205
220
|
}
|
package/src/live-fanout.ts
CHANGED
|
@@ -8,7 +8,7 @@ import type { ChangeEvent } from './changefeed';
|
|
|
8
8
|
import { advance, type LiveCursor, makeCursor, type ResumeSource } from './cursor';
|
|
9
9
|
import type { Row, RowPatch } from './json';
|
|
10
10
|
import type { LiveSubscription } from './live-contract';
|
|
11
|
-
import { applyToWindow, bridgeChange } from './matcher-bridge';
|
|
11
|
+
import { applyToWindow, bridgeChange, type Projection, projectionOf } from './matcher-bridge';
|
|
12
12
|
import { type QueryEntry, refillWindowInLane } from './query-window';
|
|
13
13
|
import { type Subscriber, type SubscriberGate, windowIndex } from './subscriber-gate';
|
|
14
14
|
import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
@@ -50,6 +50,8 @@ export async function fanoutChange(
|
|
|
50
50
|
const reread = entry.stale;
|
|
51
51
|
if (reread) {
|
|
52
52
|
await refillWindowInLane(entry);
|
|
53
|
+
// What was retained before this read has a hole exactly where the staleness came from.
|
|
54
|
+
deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
|
|
53
55
|
// Every subscriber was fed from the rows this read just replaced, so each is owed a snapshot —
|
|
54
56
|
// not only the ones `invalidate()` marked: a read that failed or timed out stales the window
|
|
55
57
|
// and marks nobody.
|
|
@@ -72,19 +74,26 @@ export async function fanoutChange(
|
|
|
72
74
|
if (!bridged) return { sent: reread ? await resnapshotAll(deps, entry) : 0, stale: 0 };
|
|
73
75
|
// Keyed ONCE, here, before the retained window stores them: a resume replays the same key.
|
|
74
76
|
const result = { ...bridged, patches: keyPatches(entry, change, bridged.patches) };
|
|
77
|
+
// Asked of the window as it stood BEFORE this change: the partial row must not teach it a shape.
|
|
78
|
+
const partial = lacksOmitted(change, result.patches, projectionOf(entry.rows));
|
|
75
79
|
entry.lsn = change.lsn;
|
|
76
80
|
entry.rows = applyToWindow(entry.rows, result.patches);
|
|
77
|
-
// The window lost its tail
|
|
78
|
-
//
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
// The window lost its tail — or adopted a row the change could not carry whole — so what it
|
|
82
|
+
// holds is a guess: the next delivery re-reads it rather than patching a guess, and every
|
|
83
|
+
// subscriber below is re-snapshotted out of what that returns.
|
|
84
|
+
const guessed = result.refill || partial;
|
|
85
|
+
if (guessed) entry.stale = true;
|
|
86
|
+
// The retained window holds the pre-policy patch; resume re-filters it per subscriber. A partial
|
|
87
|
+
// row is never retained: replayed, it is the same missing column on a resuming client.
|
|
88
|
+
if (!partial) for (const patch of result.patches) deps.source.append(entry.qid, patch);
|
|
89
|
+
// And the ring says so now, not at the re-read: a resume in between got a delta lacking it.
|
|
90
|
+
else deps.source.floorAt?.(entry.qid, change.lsn, { exclusive: true });
|
|
82
91
|
|
|
83
92
|
let sent = 0;
|
|
84
93
|
// Indexed once for every subscriber below, never searched per subscriber per patch.
|
|
85
94
|
const index = windowIndex(entry.rows);
|
|
86
95
|
for (const subscription of entry.subscribers.values()) {
|
|
87
|
-
if (
|
|
96
|
+
if (guessed) {
|
|
88
97
|
// The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
|
|
89
98
|
// the mark, because a repair reads `entry.rows` — which this fanout has just declared a
|
|
90
99
|
// guess — and then CLEARS the mark. A subscriber already diverged would be recorded as
|
|
@@ -139,6 +148,32 @@ export async function fanoutChange(
|
|
|
139
148
|
return { sent, stale: 0 };
|
|
140
149
|
}
|
|
141
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Whether a row ENTERS the window without a column the change did not carry
|
|
153
|
+
* (`ChangeEvent.omitted`: an unchanged out-of-line value Postgres logged no bytes for). An update
|
|
154
|
+
* patch is safe — it merges onto a window row that already holds the column — but an add adopts
|
|
155
|
+
* the row as the whole row, into the window, every later snapshot and the retained ring. An
|
|
156
|
+
* unread projection (an empty window) cannot rule a column out, so every omitted one counts.
|
|
157
|
+
*/
|
|
158
|
+
function lacksOmitted(
|
|
159
|
+
change: ChangeEvent,
|
|
160
|
+
patches: readonly RowPatch[],
|
|
161
|
+
projection: Projection,
|
|
162
|
+
): boolean {
|
|
163
|
+
const omitted = change.omitted;
|
|
164
|
+
if (omitted === undefined || omitted.length === 0) return false;
|
|
165
|
+
return patches.some(
|
|
166
|
+
(patch) =>
|
|
167
|
+
patch.op === 'insert' &&
|
|
168
|
+
patch.row !== null &&
|
|
169
|
+
omitted.some(
|
|
170
|
+
(column) =>
|
|
171
|
+
(projection === undefined || projection.has(column)) &&
|
|
172
|
+
!Object.hasOwn(patch.row ?? {}, column),
|
|
173
|
+
),
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
142
177
|
/**
|
|
143
178
|
* Every row of a relation this window reads is gone. There is nothing to patch from — a truncate
|
|
144
179
|
* names no row — so the window is re-read now, in the lane, and every subscriber is re-snapshotted
|
|
@@ -153,6 +188,7 @@ async function truncated(
|
|
|
153
188
|
if (!entry.shape.entities.includes(change.entity)) return { sent: 0, stale: 0 };
|
|
154
189
|
await refillWindowInLane(entry);
|
|
155
190
|
if (change.lsn > entry.lsn) entry.lsn = change.lsn;
|
|
191
|
+
deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
|
|
156
192
|
for (const subscription of entry.subscribers.values()) {
|
|
157
193
|
subscription.socket.markDesynced(subscription.sid);
|
|
158
194
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// What a window read that just returned owes the retained ring, and the position the subscriber it
|
|
2
|
+
// serves is told. The rule itself is `change-buffer.ts`'s (`Ring`); this is its one caller for a
|
|
3
|
+
// read, split from `live-query.ts` at that file's size ceiling.
|
|
4
|
+
|
|
5
|
+
import type { ResumeSource } from './cursor';
|
|
6
|
+
import type { FilledWindow, QueryEntry } from './query-window';
|
|
7
|
+
|
|
8
|
+
export interface NodePosition {
|
|
9
|
+
/** The newest change this node has received; `''` before the first. */
|
|
10
|
+
readonly lastLsn: string;
|
|
11
|
+
/** This node's own mark for a read with no position behind it. Sorts below every real lsn. */
|
|
12
|
+
readonly origin: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Synchronous on purpose: the caller awaits a policy pass next, and the position has to be the
|
|
17
|
+
* one these ROWS were read at — read after that await, a change folded in meanwhile would be
|
|
18
|
+
* claimed by a snapshot that does not hold it.
|
|
19
|
+
*/
|
|
20
|
+
export function floorAfterRead(
|
|
21
|
+
source: ResumeSource,
|
|
22
|
+
entry: QueryEntry,
|
|
23
|
+
window: FilledWindow,
|
|
24
|
+
node: NodePosition,
|
|
25
|
+
): string {
|
|
26
|
+
// Served out of a window that was already filled: its patch history is still true.
|
|
27
|
+
if (window.landed === null) return window.lsn;
|
|
28
|
+
// A read with no position behind it is marked with this node's own origin, so the cursor it
|
|
29
|
+
// mints is one no other node could have issued.
|
|
30
|
+
const lsn = window.lsn === '' ? node.origin : window.lsn;
|
|
31
|
+
if (entry.lsn === '') entry.lsn = lsn;
|
|
32
|
+
if (window.landed === 'forced') source.floorAt?.(entry.qid, lsn, { exclusive: true });
|
|
33
|
+
else source.floorAt?.(entry.qid, lsn, { sole: node.lastLsn === '' || window.lsn === '' });
|
|
34
|
+
return lsn;
|
|
35
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// What a `LiveQueryRegistry` is built from, and the one ceiling it defaults. Split from
|
|
2
|
+
// `live-query.ts` at that file's size ceiling: the registry owns behaviour, this owns its knobs.
|
|
3
|
+
|
|
4
|
+
import type { Actor, Clock } from '@ultimat3/core';
|
|
5
|
+
import type { ReconnectBudget, ResumeSource } from './cursor';
|
|
6
|
+
import type { SubscriberGateOptions } from './subscriber-gate';
|
|
7
|
+
import type { Scheduler } from './thundering-herd';
|
|
8
|
+
|
|
9
|
+
export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
|
|
10
|
+
readonly source: ResumeSource;
|
|
11
|
+
readonly budget?: ReconnectBudget;
|
|
12
|
+
readonly clock?: Clock;
|
|
13
|
+
readonly maxPerSocket?: number;
|
|
14
|
+
readonly maxPerTenant?: number;
|
|
15
|
+
readonly tenantOf?: (actor: Actor | null) => string | null;
|
|
16
|
+
/**
|
|
17
|
+
* Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
|
|
18
|
+
* client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
|
|
19
|
+
* a `WindowLock` and a fanout target each — until the process dies.
|
|
20
|
+
*/
|
|
21
|
+
readonly maxEntries?: number;
|
|
22
|
+
/**
|
|
23
|
+
* How long one entry's SHARED snapshot read may hold its slot. Defaults to
|
|
24
|
+
* `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
|
|
25
|
+
* slot for the life of the process and every later cold subscriber joined a promise nothing
|
|
26
|
+
* would resolve — one wedged read taking every future subscriber of that query id with it.
|
|
27
|
+
*/
|
|
28
|
+
readonly readDeadlineMs?: number;
|
|
29
|
+
/** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
|
|
30
|
+
readonly schedule?: Scheduler;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
|
|
35
|
+
* `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
|
|
36
|
+
* cost this bounds is the entry, not the subscriber.
|
|
37
|
+
*/
|
|
38
|
+
export const DEFAULT_MAX_ENTRIES = 10_000;
|
package/src/live-query.ts
CHANGED
|
@@ -6,54 +6,26 @@
|
|
|
6
6
|
// policy is never sent to that actor — it arrives as a `delete` if they hold it, and is dropped
|
|
7
7
|
// otherwise.
|
|
8
8
|
|
|
9
|
-
import { type
|
|
9
|
+
import { type Clock, finiteOption, systemClock, uuid } from '@ultimat3/core';
|
|
10
10
|
import { queryHash } from '@ultimat3/query';
|
|
11
11
|
import type { ChangeEvent } from './changefeed';
|
|
12
|
-
import { type LiveCursor, makeCursor
|
|
12
|
+
import { type LiveCursor, makeCursor } from './cursor';
|
|
13
13
|
import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
|
|
14
14
|
import type { JsonValue } from './json';
|
|
15
15
|
import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
|
|
16
16
|
import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
|
|
17
|
+
import { floorAfterRead } from './live-floor';
|
|
18
|
+
import { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
|
|
17
19
|
import { refuseSubscription } from './live-refusal';
|
|
18
20
|
import { resumeOnto } from './live-resume';
|
|
19
21
|
import { liveTenantOf, windowId } from './live-tenant';
|
|
20
22
|
import { createEntry, fillWindow, type QueryEntry } from './query-window';
|
|
21
23
|
import type { SyncSocket } from './socket';
|
|
22
|
-
import { type Subscriber, SubscriberGate
|
|
24
|
+
import { type Subscriber, SubscriberGate } from './subscriber-gate';
|
|
23
25
|
import { SubscriptionBook, subscriptionKey } from './subscription-book';
|
|
24
26
|
import type { Frame } from './sync-protocol';
|
|
25
|
-
import type { Scheduler } from './thundering-herd';
|
|
26
27
|
|
|
27
|
-
export
|
|
28
|
-
readonly source: ResumeSource;
|
|
29
|
-
readonly budget?: ReconnectBudget;
|
|
30
|
-
readonly clock?: Clock;
|
|
31
|
-
readonly maxPerSocket?: number;
|
|
32
|
-
readonly maxPerTenant?: number;
|
|
33
|
-
readonly tenantOf?: (actor: Actor | null) => string | null;
|
|
34
|
-
/**
|
|
35
|
-
* Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
|
|
36
|
-
* client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
|
|
37
|
-
* a `WindowLock` and a fanout target each — until the process dies.
|
|
38
|
-
*/
|
|
39
|
-
readonly maxEntries?: number;
|
|
40
|
-
/**
|
|
41
|
-
* How long one entry's SHARED snapshot read may hold its slot. Defaults to
|
|
42
|
-
* `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
|
|
43
|
-
* slot for the life of the process and every later cold subscriber joined a promise nothing
|
|
44
|
-
* would resolve — one wedged read taking every future subscriber of that query id with it.
|
|
45
|
-
*/
|
|
46
|
-
readonly readDeadlineMs?: number;
|
|
47
|
-
/** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
|
|
48
|
-
readonly schedule?: Scheduler;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
|
|
53
|
-
* `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
|
|
54
|
-
* cost this bounds is the entry, not the subscriber.
|
|
55
|
-
*/
|
|
56
|
-
export const DEFAULT_MAX_ENTRIES = 10_000;
|
|
28
|
+
export { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
|
|
57
29
|
|
|
58
30
|
export class LiveQueryRegistry {
|
|
59
31
|
readonly #definitions = new Map<string, LiveQueryDefinition>();
|
|
@@ -67,6 +39,11 @@ export class LiveQueryRegistry {
|
|
|
67
39
|
/** What one lane needs, and nothing this class holds beyond it. */
|
|
68
40
|
readonly #fanout: FanoutDeps;
|
|
69
41
|
#lastLsn = '';
|
|
42
|
+
/**
|
|
43
|
+
* This node's mark for a window read before it held any position: unique to the process, and
|
|
44
|
+
* `!` sorts below every hex digit, so every real change is above it.
|
|
45
|
+
*/
|
|
46
|
+
readonly #origin = `!${uuid()}`;
|
|
70
47
|
#staleChanges = 0;
|
|
71
48
|
|
|
72
49
|
constructor(options: LiveQueryRegistryOptions) {
|
|
@@ -166,7 +143,15 @@ export class LiveQueryRegistry {
|
|
|
166
143
|
// dispatched concurrently, N of them reading a count nothing had grown yet.
|
|
167
144
|
const slot = this.#book.reserve(args.socket, sid);
|
|
168
145
|
try {
|
|
169
|
-
|
|
146
|
+
for (;;) {
|
|
147
|
+
// `authorize`, the read and the row pass all await, and a re-auth that lands under them
|
|
148
|
+
// re-decides only what is ATTACHED. So a subscription seated for an actor the socket no
|
|
149
|
+
// longer carries is taken back and served again under the one it does.
|
|
150
|
+
const actor = args.socket.actor;
|
|
151
|
+
const served = await this.#subscribeReserved(definition, sid, args);
|
|
152
|
+
if (args.socket.actor === actor) return served;
|
|
153
|
+
this.unsubscribe(args.socket.id, sid);
|
|
154
|
+
}
|
|
170
155
|
} finally {
|
|
171
156
|
// After the attach on every path, so the slot is only ever given back to a count that has
|
|
172
157
|
// already grown — or, on a failure, to one that never will.
|
|
@@ -223,7 +208,10 @@ export class LiveQueryRegistry {
|
|
|
223
208
|
// replays. One that names another window — the same browser, signed in to another org, or a
|
|
224
209
|
// forged frame — is not a position in this one: it is a cold start, never a replay of a ring
|
|
225
210
|
// this subscriber's window does not own.
|
|
226
|
-
|
|
211
|
+
//
|
|
212
|
+
// Nor is ANY cursor a position in a window known to have missed a change: the ring still
|
|
213
|
+
// answers a delta for it, and that delta is the hole. A stale entry serves cold starts only.
|
|
214
|
+
if (args.cursor && args.cursor.qid === qid && !entry.stale) {
|
|
227
215
|
const who = { sid, actor: args.socket.actor };
|
|
228
216
|
const resumed = await resumeOnto(
|
|
229
217
|
{
|
|
@@ -319,6 +307,9 @@ export class LiveQueryRegistry {
|
|
|
319
307
|
} catch (error) {
|
|
320
308
|
if (isPolicyDenial(error)) {
|
|
321
309
|
this.unsubscribe(socket.id, subscription.sid);
|
|
310
|
+
// Said under the sid, as a refused subscribe is: unsaid, the client kept the rows on
|
|
311
|
+
// screen in state `live` for a subscription this node no longer serves.
|
|
312
|
+
refuseSubscription(socket, subscription.sid, error);
|
|
322
313
|
dropped.push(subscription.sid);
|
|
323
314
|
continue;
|
|
324
315
|
}
|
|
@@ -433,6 +424,10 @@ export class LiveQueryRegistry {
|
|
|
433
424
|
cursor: LiveCursor,
|
|
434
425
|
): LiveSubscription {
|
|
435
426
|
const subscription = this.#attach(entry, socket, sid, cursor);
|
|
427
|
+
// The policy pass ran outside the lane, so a change may have been folded into the window
|
|
428
|
+
// between the rows this subscriber was served and this attach: it is in neither their frame
|
|
429
|
+
// nor their patch stream. Marked, so the next delivery re-snapshots them out of the window.
|
|
430
|
+
if (entry.lsn !== cursor.lsn) socket.markDesynced(sid);
|
|
436
431
|
if (socket.closed) this.unsubscribe(socket.id, sid);
|
|
437
432
|
return subscription;
|
|
438
433
|
}
|
|
@@ -495,6 +490,10 @@ export class LiveQueryRegistry {
|
|
|
495
490
|
*/
|
|
496
491
|
async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
|
|
497
492
|
const window = await fillWindow(entry);
|
|
498
|
-
|
|
493
|
+
const lsn = floorAfterRead(this.#options.source, entry, window, {
|
|
494
|
+
lastLsn: this.#lastLsn,
|
|
495
|
+
origin: this.#origin,
|
|
496
|
+
});
|
|
497
|
+
return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn };
|
|
499
498
|
}
|
|
500
499
|
}
|
package/src/nats-client.ts
CHANGED
|
@@ -81,6 +81,8 @@ export interface NatsClientOptions {
|
|
|
81
81
|
export type NatsConnect = (options: NatsClientOptions) => Promise<NatsClient>;
|
|
82
82
|
|
|
83
83
|
export const DEFAULT_NATS_PORT = 4222;
|
|
84
|
+
|
|
85
|
+
const NATS_URL_FIX = 'export NATS_URL=nats://user:password@host:4222';
|
|
84
86
|
export const DEFAULT_REQUEST_TIMEOUT_MS = 5_000;
|
|
85
87
|
|
|
86
88
|
/**
|
|
@@ -92,9 +94,12 @@ export function parseNatsUrl(url: string): NatsTarget {
|
|
|
92
94
|
try {
|
|
93
95
|
parsed = new URL(url);
|
|
94
96
|
} catch {
|
|
97
|
+
// The variable, never its value: a bus URL carries the credential, and an error is the one
|
|
98
|
+
// thing that reaches a log, `--json`, an agent transcript and a ticket. `parsePgUrl`'s rule.
|
|
95
99
|
throw new TransportUnavailableError({
|
|
96
100
|
transport: 'nats',
|
|
97
|
-
reason:
|
|
101
|
+
reason: 'NATS_URL is not a connection URL',
|
|
102
|
+
fix: NATS_URL_FIX,
|
|
98
103
|
});
|
|
99
104
|
}
|
|
100
105
|
if (parsed.protocol !== 'nats:' && parsed.protocol !== 'tls:') {
|
|
@@ -104,7 +109,11 @@ export function parseNatsUrl(url: string): NatsTarget {
|
|
|
104
109
|
});
|
|
105
110
|
}
|
|
106
111
|
if (parsed.hostname === '') {
|
|
107
|
-
throw new TransportUnavailableError({
|
|
112
|
+
throw new TransportUnavailableError({
|
|
113
|
+
transport: 'nats',
|
|
114
|
+
reason: 'NATS_URL has no host',
|
|
115
|
+
fix: NATS_URL_FIX,
|
|
116
|
+
});
|
|
108
117
|
}
|
|
109
118
|
const hasUser = parsed.username !== '';
|
|
110
119
|
const hasPass = parsed.password !== '';
|
package/src/nats-transport.ts
CHANGED
|
@@ -55,6 +55,7 @@ export class NatsTransport implements Transport {
|
|
|
55
55
|
#dialing: Promise<NatsClient> | undefined;
|
|
56
56
|
#retries = 0;
|
|
57
57
|
#closed = false;
|
|
58
|
+
readonly #reconnectListeners = new Set<() => void>();
|
|
58
59
|
|
|
59
60
|
constructor(options: NatsTransportOptions) {
|
|
60
61
|
// Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
|
|
@@ -116,6 +117,19 @@ export class NatsTransport implements Transport {
|
|
|
116
117
|
return { subject, unsubscribe: () => live.unsubscribe() };
|
|
117
118
|
}
|
|
118
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Told every time the client comes back from a drop. Fanout is at-most-once, so the gap between
|
|
122
|
+
* the drop and this call is changes the subscriber never saw — and with no later message there
|
|
123
|
+
* is no sequence number to notice it by. A `sync` node invalidates its windows on it. Returns
|
|
124
|
+
* the unsubscribe.
|
|
125
|
+
*/
|
|
126
|
+
onReconnect(listener: () => void): () => void {
|
|
127
|
+
this.#reconnectListeners.add(listener);
|
|
128
|
+
return () => {
|
|
129
|
+
this.#reconnectListeners.delete(listener);
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
119
133
|
async close(): Promise<void> {
|
|
120
134
|
this.#closed = true;
|
|
121
135
|
const client = this.#client;
|
|
@@ -200,7 +214,15 @@ export class NatsTransport implements Transport {
|
|
|
200
214
|
this.#retries = 0;
|
|
201
215
|
const client = this.#client;
|
|
202
216
|
if (client === undefined) return;
|
|
217
|
+
// Not awaited: the library calls this from its status loop. The failure has one place to go.
|
|
203
218
|
void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
|
|
219
|
+
for (const listener of [...this.#reconnectListeners]) {
|
|
220
|
+
try {
|
|
221
|
+
listener();
|
|
222
|
+
} catch (error) {
|
|
223
|
+
this.#report(error, this.name);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
204
226
|
}
|
|
205
227
|
|
|
206
228
|
/**
|
package/src/offline-queue.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { renderThrowable, stringField } from '@ultimat3/core/page';
|
|
13
13
|
import type { JsonValue } from './json';
|
|
14
|
+
import { OfflineQueueAbandonedError } from './page-errors';
|
|
14
15
|
import type { WireError } from './sync-protocol';
|
|
15
16
|
|
|
16
17
|
export type MutationStatus = 'pending' | 'inflight' | 'acked' | 'failed';
|
|
@@ -91,6 +92,8 @@ export class OfflineQueue {
|
|
|
91
92
|
* is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
|
|
92
93
|
*/
|
|
93
94
|
#epoch = 0;
|
|
95
|
+
/** Set by `abandon`, never cleared: this queue's principal is gone. */
|
|
96
|
+
#abandoned = false;
|
|
94
97
|
|
|
95
98
|
private constructor(store: QueueStore, state: QueueState) {
|
|
96
99
|
this.#store = store;
|
|
@@ -103,8 +106,10 @@ export class OfflineQueue {
|
|
|
103
106
|
*
|
|
104
107
|
* An `inflight` entry on disk belonged to a page that is gone, so it goes back to `pending`. Left
|
|
105
108
|
* as it was, `#sendable` skipped it forever — no ack was coming to a page that no longer exists —
|
|
106
|
-
* and every later write overtook it. The replay carries its idempotency key,
|
|
107
|
-
*
|
|
109
|
+
* and every later write overtook it. The replay carries its idempotency key, and a `mutator()`
|
|
110
|
+
* is REQUIRED to declare `idempotent: true` (`X_MUTATOR_NOT_IDEMPOTENT` at declaration), so a
|
|
111
|
+
* write the old page did get through is answered from the action's idempotency store and is
|
|
112
|
+
* never applied twice. That requirement is what makes this resend safe, not the key alone.
|
|
108
113
|
*/
|
|
109
114
|
static async open(store: QueueStore): Promise<OfflineQueue> {
|
|
110
115
|
const queue = new OfflineQueue(store, await store.load());
|
|
@@ -119,6 +124,7 @@ export class OfflineQueue {
|
|
|
119
124
|
* other pass can have one on the wire.
|
|
120
125
|
*/
|
|
121
126
|
async reload(): Promise<void> {
|
|
127
|
+
if (this.#abandoned) return;
|
|
122
128
|
const state = await this.#store.load();
|
|
123
129
|
this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
|
|
124
130
|
this.#nextSeq = Math.max(this.#nextSeq, state.nextSeq);
|
|
@@ -167,6 +173,17 @@ export class OfflineQueue {
|
|
|
167
173
|
input: JsonValue;
|
|
168
174
|
at?: number;
|
|
169
175
|
}): Promise<QueuedMutation> {
|
|
176
|
+
// Read BEFORE the key is looked up, never between the lookup and the push: with the await in
|
|
177
|
+
// between, two enqueues of one key both found nothing and both pushed — one intent, two
|
|
178
|
+
// entries under one sequence number. Another tab of the same user may have taken sequence
|
|
179
|
+
// numbers since this one loaded, which is what the read is for.
|
|
180
|
+
//
|
|
181
|
+
// An abandoned queue REFUSES, before the read and after it (the principal can leave during
|
|
182
|
+
// it): a write accepted here would sit in memory, shown as queued, with nothing to send it.
|
|
183
|
+
if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
|
|
184
|
+
const floor = (await this.#store.load()).nextSeq;
|
|
185
|
+
if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
|
|
186
|
+
this.#nextSeq = Math.max(this.#nextSeq, floor);
|
|
170
187
|
const existing = this.find(args.key);
|
|
171
188
|
if (existing && existing.status !== 'failed') {
|
|
172
189
|
this.#collapsed += 1;
|
|
@@ -178,8 +195,6 @@ export class OfflineQueue {
|
|
|
178
195
|
// entry is dropped and this one takes a new sequence at the back of the queue.
|
|
179
196
|
// By identity: `existing` IS the entry for this key, found above — no second key comparison.
|
|
180
197
|
if (existing) this.#mutations = this.#mutations.filter((entry) => entry !== existing);
|
|
181
|
-
// Another tab of the same user may have taken sequence numbers since this one loaded.
|
|
182
|
-
this.#nextSeq = Math.max(this.#nextSeq, (await this.#store.load()).nextSeq);
|
|
183
198
|
const mutation: QueuedMutation = {
|
|
184
199
|
key: args.key,
|
|
185
200
|
seq: this.#nextSeq,
|
|
@@ -248,6 +263,22 @@ export class OfflineQueue {
|
|
|
248
263
|
return returned;
|
|
249
264
|
}
|
|
250
265
|
|
|
266
|
+
/**
|
|
267
|
+
* The principal this queue belongs to is gone (`page-outbox.ts` calls it on a `rescope`), and it
|
|
268
|
+
* is final: nothing more is SENT, WRITTEN or ACCEPTED (`enqueue` refuses). The epoch bump stops a pass parked
|
|
269
|
+
* inside `send` before its next entry — which would otherwise leave under the next principal's
|
|
270
|
+
* session — and the flag stops a pass that starts later, and every write-back: the store was
|
|
271
|
+
* wiped for this principal, and a pass that then saved what it touched put the writes back.
|
|
272
|
+
*/
|
|
273
|
+
abandon(): void {
|
|
274
|
+
this.#epoch += 1;
|
|
275
|
+
this.#abandoned = true;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
get abandoned(): boolean {
|
|
279
|
+
return this.#abandoned;
|
|
280
|
+
}
|
|
281
|
+
|
|
251
282
|
/** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
|
|
252
283
|
async ack(key: string): Promise<void> {
|
|
253
284
|
const mutation = this.find(key);
|
|
@@ -294,7 +325,7 @@ export class OfflineQueue {
|
|
|
294
325
|
/** One drain pass. Never called concurrently with itself — `drain` owns that. */
|
|
295
326
|
async #pass(send: MutationSender): Promise<DrainReport> {
|
|
296
327
|
const epoch = this.#epoch;
|
|
297
|
-
const sendable = this.#sendable();
|
|
328
|
+
const sendable = this.#abandoned ? [] : this.#sendable();
|
|
298
329
|
// Nothing to do: a pass chained behind one that already sent everything must not rewrite the
|
|
299
330
|
// durable state for the privilege of reporting zero.
|
|
300
331
|
if (sendable.length === 0) {
|
|
@@ -326,6 +357,12 @@ export class OfflineQueue {
|
|
|
326
357
|
// `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
|
|
327
358
|
// an ack drops the mutation on exactly the socket death this queue exists to survive.
|
|
328
359
|
// Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
|
|
360
|
+
//
|
|
361
|
+
// A sender may settle the write itself before it returns — the page outbox does, over
|
|
362
|
+
// HTTP. One it REFUSED keeps the error `fail` just recorded and is not counted as sent:
|
|
363
|
+
// clearing it here put `{ status: 'failed', error: null }` on disk and reported a refused
|
|
364
|
+
// write as a sent one.
|
|
365
|
+
if (statusOf(mutation) === 'failed') continue;
|
|
329
366
|
mutation.error = null;
|
|
330
367
|
sent += 1;
|
|
331
368
|
} catch (error) {
|
|
@@ -364,6 +401,7 @@ export class OfflineQueue {
|
|
|
364
401
|
* queue: a second tab's entries are not this tab's to overwrite or delete.
|
|
365
402
|
*/
|
|
366
403
|
async #persist(puts: readonly QueuedMutation[], deletes: readonly string[] = []): Promise<void> {
|
|
404
|
+
if (this.#abandoned) return;
|
|
367
405
|
await this.#store.write({
|
|
368
406
|
puts: puts.map((mutation) => ({ ...mutation })),
|
|
369
407
|
deletes,
|
|
@@ -372,6 +410,11 @@ export class OfflineQueue {
|
|
|
372
410
|
}
|
|
373
411
|
}
|
|
374
412
|
|
|
413
|
+
/** Read through a call: `send` awaited in between, and the compiler still narrows to `inflight`. */
|
|
414
|
+
function statusOf(mutation: QueuedMutation): MutationStatus {
|
|
415
|
+
return mutation.status;
|
|
416
|
+
}
|
|
417
|
+
|
|
375
418
|
/** A thrown value as the queue records it — `code`, `cause`, `fix`, never a throw of its own. */
|
|
376
419
|
export function toQueueError(error: unknown): WireError {
|
|
377
420
|
return {
|
package/src/page-errors.ts
CHANGED
|
@@ -93,6 +93,23 @@ export class LocalStoreUnavailableError extends RealtimeError {
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/**
|
|
97
|
+
* A write reached a queue whose principal is gone: the page signed out, or in as someone else,
|
|
98
|
+
* while the write was being queued. It is not stored and never sent — a write issued under one
|
|
99
|
+
* session must not leave under the next — and it is RAISED, because a user's write dropped in
|
|
100
|
+
* silence is a loss nobody is told about. `name` is the mutator's registered name; the write's
|
|
101
|
+
* idempotency key is deliberately not rendered.
|
|
102
|
+
*/
|
|
103
|
+
export class OfflineQueueAbandonedError extends RealtimeError {
|
|
104
|
+
constructor(args: { name: string }) {
|
|
105
|
+
super({
|
|
106
|
+
code: 'X_OFFLINE_QUEUE_ABANDONED',
|
|
107
|
+
cause: `the write "${args.name}" was queued after its page changed principal, so it was not stored and will not be sent — call the mutation again under the current session`,
|
|
108
|
+
fix: 'x errors explain X_OFFLINE_QUEUE_ABANDONED --json',
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
96
113
|
/**
|
|
97
114
|
* Something that can only mean "talk to the socket" ran on the server client — a mutation, a
|
|
98
115
|
* publish, a topic subscription, a dial. There is no socket during a server render and there never
|