@ultimat3/realtime 23.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +75 -81
- package/README.md +107 -17
- package/package.json +4 -4
- package/src/advisory-lock.ts +57 -0
- package/src/boot.ts +12 -7
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +2 -2
- package/src/changefeed.ts +39 -1
- package/src/channel-authz.ts +43 -12
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +34 -4
- package/src/channel-records.ts +33 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +143 -44
- package/src/client-channels.ts +10 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +7 -1
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +1 -0
- package/src/cursor.ts +11 -0
- package/src/errors.ts +70 -19
- package/src/fanout.ts +13 -0
- package/src/index.ts +1 -1
- package/src/live-definition.ts +22 -7
- package/src/live-fanout.ts +43 -7
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +38 -0
- package/src/live-query.ts +36 -37
- package/src/nats-client.ts +11 -2
- package/src/nats-transport.ts +22 -0
- package/src/offline-queue.ts +48 -5
- package/src/page-errors.ts +17 -0
- package/src/page-outbox.ts +121 -50
- package/src/pg-advisory-lock.ts +45 -1
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +119 -21
- package/src/pg-entity-row.ts +16 -0
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +44 -45
- package/src/pg-socket.ts +6 -4
- package/src/pg-tls.ts +18 -0
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/presence.ts +85 -8
- package/src/query-window.ts +16 -4
- package/src/record-persister.ts +7 -2
- package/src/replication-errors.ts +8 -4
- package/src/replicator-envelope.ts +110 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +319 -186
- package/src/server.ts +10 -6
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/subscriber-gate.ts +11 -1
- package/src/sync-auth.ts +47 -2
- package/src/sync-frames.ts +73 -20
- package/src/sync-node-bounds.ts +15 -0
- package/src/sync-node-contract.ts +20 -2
- package/src/sync-node.ts +92 -17
- package/src/sync-origin.ts +26 -11
- package/src/sync-protocol.ts +13 -3
- package/src/sync-upgrade.ts +76 -8
- package/src/use-mutation.ts +33 -5
- package/src/wire-channel.ts +13 -4
package/src/boot.ts
CHANGED
|
@@ -15,9 +15,8 @@ import type { RecordStore } from './record-store';
|
|
|
15
15
|
* The types the document marks persisted, read back off disk, then the page's outbox opened. None
|
|
16
16
|
* marked is no record disk.
|
|
17
17
|
*/
|
|
18
|
-
async function restoreFromDisk(store: RecordStore, principal: Principal): Promise<void> {
|
|
18
|
+
async function restoreFromDisk(store: RecordStore, principal: () => Principal): Promise<void> {
|
|
19
19
|
const types = persistedTypes();
|
|
20
|
-
const keep = scopeKey(principal);
|
|
21
20
|
let persister: RecordPersister | undefined;
|
|
22
21
|
try {
|
|
23
22
|
// A sign-out by full navigation (a form post and a redirect) never calls `rescope()`, so the
|
|
@@ -26,7 +25,12 @@ async function restoreFromDisk(store: RecordStore, principal: Principal): Promis
|
|
|
26
25
|
// restored. An unscoped page (rendered for nobody) wipes nothing — it knows no principal.
|
|
27
26
|
// Trade-off: two principals in two tabs of one browser — the newer boot wipes the other's
|
|
28
27
|
// disk; that tab keeps its records in memory and re-persists them on its next write.
|
|
29
|
-
|
|
28
|
+
//
|
|
29
|
+
// WHOSE disk is kept is decided AT the wipe, after the store has opened: decided before that
|
|
30
|
+
// await, a `rescope()` during it kept the principal that left and wiped the one that arrived.
|
|
31
|
+
const disk = await pageLocalStore();
|
|
32
|
+
const keep = scopeKey(principal());
|
|
33
|
+
if (keep !== undefined) await disk.wipeOthers(keep);
|
|
30
34
|
if (types.size > 0) {
|
|
31
35
|
persister = recordPersister({ store, local: await pageLocalStore(), types });
|
|
32
36
|
await persister.restore();
|
|
@@ -50,16 +54,17 @@ type Principal = ClientScope['principal'];
|
|
|
50
54
|
* scope is the page's own — a parameter only so a test can boot an unscoped page (an OBJECT, so an
|
|
51
55
|
* explicit `undefined` principal is not mistaken for "use the default").
|
|
52
56
|
*/
|
|
53
|
-
export function bootPage(
|
|
54
|
-
scope: { readonly principal: Principal } = pageClient().scope,
|
|
55
|
-
): Promise<void> {
|
|
57
|
+
export function bootPage(scope?: { readonly principal: Principal }): Promise<void> {
|
|
56
58
|
const host = globalThis as BootHost;
|
|
57
59
|
const started = host[BOOT_KEY];
|
|
58
60
|
const waiting = host[BOOT_RELEASE_KEY];
|
|
59
61
|
// Already booting — unless what sits there is an island's placeholder, which this boot claims.
|
|
60
62
|
if (started !== undefined && waiting === undefined) return started;
|
|
61
63
|
Reflect.deleteProperty(host, BOOT_RELEASE_KEY);
|
|
62
|
-
|
|
64
|
+
// Asked each time, never captured: the page's principal is whatever `rescope()` last said.
|
|
65
|
+
const principal = (): Principal =>
|
|
66
|
+
scope === undefined ? pageClient().scope.principal : scope.principal;
|
|
67
|
+
const booted = restoreFromDisk(pageRealtime().store, principal);
|
|
63
68
|
if (started !== undefined && waiting !== undefined) {
|
|
64
69
|
void booted.then(waiting);
|
|
65
70
|
return started;
|
package/src/change-buffer.ts
CHANGED
|
@@ -36,11 +36,41 @@ export interface ChangeBufferOptions {
|
|
|
36
36
|
export const DEFAULT_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
|
|
37
37
|
export const DEFAULT_MAX_BUFFER_BYTES_PER_QUERY = 1024 * 1024;
|
|
38
38
|
|
|
39
|
+
/**
|
|
40
|
+
* THE FLOOR RULE, stated once. A ring answers a resume only for a cursor it can PROVE it holds
|
|
41
|
+
* every later change for; anything else is `null`, and the caller takes one bounded snapshot.
|
|
42
|
+
*
|
|
43
|
+
* | The floor is set by | to | a cursor AT the floor |
|
|
44
|
+
* |---|---|---|
|
|
45
|
+
* | the ring's first patch (no read floored it) | that patch's lsn | resumes — it has that patch |
|
|
46
|
+
* | an eviction | the evicted patch's lsn | resumes — everything after it is retained |
|
|
47
|
+
* | a window's FIRST read, on a node that holds a position | the read's lsn | resumes — the cursor may be that read's own |
|
|
48
|
+
* | a window's FIRST read with NO position behind it (`sole`) | the read's own mark | resumes, and it is the ONLY cursor below the first patch that does |
|
|
49
|
+
* | a FORCED re-read (the window missed a change), a truncate, a partial row | the read's lsn, EXCLUSIVE | is refused |
|
|
50
|
+
*
|
|
51
|
+
* Why `sole`: a node that has received no change holds no position, so its read is marked with
|
|
52
|
+
* the node's own origin (`LiveQueryRegistry`), which sorts below every real lsn. A plain floor
|
|
53
|
+
* there admits every foreign cursor — each is "above" it — for a history this node never held. So
|
|
54
|
+
* the ring answers that one mark, and otherwise only a cursor at or after its FIRST patch.
|
|
55
|
+
*
|
|
56
|
+
* Why the last row is exclusive: a re-read's lsn is the node's last SEEN position, which does not
|
|
57
|
+
* move for the changes it missed — so "the window at L before the gap" and "the window re-read at
|
|
58
|
+
* L with the missed rows in it" carry the same cursor, and only refusing L tells them apart.
|
|
59
|
+
*
|
|
60
|
+
* What never moves the floor: a read served out of a window that is already filled. The retained
|
|
61
|
+
* patches are still its true history, and flooring it evicts every other subscriber's resume.
|
|
62
|
+
*/
|
|
39
63
|
interface Ring {
|
|
40
64
|
patches: RowPatch[];
|
|
41
65
|
bytes: number;
|
|
42
|
-
/**
|
|
43
|
-
evictedThrough: string
|
|
66
|
+
/** The lsn this ring is complete from. Set at birth and only ever raised. */
|
|
67
|
+
evictedThrough: string;
|
|
68
|
+
/** Whether a cursor exactly AT the floor is refused — see the rule above. */
|
|
69
|
+
exclusive: boolean;
|
|
70
|
+
/** The one cursor a position-less birth vouches for, until an eviction or a re-read ends it. */
|
|
71
|
+
sole: string | null;
|
|
72
|
+
/** The first patch appended after a `sole` birth: where foreign cursors become answerable. */
|
|
73
|
+
anchor: string | null;
|
|
44
74
|
}
|
|
45
75
|
|
|
46
76
|
const encoder = new TextEncoder();
|
|
@@ -52,12 +82,6 @@ function patchBytes(patch: RowPatch): number {
|
|
|
52
82
|
|
|
53
83
|
export class RingChangeBuffer implements ResumeSource {
|
|
54
84
|
readonly #rings = new Map<string, Ring>();
|
|
55
|
-
/**
|
|
56
|
-
* Query hashes whose ring was dropped, so the ring the NEXT change builds knows it does not
|
|
57
|
-
* carry the history before it. A `Set` of ids and not a `Map` of lsns: what the next ring is
|
|
58
|
-
* complete from is its own first patch, which only that patch can name.
|
|
59
|
-
*/
|
|
60
|
-
readonly #forgotten = new Set<string>();
|
|
61
85
|
readonly #capacity: number;
|
|
62
86
|
readonly #maxQueries: number;
|
|
63
87
|
readonly #maxBytesPerQuery: number;
|
|
@@ -86,17 +110,20 @@ export class RingChangeBuffer implements ResumeSource {
|
|
|
86
110
|
|
|
87
111
|
append(qid: string, patch: RowPatch): void {
|
|
88
112
|
const existing = this.#rings.get(qid);
|
|
89
|
-
// A ring
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
// `
|
|
94
|
-
const reborn = this.#forgotten.delete(qid);
|
|
113
|
+
// A ring is complete only from where it was born. Born HERE, from a patch, that is the patch
|
|
114
|
+
// itself: this node held no entry for whatever came before, so nothing before it was ever
|
|
115
|
+
// appended. It used to be born with no floor at all, and a cursor minted on another node —
|
|
116
|
+
// the rolling-deploy shape — read as in-window on a ring that had held none of its history.
|
|
117
|
+
// `floorAt` is the earlier, cheaper birth: the lsn the window was read at.
|
|
95
118
|
const ring: Ring = existing ?? {
|
|
96
119
|
patches: [],
|
|
97
120
|
bytes: 0,
|
|
98
|
-
evictedThrough:
|
|
121
|
+
evictedThrough: patch.lsn,
|
|
122
|
+
exclusive: false,
|
|
123
|
+
sole: null,
|
|
124
|
+
anchor: null,
|
|
99
125
|
};
|
|
126
|
+
if (ring.sole !== null && ring.anchor === null) ring.anchor = patch.lsn;
|
|
100
127
|
ring.patches.push(patch);
|
|
101
128
|
const cost = patchBytes(patch);
|
|
102
129
|
ring.bytes += cost;
|
|
@@ -106,8 +133,48 @@ export class RingChangeBuffer implements ResumeSource {
|
|
|
106
133
|
while (ring.patches.length > this.#capacity || ring.bytes > this.#maxBytesPerQuery) {
|
|
107
134
|
if (!this.#shift(ring)) break;
|
|
108
135
|
}
|
|
109
|
-
|
|
110
|
-
|
|
136
|
+
this.#touch(qid, ring);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The window behind `qid` was read at `lsn`: the ring is complete from there, and from nowhere
|
|
141
|
+
* earlier. The caller decides WHEN (a read that landed, never one served from a filled window)
|
|
142
|
+
* and whether the floor is `exclusive` (a forced re-read) — the rule is on `Ring`.
|
|
143
|
+
*
|
|
144
|
+
* Only ever raises the floor, or tightens it at the same lsn. `''` is no position and says nothing.
|
|
145
|
+
*/
|
|
146
|
+
floorAt(
|
|
147
|
+
qid: string,
|
|
148
|
+
lsn: string,
|
|
149
|
+
options: { readonly exclusive?: boolean; readonly sole?: boolean } = {},
|
|
150
|
+
): void {
|
|
151
|
+
if (lsn === '') return;
|
|
152
|
+
const exclusive = options.exclusive === true;
|
|
153
|
+
const existing = this.#rings.get(qid);
|
|
154
|
+
const ring: Ring = existing ?? {
|
|
155
|
+
patches: [],
|
|
156
|
+
bytes: 0,
|
|
157
|
+
evictedThrough: lsn,
|
|
158
|
+
exclusive,
|
|
159
|
+
sole: options.sole === true ? lsn : null,
|
|
160
|
+
anchor: null,
|
|
161
|
+
};
|
|
162
|
+
if (lsn > ring.evictedThrough || (lsn === ring.evictedThrough && exclusive)) {
|
|
163
|
+
// Superseded, not evicted: what the read replaced is history with a gap in it.
|
|
164
|
+
for (let first = ring.patches[0]; first !== undefined && first.lsn <= lsn; ) {
|
|
165
|
+
this.#shift(ring);
|
|
166
|
+
first = ring.patches[0];
|
|
167
|
+
}
|
|
168
|
+
ring.evictedThrough = lsn;
|
|
169
|
+
ring.exclusive = exclusive;
|
|
170
|
+
ring.sole = null;
|
|
171
|
+
}
|
|
172
|
+
this.#touch(qid, ring);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Move `qid` to the tail of the LRU order, then hold both node-wide ceilings. */
|
|
176
|
+
#touch(qid: string, ring: Ring): void {
|
|
177
|
+
this.#rings.delete(qid);
|
|
111
178
|
this.#rings.set(qid, ring);
|
|
112
179
|
while (this.#rings.size > this.#maxQueries || this.#bytes > this.#maxBytes) {
|
|
113
180
|
const oldest = this.#rings.keys().next();
|
|
@@ -126,13 +193,19 @@ export class RingChangeBuffer implements ResumeSource {
|
|
|
126
193
|
ring.bytes -= cost;
|
|
127
194
|
this.#bytes -= cost;
|
|
128
195
|
ring.evictedThrough = dropped.lsn;
|
|
196
|
+
ring.exclusive = false;
|
|
197
|
+
ring.sole = null;
|
|
129
198
|
return true;
|
|
130
199
|
}
|
|
131
200
|
|
|
132
201
|
since(qid: string, lsn: string): RowPatch[] | null {
|
|
133
202
|
const ring = this.#rings.get(qid);
|
|
134
203
|
if (!ring) return null;
|
|
135
|
-
if (
|
|
204
|
+
if (lsn < ring.evictedThrough) return null;
|
|
205
|
+
if (ring.exclusive && lsn === ring.evictedThrough) return null;
|
|
206
|
+
if (ring.sole !== null && lsn !== ring.sole && (ring.anchor === null || lsn < ring.anchor)) {
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
136
209
|
return ring.patches.filter((patch) => patch.lsn > lsn);
|
|
137
210
|
}
|
|
138
211
|
|
|
@@ -149,29 +222,11 @@ export class RingChangeBuffer implements ResumeSource {
|
|
|
149
222
|
*/
|
|
150
223
|
forget(qid: string): void {
|
|
151
224
|
const ring = this.#rings.get(qid);
|
|
152
|
-
if (ring
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
//
|
|
157
|
-
// absent is not the history being intact. Both callers lose history here and neither could say
|
|
158
|
-
// so: `LiveQueryRegistry.unsubscribe` drops the ENTRY, so every change until the next
|
|
159
|
-
// subscriber is never appended at all, and the LRU fires on a query that still has LIVE
|
|
160
|
-
// subscribers. Either way the next `append` was building a ring that reported itself complete
|
|
161
|
-
// from the beginning of time, so a client reconnecting inside `maxLagMs` folded a partial patch
|
|
162
|
-
// list onto a stale window with `shouldResnapshot` answering `in-window` and nothing marked
|
|
163
|
-
// desynced — permanently divergent on a healthy socket. What the tombstone costs in exchange is
|
|
164
|
-
// a resume that could have been a delta taking the snapshot path; that is one bounded read, and
|
|
165
|
-
// it is the direction this package errs in everywhere else.
|
|
166
|
-
this.#forgotten.add(qid);
|
|
167
|
-
// Bounded like everything else here: insertion-ordered, so the oldest tombstone goes first.
|
|
168
|
-
// Losing one costs a resume that could have been a delta; keeping them unbounded costs memory
|
|
169
|
-
// a client-chosen input mints at will.
|
|
170
|
-
while (this.#forgotten.size > this.#maxQueries) {
|
|
171
|
-
const oldest = this.#forgotten.values().next();
|
|
172
|
-
if (oldest.done === true) break;
|
|
173
|
-
this.#forgotten.delete(oldest.value);
|
|
174
|
-
}
|
|
225
|
+
if (ring === undefined) return;
|
|
226
|
+
this.#bytes -= ring.bytes;
|
|
227
|
+
this.#rings.delete(qid);
|
|
228
|
+
// Nothing is remembered about it: the next ring for this qid is born with its own floor, so
|
|
229
|
+
// it cannot claim the history that went with this one.
|
|
175
230
|
}
|
|
176
231
|
|
|
177
232
|
get queryCount(): number {
|
package/src/changefeed-env.ts
CHANGED
|
@@ -5,14 +5,14 @@
|
|
|
5
5
|
|
|
6
6
|
import type { Clock } from '@ultimat3/core';
|
|
7
7
|
import { ConfigInvalidError } from '@ultimat3/core';
|
|
8
|
+
import type { AdvisoryLock } from './advisory-lock';
|
|
9
|
+
import { InMemoryAdvisoryLock } from './advisory-lock';
|
|
8
10
|
import type { ChangeFeed } from './changefeed';
|
|
9
11
|
import { InMemoryChangeFeed, PgLogicalReplicationFeed } from './changefeed';
|
|
10
12
|
import { PgAdvisoryLock } from './pg-advisory-lock';
|
|
11
13
|
import type { PgTarget } from './pg-socket';
|
|
12
14
|
import { parsePgUrl } from './pg-socket';
|
|
13
15
|
import type { PgStream } from './pg-wire';
|
|
14
|
-
import type { AdvisoryLock } from './replicator';
|
|
15
|
-
import { InMemoryAdvisoryLock } from './replicator';
|
|
16
16
|
import type { Rng } from './thundering-herd';
|
|
17
17
|
|
|
18
18
|
/** The keys read here, and nothing else. Named once so docs and tests cannot drift from the code. */
|
package/src/changefeed.ts
CHANGED
|
@@ -39,18 +39,38 @@ export interface ChangeEvent<R extends Row = Row> {
|
|
|
39
39
|
* channel stamps it on the `records` frame so the writing page recognises its own echo.
|
|
40
40
|
*/
|
|
41
41
|
readonly write?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Row properties `after` does NOT carry because Postgres did not log them: an UPDATE that left
|
|
44
|
+
* an out-of-line (TOAST) value untouched sends no bytes for it, and under any replica identity
|
|
45
|
+
* but FULL there is no old image to read it from. Absent when `after` is the whole row. A
|
|
46
|
+
* consumer must not adopt `after` as the row while this is set — it re-reads instead.
|
|
47
|
+
*/
|
|
48
|
+
readonly omitted?: readonly string[];
|
|
42
49
|
}
|
|
43
50
|
|
|
44
51
|
export interface ChangeFeedStartOptions {
|
|
45
52
|
/** Resume position. Omitted means "from now". */
|
|
46
53
|
readonly from?: string;
|
|
47
54
|
readonly onChange: (event: ChangeEvent) => void | Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* The feed stopped delivering ON ITS OWN — the walsender ended the copy, a decode failed, the
|
|
57
|
+
* handler rejected — and will not resume until `start()` is called again. Never called for a
|
|
58
|
+
* `stop()`. This is the only way out a pump has: nothing awaits its read loop, so a death that
|
|
59
|
+
* was only recorded was a death nobody saw.
|
|
60
|
+
*/
|
|
61
|
+
readonly onEnd?: (reason: string) => void;
|
|
48
62
|
}
|
|
49
63
|
|
|
50
64
|
export interface ChangeFeed {
|
|
51
65
|
readonly source: string;
|
|
52
66
|
start(options: ChangeFeedStartOptions): Promise<void>;
|
|
53
67
|
stop(): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Go silent NOW, without asking: synchronous, nothing written, nothing awaited. For a stream
|
|
70
|
+
* that may already be dead — `stop()` says goodbye first, and a goodbye to a black-holed socket
|
|
71
|
+
* is never answered. After it returns no further change is delivered and `onEnd` is not called.
|
|
72
|
+
*/
|
|
73
|
+
abandon(): void;
|
|
54
74
|
/** Highest lsn delivered to the handler; the replicator persists this to survive a restart. */
|
|
55
75
|
lastLsn(): string | null;
|
|
56
76
|
}
|
|
@@ -103,6 +123,10 @@ export class InMemoryChangeFeed implements ChangeFeed {
|
|
|
103
123
|
await this.#tail;
|
|
104
124
|
}
|
|
105
125
|
|
|
126
|
+
abandon(): void {
|
|
127
|
+
this.#handler = null;
|
|
128
|
+
}
|
|
129
|
+
|
|
106
130
|
lastLsn(): string | null {
|
|
107
131
|
return this.#lastLsn;
|
|
108
132
|
}
|
|
@@ -175,6 +199,12 @@ export interface PgLogicalReplicationOptions {
|
|
|
175
199
|
readonly rng?: Rng | undefined;
|
|
176
200
|
/** The byte pipe, injected. Defaults to `Bun.connect`; a test drives a scripted server instead. */
|
|
177
201
|
readonly stream?: ((target: PgTarget) => Promise<PgStream>) | undefined;
|
|
202
|
+
/**
|
|
203
|
+
* Tables whose DELETE must carry the whole old row — a channel with params routes a removal by
|
|
204
|
+
* columns only `REPLICA IDENTITY FULL` logs. Preflight warns for each that is not FULL. Defaults
|
|
205
|
+
* to what the declared channels need (`pg-identity-tables.ts`).
|
|
206
|
+
*/
|
|
207
|
+
readonly fullIdentityTables?: readonly string[] | undefined;
|
|
178
208
|
}
|
|
179
209
|
|
|
180
210
|
/**
|
|
@@ -199,13 +229,21 @@ export class PgLogicalReplicationFeed implements ChangeFeed {
|
|
|
199
229
|
}
|
|
200
230
|
|
|
201
231
|
async start(options: ChangeFeedStartOptions): Promise<void> {
|
|
202
|
-
await this.#stream.start({
|
|
232
|
+
await this.#stream.start({
|
|
233
|
+
from: options.from,
|
|
234
|
+
onChange: options.onChange,
|
|
235
|
+
onEnd: options.onEnd,
|
|
236
|
+
});
|
|
203
237
|
}
|
|
204
238
|
|
|
205
239
|
async stop(): Promise<void> {
|
|
206
240
|
await this.#stream.stop();
|
|
207
241
|
}
|
|
208
242
|
|
|
243
|
+
abandon(): void {
|
|
244
|
+
this.#stream.abandon();
|
|
245
|
+
}
|
|
246
|
+
|
|
209
247
|
lastLsn(): string | null {
|
|
210
248
|
return this.#stream.lastLsn();
|
|
211
249
|
}
|
package/src/channel-authz.ts
CHANGED
|
@@ -5,11 +5,17 @@
|
|
|
5
5
|
import { type Actor, type Ctx, createContext, runWithContext } from '@ultimat3/core';
|
|
6
6
|
import { guard, QueryDeniedError } from '@ultimat3/query';
|
|
7
7
|
import type { Channel } from './channel-decl';
|
|
8
|
-
import { TopicForbiddenError } from './errors';
|
|
8
|
+
import { isTenancyDenial, TopicForbiddenError } from './errors';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* A denial is `X_TOPIC_FORBIDDEN`. A loader or a rule that RAISED is not a denial and leaves as it
|
|
12
|
-
* came, so the caller can tell an outage from a decision — `onActorChange`
|
|
12
|
+
* came, so the caller can tell an outage from a decision — `onActorChange` suspends the topic on one.
|
|
13
|
+
*
|
|
14
|
+
* Two answers are denials before any rule runs, because neither is an outage:
|
|
15
|
+
* - **nobody, on a channel that decides on a row.** A loader runs AS the subscriber; with no actor
|
|
16
|
+
* it would run as the node, and a row is not something nobody is entitled to.
|
|
17
|
+
* - **a loader the tenant guard refused** (`TENANCY_DENIAL_CODES`): the actor has no org, or names
|
|
18
|
+
* another one. That is the verdict about a removed member, not a store that could not answer.
|
|
13
19
|
*/
|
|
14
20
|
export async function authorizeChannel(
|
|
15
21
|
channel: Channel,
|
|
@@ -18,17 +24,42 @@ export async function authorizeChannel(
|
|
|
18
24
|
topic: string,
|
|
19
25
|
params: Readonly<Record<string, string>>,
|
|
20
26
|
): Promise<void> {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
const loader = channel.row;
|
|
28
|
+
if (loader === undefined) return decide(channel, ctx, actor, topic, params, null);
|
|
29
|
+
if (actor === null) {
|
|
30
|
+
throw new TopicForbiddenError({
|
|
31
|
+
topic,
|
|
32
|
+
actorId: null,
|
|
33
|
+
reason: `channel "${channel.name}" decides on a row, and an anonymous socket has no actor to load one as`,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
// The loader and the rule run AS the subscriber, never as the node. Services are rebuilt for
|
|
37
|
+
// this actor by `createContext`, the rule `withChildContext` follows.
|
|
38
|
+
const scoped = subscriberContext(ctx, actor);
|
|
39
|
+
let row: unknown;
|
|
40
|
+
try {
|
|
41
|
+
row = await runWithContext(scoped, async () => await loader({ params, ctx: scoped }));
|
|
42
|
+
} catch (error) {
|
|
43
|
+
if (!isTenancyDenial(error)) throw error;
|
|
44
|
+
throw new TopicForbiddenError({
|
|
45
|
+
topic,
|
|
46
|
+
actorId: actor.id,
|
|
47
|
+
reason: `channel "${channel.name}" reads a row outside this actor's tenant`,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
decide(channel, scoped, actor, topic, params, row);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function decide(
|
|
54
|
+
channel: Channel,
|
|
55
|
+
ctx: Ctx,
|
|
56
|
+
actor: Actor | null,
|
|
57
|
+
topic: string,
|
|
58
|
+
params: Readonly<Record<string, string>>,
|
|
59
|
+
row: unknown,
|
|
60
|
+
): void {
|
|
30
61
|
try {
|
|
31
|
-
guard(channel.policy, { actor, input: params, row, ctx
|
|
62
|
+
guard(channel.policy, { actor, input: params, row, ctx, query: channel.name }, 'live');
|
|
32
63
|
} catch (error) {
|
|
33
64
|
if (!(error instanceof QueryDeniedError)) throw error;
|
|
34
65
|
throw new TopicForbiddenError({
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// The denials one socket has been answered with, latched until its session changes — so a client
|
|
2
|
+
// re-asking in a loop is answered without re-running the policy, and only THAT topic is refused.
|
|
3
|
+
// Bounded: a topic is spelled from client params, so an unbounded latch was one entry per refusal.
|
|
4
|
+
|
|
5
|
+
import { SubscriptionLimitError } from './errors';
|
|
6
|
+
import type { SyncSocket } from './socket';
|
|
7
|
+
|
|
8
|
+
export class DenialLatch {
|
|
9
|
+
/** Weakly keyed, so a socket's latch dies with it and no close path has to remember it. */
|
|
10
|
+
readonly #bySocket = new WeakMap<SyncSocket, Set<string>>();
|
|
11
|
+
readonly #max: number;
|
|
12
|
+
readonly #knob: string;
|
|
13
|
+
|
|
14
|
+
constructor(max: number, knob: string) {
|
|
15
|
+
this.#max = max;
|
|
16
|
+
this.#knob = knob;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
has(socket: SyncSocket, name: string): boolean {
|
|
20
|
+
return this.#bySocket.get(socket)?.has(name) === true;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Refused BEFORE a policy runs once the latch is full: a socket that has been denied this many
|
|
25
|
+
* distinct topics is asking for more than it may hold, and each further ask would otherwise be
|
|
26
|
+
* either a retained entry or a policy pass the client schedules at will.
|
|
27
|
+
*/
|
|
28
|
+
assertRoom(socket: SyncSocket): void {
|
|
29
|
+
if ((this.#bySocket.get(socket)?.size ?? 0) < this.#max) return;
|
|
30
|
+
throw new SubscriptionLimitError({
|
|
31
|
+
scope: 'socket',
|
|
32
|
+
id: socket.id,
|
|
33
|
+
limit: this.#max,
|
|
34
|
+
knob: this.#knob,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
add(socket: SyncSocket, name: string): void {
|
|
39
|
+
const latched = this.#bySocket.get(socket) ?? new Set<string>();
|
|
40
|
+
if (latched.size < this.#max) this.#bySocket.set(socket, latched.add(name));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** A new session re-decides everything. */
|
|
44
|
+
clear(socket: SyncSocket): void {
|
|
45
|
+
this.#bySocket.delete(socket);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Denials held for one socket — what a test bounds. */
|
|
49
|
+
count(socket: SyncSocket): number {
|
|
50
|
+
return this.#bySocket.get(socket)?.size ?? 0;
|
|
51
|
+
}
|
|
52
|
+
}
|
package/src/channel-logs.ts
CHANGED
|
@@ -74,6 +74,11 @@ export class ChannelLogs {
|
|
|
74
74
|
for (const update of updates) {
|
|
75
75
|
const open = this.#byTopic.get(update.topic);
|
|
76
76
|
if (open === undefined) continue;
|
|
77
|
+
// The change could not carry the whole row (`channel-records.ts`): members re-read it.
|
|
78
|
+
if (update.gap === true) {
|
|
79
|
+
frames += this.#reopen(update.topic, open.target);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
77
82
|
const entry = open.ring.append(update.adopt, update.remove, change.write);
|
|
78
83
|
const frame = renderRecords(update.topic, open.ring.epoch, entry);
|
|
79
84
|
frames += this.#sockets.deliverRecords(update.topic, open.ring.epoch, frame);
|
|
@@ -91,13 +96,33 @@ export class ChannelLogs {
|
|
|
91
96
|
let announced = 0;
|
|
92
97
|
for (const [topic, open] of [...this.#byTopic]) {
|
|
93
98
|
if (!carriesTable(open.target.channel, table)) continue;
|
|
94
|
-
this.#
|
|
95
|
-
const ring = this.open(topic, open.target);
|
|
96
|
-
announced += this.#sockets.announceGap(topic, ring.epoch);
|
|
99
|
+
announced += this.#reopen(topic, open.target);
|
|
97
100
|
}
|
|
98
101
|
return announced;
|
|
99
102
|
}
|
|
100
103
|
|
|
104
|
+
/**
|
|
105
|
+
* This node may have missed a change (the bus dropped, or skipped a sequence). Seq is minted
|
|
106
|
+
* HERE from the changes this node sees, so the hole is invisible to the ring: the next change
|
|
107
|
+
* would be appended as the next seq and a `since` replayed across it as complete history. Every
|
|
108
|
+
* open topic that carries records therefore starts a new epoch and tells its members.
|
|
109
|
+
*/
|
|
110
|
+
invalidate(): number {
|
|
111
|
+
let announced = 0;
|
|
112
|
+
for (const [topic, open] of [...this.#byTopic]) {
|
|
113
|
+
if (open.target.channel.records.length === 0) continue;
|
|
114
|
+
announced += this.#reopen(topic, open.target);
|
|
115
|
+
}
|
|
116
|
+
return announced;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** A new epoch for one topic — nothing in the old ring may be replayed — and a gap to each member. */
|
|
120
|
+
#reopen(topic: string, target: ChannelTopic): number {
|
|
121
|
+
this.#byTopic.delete(topic);
|
|
122
|
+
const ring = this.open(topic, target);
|
|
123
|
+
return this.#sockets.announceGap(topic, ring.epoch);
|
|
124
|
+
}
|
|
125
|
+
|
|
101
126
|
/**
|
|
102
127
|
* A (re)subscribe `since` a position: every frame after it from the ring, or — when the ring
|
|
103
128
|
* cannot prove it holds them all, or the epoch moved — one `replay-gap`.
|
|
@@ -107,6 +132,10 @@ export class ChannelLogs {
|
|
|
107
132
|
* reached nobody — and nothing the client holds can name it. The seat is the one point after
|
|
108
133
|
* which every commit is a frame, so the re-read is told to start from here. A repeated `add` on
|
|
109
134
|
* a socket already seated (the presence beat), or a channel with no records, is sent nothing.
|
|
135
|
+
*
|
|
136
|
+
* A `since` that is already CURRENT is answered too: the frame AT its cursor, carrying nothing.
|
|
137
|
+
* Sent nothing, the client had no frame to leave `joining` on — a channel that was up to date
|
|
138
|
+
* read as never joined. The seq is one it already holds, so its cursor does not move.
|
|
110
139
|
*/
|
|
111
140
|
resume(socket: SyncSocket, topic: string, since: ChannelSince | undefined, fresh: boolean): void {
|
|
112
141
|
const open = this.#byTopic.get(topic);
|
|
@@ -124,7 +153,8 @@ export class ChannelLogs {
|
|
|
124
153
|
if (!socket.send(frame)) socket.gaps.set(topic, open.ring.epoch);
|
|
125
154
|
return;
|
|
126
155
|
}
|
|
127
|
-
|
|
156
|
+
const answer = entries.length > 0 ? entries : [{ seq: open.ring.seq, adopt: [], remove: [] }];
|
|
157
|
+
for (const entry of answer) {
|
|
128
158
|
if (!socket.send(renderRecords(topic, open.ring.epoch, entry))) {
|
|
129
159
|
socket.gaps.set(topic, open.ring.epoch);
|
|
130
160
|
return;
|
package/src/channel-records.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// socket, no seq, no policy — `ChannelHub` does those per node and per socket. The only derivation
|
|
3
3
|
// of "which channel carries this row" lives here, so a write never names a channel (axiom 2).
|
|
4
4
|
|
|
5
|
+
import { type Counter, counter, logger } from '@ultimat3/core';
|
|
5
6
|
import type { Row } from '@ultimat3/core/page';
|
|
6
7
|
import type { RecordProjection } from '@ultimat3/entity/record';
|
|
7
8
|
import type { ChangeEvent } from './changefeed';
|
|
@@ -16,6 +17,23 @@ export interface TopicUpdate {
|
|
|
16
17
|
readonly adopt: readonly RecordPart[];
|
|
17
18
|
readonly remove: readonly { readonly type: string; readonly key: string }[];
|
|
18
19
|
readonly row: Row;
|
|
20
|
+
/**
|
|
21
|
+
* The change could not carry this row whole (`ChangeEvent.omitted` names a column the record
|
|
22
|
+
* projects), so there is nothing sound to adopt: the topic's members are told to re-read.
|
|
23
|
+
*/
|
|
24
|
+
readonly gap?: true;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const removalsSkipped: Counter = counter('channel_removals_skipped_total', {
|
|
28
|
+
unit: '{removal}',
|
|
29
|
+
description:
|
|
30
|
+
'Channel record removals that could not be routed: the old row image lacked the channel params (the table is not REPLICA IDENTITY FULL), so members keep the removed record.',
|
|
31
|
+
});
|
|
32
|
+
let skipped = 0;
|
|
33
|
+
|
|
34
|
+
/** Removals this process could not route since boot — the in-process read of the counter above. */
|
|
35
|
+
export function skippedRemovals(): number {
|
|
36
|
+
return skipped;
|
|
19
37
|
}
|
|
20
38
|
|
|
21
39
|
/**
|
|
@@ -25,7 +43,9 @@ export interface TopicUpdate {
|
|
|
25
43
|
* - insert/update: the new row is adopted on the topic its params name.
|
|
26
44
|
* - an update that moved the row to other params (`orgId` changed): a remove on the OLD topic too.
|
|
27
45
|
* - delete: a remove on the topic the old image names — which needs the params columns in the
|
|
28
|
-
* `before` image, i.e. `REPLICA IDENTITY FULL`; a key-only image names no topic and is skipped
|
|
46
|
+
* `before` image, i.e. `REPLICA IDENTITY FULL`; a key-only image names no topic and is skipped
|
|
47
|
+
* — COUNTED and logged (`channel.removal_skipped`), because the members of that topic keep a
|
|
48
|
+
* record the database no longer has and nothing else would ever say so.
|
|
29
49
|
*/
|
|
30
50
|
export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): TopicUpdate[] {
|
|
31
51
|
const updates: TopicUpdate[] = [];
|
|
@@ -39,17 +59,28 @@ export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): To
|
|
|
39
59
|
const before = change.before;
|
|
40
60
|
const afterParams = after === null ? null : channel.paramsOf(after);
|
|
41
61
|
const beforeParams = before === null ? null : channel.paramsOf(before);
|
|
62
|
+
if (before !== null && beforeParams === null && channel.params.length > 0) {
|
|
63
|
+
skipped += 1;
|
|
64
|
+
removalsSkipped.add(1);
|
|
65
|
+
logger.warn('channel.removal_skipped', { channel: channel.name, table: change.entity });
|
|
66
|
+
}
|
|
42
67
|
const afterTopic = afterParams === null ? null : channel.topic(afterParams);
|
|
43
68
|
const beforeTopic = beforeParams === null ? null : channel.topic(beforeParams);
|
|
44
69
|
|
|
45
70
|
if (after !== null && afterParams !== null && afterTopic !== null) {
|
|
71
|
+
// An unchanged out-of-line value Postgres logged no bytes for: adopted, the partial row
|
|
72
|
+
// would be retained in the ring as the whole record and handed to every later joiner.
|
|
73
|
+
const partial = (change.omitted ?? []).some((column) =>
|
|
74
|
+
Object.hasOwn(projection.schema.properties ?? {}, column),
|
|
75
|
+
);
|
|
46
76
|
updates.push({
|
|
47
77
|
channel,
|
|
48
78
|
topic: afterTopic,
|
|
49
79
|
params: afterParams,
|
|
50
|
-
adopt: [{ type: projection.type, key: projection.key(after), row: after }],
|
|
80
|
+
adopt: partial ? [] : [{ type: projection.type, key: projection.key(after), row: after }],
|
|
51
81
|
remove: [],
|
|
52
82
|
row: after,
|
|
83
|
+
...(partial ? { gap: true as const } : {}),
|
|
53
84
|
});
|
|
54
85
|
}
|
|
55
86
|
if (
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// The sid each of a socket's channel seats was answered under, keyed by TOPIC. A sid is client
|
|
2
|
+
// data: keyed by it, one socket minted an entry per fresh sid without bound, and two sids naming
|
|
3
|
+
// one topic aliased one membership — dropping either unsubscribed the other.
|
|
4
|
+
|
|
5
|
+
import type { Topic } from './channel-decl';
|
|
6
|
+
import type { SyncSocket } from './socket';
|
|
7
|
+
|
|
8
|
+
/** Weakly keyed by socket, so a socket's entries die with it; bounded by the hub's topic cap. */
|
|
9
|
+
export class ChannelSids {
|
|
10
|
+
readonly #bySocket = new WeakMap<SyncSocket, Map<Topic, string>>();
|
|
11
|
+
|
|
12
|
+
/** The latest sid wins: a repeated `add` under a new sid re-names the one seat, never adds one. */
|
|
13
|
+
set(socket: SyncSocket, topic: Topic, sid: string): void {
|
|
14
|
+
const topics = this.#bySocket.get(socket) ?? new Map<Topic, string>();
|
|
15
|
+
this.#bySocket.set(socket, topics.set(topic, sid));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** The sid a refusal for this seat must name — `undefined` when the socket holds no such seat. */
|
|
19
|
+
sidOf(socket: SyncSocket, topic: Topic): string | undefined {
|
|
20
|
+
return this.#bySocket.get(socket)?.get(topic);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** The topic currently held under `sid`. A sid an earlier `add` was re-named away from holds none. */
|
|
24
|
+
topicOf(socket: SyncSocket, sid: string): Topic | undefined {
|
|
25
|
+
for (const [topic, held] of this.#bySocket.get(socket) ?? []) if (held === sid) return topic;
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
delete(socket: SyncSocket, topic: Topic): void {
|
|
30
|
+
this.#bySocket.get(socket)?.delete(topic);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Seats recorded for one socket — what a test bounds. */
|
|
34
|
+
count(socket: SyncSocket): number {
|
|
35
|
+
return this.#bySocket.get(socket)?.size ?? 0;
|
|
36
|
+
}
|
|
37
|
+
}
|