@ultimat3/realtime 2.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -90,6 +90,7 @@ export {
90
90
  RealtimeError,
91
91
  type RealtimeErrorCode,
92
92
  RebaseConflictError,
93
+ ReplicaIdentityError,
93
94
  ReplicationFailedError,
94
95
  ReplicationProtocolError,
95
96
  ReplicatorSlotHeldError,
@@ -136,7 +137,6 @@ export {
136
137
  } from './identity-map';
137
138
  // ---- shared value domain ---------------------------------------------------------------------
138
139
  export {
139
- canonicalJson,
140
140
  changedColumns,
141
141
  fnv1a,
142
142
  isJsonObject,
@@ -147,11 +147,10 @@ export {
147
147
  type RowOp,
148
148
  type RowPatch,
149
149
  } from './json';
150
- export {
151
- type LiveQueryDefinition,
152
- type LiveSubscription,
153
- qidOf,
154
- type SnapshotResult,
150
+ export type {
151
+ LiveQueryDefinition,
152
+ LiveSubscription,
153
+ SnapshotResult,
155
154
  } from './live-contract';
156
155
  export { type LiveDefinitionOptions, liveQueryDefinition } from './live-definition';
157
156
  export {
@@ -263,6 +262,7 @@ export {
263
262
  createEntry,
264
263
  fillWindow,
265
264
  orgIdOf,
265
+ type PendingRead,
266
266
  type QueryEntry,
267
267
  refillWindowInLane,
268
268
  } from './query-window';
@@ -299,8 +299,10 @@ export {
299
299
  actorIdOf,
300
300
  CLOSE,
301
301
  DEFAULT_FRAME_BURST,
302
+ DEFAULT_IDLE_TIMEOUT_MS,
302
303
  DEFAULT_MAX_BUFFERED_BYTES,
303
304
  DEFAULT_MAX_FRAMES_PER_SECOND,
305
+ idleSweepPeriodMs,
304
306
  SocketRegistry,
305
307
  type SocketRegistryOptions,
306
308
  SyncSocket,
@@ -374,6 +376,7 @@ export {
374
376
  type AcceptBudgetOptions,
375
377
  type BackoffPolicy,
376
378
  backoffDelay,
379
+ type DrainedSocket,
377
380
  type DrainPlanEntry,
378
381
  type DrainPlanOptions,
379
382
  defaultBackoff,
package/src/json.ts CHANGED
@@ -1,5 +1,11 @@
1
- // The JSON value domain shared by the wire, the matcher, and the local store.
2
- // Kept dependency-free so every other module in this package can import it without a cycle.
1
+ // The JSON value domain shared by the wire, the matcher, and the local store, plus `fnv1a` — the
2
+ // one hash this package still owns.
3
+ //
4
+ // `canonicalJson` and `stableDigest` USED to live here and are `@ultimat3/core`'s now: they were a
5
+ // third copy of one injective canonical form and one sharing-key hash, beside `@ultimat3/action`'s
6
+ // and `@ultimat3/query`'s, and the copies had already diverged. `fnv1a` stays because its job is
7
+ // genuinely different — it is a cursor's result-set digest, where a collision costs a missed
8
+ // re-sort and never one client served out of another's window.
3
9
 
4
10
  export type JsonValue =
5
11
  | string
@@ -49,52 +55,6 @@ export function changedColumns(before: JsonObject | null, after: JsonObject): Js
49
55
  return out;
50
56
  }
51
57
 
52
- /**
53
- * Key-sorted JSON so a query id derived from input is stable across property order — and
54
- * INJECTIVE, because that id decides who shares a window.
55
- *
56
- * `qidOf` is `stableDigest(canonicalJson(input))` and a qid HIT hands the joiner the existing
57
- * entry: the first subscriber's compiled source, its matcher and its seated rows. Two inputs that
58
- * canonicalise to one string are therefore two clients served out of one window. `JSON.stringify`
59
- * is not injective over numbers — `NaN` and `±Infinity` are both `"null"`, which also collides
60
- * with JSON `null` itself, and `-0` is `"0"` — so the number branch is spelled out here.
61
- *
62
- * `-0` is the one of those a client can put on the wire (`JSON.parse('{"a":-0}')` answers `-0`);
63
- * the non-finite three have no JSON spelling and arrive only from a caller building `input` in JS,
64
- * such as `useLive(feed, () => ({ limit: Number.parseInt(raw) }))` on an unparseable `raw`.
65
- * The tokens are bare, never quoted: this output is only ever hashed, and the `string` branch
66
- * always quotes, so an unquoted word cannot collide with the text that spells it.
67
- *
68
- * The twin of `@ultimat3/query`'s and `@ultimat3/action`'s rules in their own `stable.ts`. All
69
- * three are tier 3, so no two of them can import each other; the shared home is `@ultimat3/core`
70
- * if one is ever made.
71
- */
72
- export function canonicalJson(value: JsonValue): string {
73
- if (typeof value === 'number') return canonicalNumber(value);
74
- if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
75
- if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
76
- const keys = Object.keys(value).sort();
77
- const parts = keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key] ?? null)}`);
78
- return `{${parts.join(',')}}`;
79
- }
80
-
81
- /** Ordinary numbers are `String(n)`, byte-identical to what this emitted before. */
82
- function canonicalNumber(value: number): string {
83
- if (Number.isNaN(value)) return 'NaN';
84
- if (!Number.isFinite(value)) return value > 0 ? 'Infinity' : '-Infinity';
85
- return Object.is(value, -0) ? '-0' : String(value);
86
- }
87
-
88
- /**
89
- * SHA-256, first 16 hex characters. For the hashes that are also SHARING keys — a `qid` decides
90
- * which subscribers are served from one window, and it is derived from input a client chooses, so
91
- * the 32 bits `fnv1a` answers are a collision anyone can find offline in seconds. Same primitive
92
- * and same width `@ultimat3/entity`'s `planScope` already chose for a cursor's scope.
93
- */
94
- export function stableDigest(text: string): string {
95
- return new Bun.CryptoHasher('sha256').update(text).digest('hex').slice(0, 16);
96
- }
97
-
98
58
  /** FNV-1a, 32-bit, hex. Not cryptographic — it identifies and detects drift, it does not protect. */
99
59
  export function fnv1a(text: string): string {
100
60
  let hash = 0x811c9dc5;
@@ -1,27 +1,20 @@
1
- // What a live query IS: the id one is keyed by, the contract a definition satisfies, and the
2
- // subscription one socket holds. Split from `live-query.ts` because four modules need the shape
3
- // and none of them needs the registry that runs it — and because one file runs one job.
1
+ // What a live query IS: the contract a definition satisfies and the subscription one socket holds.
2
+ // Split from `live-query.ts` because four modules need the shape and none of them needs the
3
+ // registry that runs it — and because one file runs one job.
4
+ //
5
+ // The id is NOT here, and no longer anywhere in this package: a `qid` is `@ultimat3/query`'s
6
+ // `queryHash(name, input)`, imported across the declared `realtime -> query` edge. `qidOf` was the
7
+ // same two lines over this package's own copy of the canonical form, and the two had already
8
+ // diverged on an `undefined`-valued key — while `planResume` compares a cursor's `queryHash` and
9
+ // `liveQueryDefinition` keys the shared window by the qid, so a divergence is every resume
10
+ // decision and every window lookup keyed differently.
4
11
 
5
12
  import type { Actor } from '@ultimat3/core';
6
13
  import type { LiveCursor } from './cursor';
7
- import { canonicalJson, type JsonValue, type Row, stableDigest } from './json';
14
+ import type { JsonValue, Row } from './json';
8
15
  import type { IncrementalMatcher } from './matcher-bridge';
9
16
  import type { SyncSocket } from './socket';
10
17
 
11
- /**
12
- * `qid` = hash(query name, input). Fanout subjects and change windows are keyed by it.
13
- *
14
- * The hash is a **sharing** key, which is why it is `stableDigest` and not `fnv1a`: `#entryFor`
15
- * answers a hit with the EXISTING entry and `liveQueryDefinition` answers with the seated
16
- * `SharedWindow`, both carrying the first subscriber's input, compiled source and rows. Input is
17
- * client-chosen, so a second input colliding with the first passes `authorize` against its own
18
- * arguments and is then served out of somebody else's window — and 32 bits is a collision found
19
- * offline in seconds.
20
- */
21
- export function qidOf(name: string, input: JsonValue): string {
22
- return `${name}:${stableDigest(canonicalJson(input))}`;
23
- }
24
-
25
18
  export interface SnapshotResult<R extends Row = Row> {
26
19
  readonly rows: readonly R[];
27
20
  readonly lsn: string;
@@ -9,10 +9,10 @@
9
9
  // every time. Collapsing the second onto the first is privilege escalation with a cache hit rate.
10
10
 
11
11
  import type { Ctx } from '@ultimat3/core';
12
- import { type AnyQuery, queryName } from '@ultimat3/query';
12
+ import { type AnyQuery, queryHash, queryName } from '@ultimat3/query';
13
13
  import { LiveRowUnidentifiedError } from './errors';
14
14
  import { isRow, type JsonValue, type Row } from './json';
15
- import { type LiveQueryDefinition, qidOf, type SnapshotResult } from './live-contract';
15
+ import type { LiveQueryDefinition, SnapshotResult } from './live-contract';
16
16
  import { type IncrementalMatcher, matcherFor } from './matcher-bridge';
17
17
  import { authorizeWithPolicy, visibleWithPolicy } from './policy-gate';
18
18
 
@@ -73,7 +73,7 @@ export function liveQueryDefinition(
73
73
  const windows = new Map<string, SharedWindow>();
74
74
 
75
75
  const resolve = async (input: JsonValue): Promise<SharedWindow> => {
76
- const qid = qidOf(name, input);
76
+ const qid = queryHash(name, input);
77
77
  const seated = windows.get(qid);
78
78
  if (seated !== undefined) return seated;
79
79
  const live = await target.live(input, {
@@ -113,10 +113,10 @@ export function liveQueryDefinition(
113
113
  const window = await resolve(input);
114
114
  return { rows: await window.read(), lsn: options.lsn?.() ?? '' };
115
115
  },
116
- matcher: (input) => windows.get(qidOf(name, input))?.matcher ?? UNRESOLVED,
116
+ matcher: (input) => windows.get(queryHash(name, input))?.matcher ?? UNRESOLVED,
117
117
  // Read off the same resolved window as the matcher, so the scope the client keys rows under and
118
118
  // the entity the matcher patches them from can never be two different names.
119
- rowEntity: (input) => windows.get(qidOf(name, input))?.rowEntity ?? null,
119
+ rowEntity: (input) => windows.get(queryHash(name, input))?.rowEntity ?? null,
120
120
  // The two per-subscriber gates, both through the package's one authz seam. Neither result is
121
121
  // memoised anywhere: `authorize` runs on every subscribe, `visible` on every row of every
122
122
  // delivery, and there is no key here an actor could share with another actor.
package/src/live-query.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  // otherwise.
8
8
 
9
9
  import { type Actor, type Clock, systemClock, uuid } from '@ultimat3/core';
10
+ import { queryHash } from '@ultimat3/query';
10
11
  import type { ChangeEvent } from './changefeed';
11
12
  import {
12
13
  type LiveCursor,
@@ -17,12 +18,7 @@ import {
17
18
  } from './cursor';
18
19
  import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
19
20
  import type { JsonValue } from './json';
20
- import {
21
- type LiveQueryDefinition,
22
- type LiveSubscription,
23
- qidOf,
24
- type SnapshotResult,
25
- } from './live-contract';
21
+ import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
26
22
  import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
27
23
  import { createEntry, fillWindow, type QueryEntry } from './query-window';
28
24
  import type { SyncSocket } from './socket';
@@ -176,7 +172,7 @@ export class LiveQueryRegistry {
176
172
  // may not subscribe is work an unauthorized client gets to schedule.
177
173
  await definition.prepare?.(args.input);
178
174
 
179
- const qid = qidOf(args.name, args.input);
175
+ const qid = queryHash(args.name, args.input);
180
176
  const entry = this.#entryFor(qid, definition, args.input);
181
177
  const now = this.#clock.now().getTime();
182
178
 
@@ -0,0 +1,117 @@
1
+ // Single responsibility: the four questions asked of a database BEFORE `START_REPLICATION`, and
2
+ // the identifier charset every one of them interpolates through. Three refuse the boot with the
3
+ // exact statement that fixes them; the fourth warns, because refusing it would stop every app on
4
+ // the default replica identity from starting.
5
+
6
+ import { logger } from '@ultimat3/core';
7
+ import { ReplicaIdentityError, ReplicationFailedError } from './errors';
8
+ import type { PgConnection } from './pg-connection';
9
+
10
+ /** Identifiers reach a simple query unparameterised, so the charset is the injection boundary. */
11
+ const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
12
+
13
+ /**
14
+ * The one gate between a caller-supplied name and a simple query. Exported because
15
+ * `PgReplicationStream` checks its slot, publication and entity names in its CONSTRUCTOR — a
16
+ * mistyped `REPLICATION_SLOT` is a boot-time fact, and finding it at the first WAL read means a
17
+ * replicator that reported itself started and then never delivered a change.
18
+ */
19
+ export const assertIdentifier = (kind: string, value: string): string => {
20
+ if (IDENTIFIER.test(value)) return value;
21
+ throw new ReplicationFailedError({
22
+ stage: 'preflight',
23
+ detail: `${kind} "${value}" is not a lower-case postgres identifier`,
24
+ fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
25
+ });
26
+ };
27
+
28
+ /**
29
+ * The four things that are always misconfigured. Three produce an unreadable server message if
30
+ * left to the server, so each gets its own `fix:` line; the fourth is `warnPartialIdentity` and
31
+ * only warns. `slot` and `publication` are interpolated into simple queries, so the `IDENTIFIER`
32
+ * charset is the injection boundary; re-asserted here rather than trusted, so the guarantee
33
+ * travels with the function instead of living only in `start()`.
34
+ */
35
+ export async function preflight(
36
+ connection: PgConnection,
37
+ slot: string,
38
+ publication: string,
39
+ entities: ReadonlySet<string>,
40
+ ): Promise<void> {
41
+ assertIdentifier('slot', slot);
42
+ assertIdentifier('publication', publication);
43
+ const [walLevel] = await connection.query('SHOW wal_level');
44
+ if (walLevel?.[0] !== 'logical') {
45
+ throw new ReplicationFailedError({
46
+ stage: 'preflight',
47
+ detail: `wal_level is "${walLevel?.[0] ?? 'unknown'}", so the server writes no logical WAL`,
48
+ fix: "ALTER SYSTEM SET wal_level = 'logical'; -- then restart postgres",
49
+ });
50
+ }
51
+ const publications = await connection.query(
52
+ `SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
53
+ );
54
+ if (publications.length === 0) {
55
+ throw new ReplicationFailedError({
56
+ stage: 'preflight',
57
+ detail: `no publication named "${publication}" exists`,
58
+ fix: `CREATE PUBLICATION ${publication} FOR ALL TABLES;`,
59
+ });
60
+ }
61
+ await warnPartialIdentity(connection, entities);
62
+ const [existing] = await connection.query(
63
+ `SELECT plugin FROM pg_replication_slots WHERE slot_name = '${slot}'`,
64
+ );
65
+ if (existing === undefined) {
66
+ // Plain SQL rather than CREATE_REPLICATION_SLOT: the replication command exports a snapshot
67
+ // that pins xmin for the session, and its option syntax changed in postgres 15.
68
+ await connection.query(`SELECT pg_create_logical_replication_slot('${slot}', 'pgoutput')`);
69
+ return;
70
+ }
71
+ if (existing[0] !== 'pgoutput') {
72
+ throw new ReplicationFailedError({
73
+ stage: 'preflight',
74
+ detail: `slot "${slot}" decodes with "${existing[0] ?? 'unknown'}", not pgoutput`,
75
+ fix: `SELECT pg_drop_replication_slot('${slot}'); -- then start the replicator again`,
76
+ });
77
+ }
78
+ }
79
+
80
+ /**
81
+ * The fourth preflight question, and the one that does NOT refuse. A live query decides whether a
82
+ * row left its result set from `change.before`, and under any replica identity but FULL that tuple
83
+ * is the key columns alone — which `toRow` accepts, since it only requires a text `id`.
84
+ *
85
+ * It runs BEFORE `pg_create_logical_replication_slot`: a slot decodes with the identity the
86
+ * catalog held when the rows were written, so asking after the slot exists answers about a stream
87
+ * nobody is reading yet. It WARNS rather than throws because every app on the default identity
88
+ * would otherwise stop booting, and a replicator that will not start is worse than the partial
89
+ * rows it is complaining about — `ReplicationStreamStats.partialBefore` is the running half.
90
+ *
91
+ * Entity names are the ones the constructor already put through `assertIdentifier`, which is what
92
+ * makes both the interpolation and the `fix:` safe; a name postgres answers with that is not in
93
+ * that set is dropped rather than rendered.
94
+ */
95
+ async function warnPartialIdentity(
96
+ connection: PgConnection,
97
+ entities: ReadonlySet<string>,
98
+ ): Promise<void> {
99
+ if (entities.size === 0) return;
100
+ const names = [...entities].map((name) => `'${name}'`).join(', ');
101
+ const rows = await connection.query(
102
+ `SELECT relname FROM pg_class WHERE relkind = 'r' AND relreplident <> 'f' ` +
103
+ `AND relname IN (${names})`,
104
+ );
105
+ const tables = [
106
+ ...new Set(
107
+ rows
108
+ .map((row) => row[0])
109
+ .filter((name): name is string => typeof name === 'string' && entities.has(name)),
110
+ ),
111
+ ].sort();
112
+ if (tables.length === 0) return;
113
+ const warning = new ReplicaIdentityError({ tables });
114
+ // FIELDS, never interpolation, and the message is the CODE alone — the same rule
115
+ // `@ultimat3/http`'s error-map stage follows, so a log index can be alerted on by code.
116
+ logger.warn(warning.code, { cause: warning.cause, fix: warning.fix, tables });
117
+ }
@@ -1,21 +1,19 @@
1
1
  // Single responsibility: turn a Postgres logical-replication slot into ordered `ChangeEvent`s —
2
- // preflight the three things that are always misconfigured, START_REPLICATION, decode pgoutput,
3
- // and keep the slot confirmed. The connection, the framing and the pgoutput decode live next door;
4
- // what is decided here is *ordering*, because the lsn is the only authority the pipeline has.
2
+ // START_REPLICATION, decode pgoutput, and keep the slot confirmed. The preflight, the connection,
3
+ // the framing and the pgoutput decode live next door; what is decided here is *ordering*, because
4
+ // the lsn is the only authority the pipeline has.
5
5
 
6
6
  import { type Clock, logger, systemClock } from '@ultimat3/core';
7
7
  import type { ChangeEvent, ChangeOp, PgLogicalReplicationOptions } from './changefeed';
8
- import { ReplicationFailedError, ReplicationProtocolError } from './errors';
8
+ import { ReplicationProtocolError } from './errors';
9
9
  import { isRow, type JsonObject, type Row } from './json';
10
10
  import { ByteReader, ByteWriter, epochMsToPgTimestamp, printLsn } from './pg-bytes';
11
11
  import { PgConnection } from './pg-connection';
12
12
  import { entityRow } from './pg-entity-row';
13
+ import { assertIdentifier, preflight } from './pg-preflight';
13
14
  import { bunPgStream, parsePgUrl } from './pg-socket';
14
15
  import { PgOutputDecoder, type PgOutputMessage, type PgRelation } from './pgoutput';
15
16
 
16
- /** Identifiers reach a simple query unparameterised, so the charset is the injection boundary. */
17
- const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
18
-
19
17
  const DEFAULT_STATUS_INTERVAL_MS = 10_000;
20
18
 
21
19
  /** `r` — the standby status update, the only frontend message a walsender listens for. */
@@ -27,6 +25,14 @@ export interface ReplicationStreamStats {
27
25
  readonly skipped: number;
28
26
  /** Rows replayed from before the resume position, dropped so `onChange` sees each one once. */
29
27
  readonly replayed: number;
28
+ /**
29
+ * Changes delivered from a relation whose replica identity is not FULL — so the `before` row is
30
+ * the key columns alone (or absent), and the live matcher's "did this row leave the result set"
31
+ * is decided on a partial row. `X_LIVE_REPLICA_IDENTITY` warns about the CONFIGURATION once at
32
+ * preflight; this counts the decisions it actually cost, which is the half a running node can
33
+ * be alerted on. Inserts never count: there is no `before` to be partial.
34
+ */
35
+ readonly partialBefore: number;
30
36
  /**
31
37
  * Why the pump stopped, or `null` while it is live. The read loop cannot throw into a caller —
32
38
  * nothing awaits it — so this is the one place `/readyz` and a test can see that it died at all.
@@ -58,15 +64,6 @@ export const changeLsn = (commitLsn: bigint, sequence: number): string =>
58
64
  /** The commit position inside a change lsn — where a resume asks the server to restart. */
59
65
  export const commitPositionOf = (lsn: string): bigint => BigInt(`0x${lsn.slice(0, 16) || '0'}`);
60
66
 
61
- const assertIdentifier = (kind: string, value: string): string => {
62
- if (IDENTIFIER.test(value)) return value;
63
- throw new ReplicationFailedError({
64
- stage: 'preflight',
65
- detail: `${kind} "${value}" is not a lower-case postgres identifier`,
66
- fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
67
- });
68
- };
69
-
70
67
  export interface ReplicationStreamHandlers {
71
68
  readonly from?: string | undefined;
72
69
  onChange(event: ChangeEvent): void | Promise<void>;
@@ -94,6 +91,7 @@ export class PgReplicationStream {
94
91
  #delivered = 0;
95
92
  #skipped = 0;
96
93
  #replayed = 0;
94
+ #partialBefore = 0;
97
95
  #failure: string | null = null;
98
96
 
99
97
  constructor(options: PgLogicalReplicationOptions) {
@@ -116,6 +114,7 @@ export class PgReplicationStream {
116
114
  delivered: this.#delivered,
117
115
  skipped: this.#skipped,
118
116
  replayed: this.#replayed,
117
+ partialBefore: this.#partialBefore,
119
118
  failure: this.#failure,
120
119
  };
121
120
  }
@@ -147,7 +146,7 @@ export class PgReplicationStream {
147
146
  });
148
147
  this.#connection = connection;
149
148
  try {
150
- await preflight(connection, slot, publication);
149
+ await preflight(connection, slot, publication, this.#entities);
151
150
  const from = handlers.from;
152
151
  this.#confirmed = from === undefined ? 0n : commitPositionOf(from);
153
152
  await connection.startCopyBoth(
@@ -339,6 +338,10 @@ export class PgReplicationStream {
339
338
  this.#replayed += 1;
340
339
  return;
341
340
  }
341
+ // Read off the Relation message rather than off the tuple: a DEFAULT-identity table whose
342
+ // non-key columns happen to be NULL sends the same bytes a FULL one does, so counting missing
343
+ // keys would undercount exactly the rows a policy is most likely to misjudge.
344
+ if (op !== 'insert' && relation.replicaIdentity !== 'f') this.#partialBefore += 1;
342
345
  const before = toRow(relation, oldTuple);
343
346
  const after = toRow(relation, newTuple);
344
347
  const event: ChangeEvent = {
@@ -380,55 +383,6 @@ export class PgReplicationStream {
380
383
  }
381
384
  }
382
385
 
383
- /**
384
- * The three misconfigurations that produce an unreadable server message if left to the server.
385
- * `slot` and `publication` are interpolated into simple queries, so the `IDENTIFIER` charset is the
386
- * injection boundary; re-asserted here rather than trusted, so the guarantee travels with the
387
- * function instead of living only in `start()`.
388
- */
389
- async function preflight(
390
- connection: PgConnection,
391
- slot: string,
392
- publication: string,
393
- ): Promise<void> {
394
- assertIdentifier('slot', slot);
395
- assertIdentifier('publication', publication);
396
- const [walLevel] = await connection.query('SHOW wal_level');
397
- if (walLevel?.[0] !== 'logical') {
398
- throw new ReplicationFailedError({
399
- stage: 'preflight',
400
- detail: `wal_level is "${walLevel?.[0] ?? 'unknown'}", so the server writes no logical WAL`,
401
- fix: "ALTER SYSTEM SET wal_level = 'logical'; -- then restart postgres",
402
- });
403
- }
404
- const publications = await connection.query(
405
- `SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
406
- );
407
- if (publications.length === 0) {
408
- throw new ReplicationFailedError({
409
- stage: 'preflight',
410
- detail: `no publication named "${publication}" exists`,
411
- fix: `CREATE PUBLICATION ${publication} FOR ALL TABLES;`,
412
- });
413
- }
414
- const [existing] = await connection.query(
415
- `SELECT plugin FROM pg_replication_slots WHERE slot_name = '${slot}'`,
416
- );
417
- if (existing === undefined) {
418
- // Plain SQL rather than CREATE_REPLICATION_SLOT: the replication command exports a snapshot
419
- // that pins xmin for the session, and its option syntax changed in postgres 15.
420
- await connection.query(`SELECT pg_create_logical_replication_slot('${slot}', 'pgoutput')`);
421
- return;
422
- }
423
- if (existing[0] !== 'pgoutput') {
424
- throw new ReplicationFailedError({
425
- stage: 'preflight',
426
- detail: `slot "${slot}" decodes with "${existing[0] ?? 'unknown'}", not pgoutput`,
427
- fix: `SELECT pg_drop_replication_slot('${slot}'); -- then start the replicator again`,
428
- });
429
- }
430
- }
431
-
432
386
  /** A physical tuple becomes the row the matcher's predicates are written against, or nothing. */
433
387
  function toRow(relation: PgRelation, physical: JsonObject | null): Row | null {
434
388
  if (physical === null) return null;
@@ -32,7 +32,24 @@ export interface QueryEntry {
32
32
  /** Serial lane over `rows`/`lsn`. Every fanout and every window assignment takes its turn here. */
33
33
  readonly lock: WindowLock;
34
34
  /** The read in flight, shared by every subscriber that arrives during it. `null` between reads. */
35
- reading: Promise<SnapshotResult> | null;
35
+ reading: PendingRead | null;
36
+ /**
37
+ * Reads issued against this entry, ever. It is the ORDER of two reads, which nothing else here
38
+ * can answer: a definition with no lsn provider returns `''` from every snapshot.
39
+ */
40
+ generation: number;
41
+ /** The generation of the newest read whose rows are in `rows`. `0` before the first one lands. */
42
+ applied: number;
43
+ }
44
+
45
+ /**
46
+ * One read and which read it is. They are one fact — a joiner needs the promise AND the generation
47
+ * it will have to compare against when it lands — and two fields on the entry is two writes a later
48
+ * edit can separate.
49
+ */
50
+ export interface PendingRead {
51
+ readonly generation: number;
52
+ readonly result: Promise<SnapshotResult>;
36
53
  }
37
54
 
38
55
  export function createEntry(
@@ -64,6 +81,8 @@ export function createEntry(
64
81
  stale: false,
65
82
  lock: new WindowLock(),
66
83
  reading: null,
84
+ generation: 0,
85
+ applied: 0,
67
86
  };
68
87
  }
69
88
 
@@ -82,18 +101,21 @@ export async function fillWindow(
82
101
  // Read before `startRead` clears it: a second caller arriving during the read joins it and is
83
102
  // not the one that forced it, which is what keeps one forced read from becoming N.
84
103
  const forced = entry.stale;
85
- const result = await (forced || entry.reading === null ? startRead(entry) : entry.reading);
104
+ const pending = forced || entry.reading === null ? startRead(entry) : entry.reading;
105
+ const result = await pending.result;
86
106
  return await entry.lock.run(async () => {
87
- if (forced) {
88
- // A forced read replaces the window whatever its lsn says: it was issued *because* what is
89
- // under it is wrong, and a definition with no lsn provider answers `''` — which the
90
- // never-backwards rule below would read as older than what we hold and discard, leaving
91
- // every subscriber served from the window the gap already invalidated.
92
- entry.rows = result.rows;
93
- if (result.lsn > entry.lsn) entry.lsn = result.lsn;
94
- } else if (result.lsn >= entry.lsn) {
95
- entry.rows = result.rows;
96
- entry.lsn = result.lsn;
107
+ // Two rules, and neither can stand in for the other. Against another READ it is identity —
108
+ // the same check `startRead` makes on `entry.reading` one function down, and the one
109
+ // `packages/cache/src/single-flight.ts` makes for the same reason because an lsn cannot
110
+ // order two reads at all: a definition with no lsn provider answers `''` for both, and
111
+ // `'' >= ''` let the older one overwrite the gap repair the newer one had just landed, with
112
+ // `stale` already cleared by its issue and therefore nothing left to re-read. Against a
113
+ // CHANGE it is still the lsn, because a fanout moved `entry.lsn` forwards while this read was
114
+ // in flight and rewinding to what the read saw hands that subscriber rows the fanout has
115
+ // moved past — except for a forced read, which was issued *because* what is under it is
116
+ // wrong.
117
+ if (isNewestRead(entry, pending) && (forced || result.lsn >= entry.lsn)) {
118
+ applyRead(entry, pending, result);
97
119
  }
98
120
  return { rows: entry.rows, lsn: entry.lsn };
99
121
  });
@@ -105,22 +127,36 @@ export async function fillWindow(
105
127
  * that repairs a stale window mid-fanout is spelled here rather than deadlocking on the other.
106
128
  */
107
129
  export async function refillWindowInLane(entry: QueryEntry): Promise<void> {
108
- const result = await startRead(entry);
130
+ const pending = startRead(entry);
131
+ const result = await pending.result;
132
+ // Same identity rule as `fillWindow`: a read issued before this one may still be in flight, and
133
+ // whichever was issued LAST is the one the window keeps.
134
+ if (isNewestRead(entry, pending)) applyRead(entry, pending, result);
135
+ }
136
+
137
+ /** Is this the newest read to have landed? An older one's rows are behind the window, not on it. */
138
+ function isNewestRead(entry: QueryEntry, pending: PendingRead): boolean {
139
+ return pending.generation > entry.applied;
140
+ }
141
+
142
+ function applyRead(entry: QueryEntry, pending: PendingRead, result: SnapshotResult): void {
143
+ entry.applied = pending.generation;
109
144
  entry.rows = result.rows;
110
145
  if (result.lsn > entry.lsn) entry.lsn = result.lsn;
111
146
  }
112
147
 
113
148
  /** Publishes the in-flight read, and clears it as it settles — the share is per read, not a cache. */
114
- function startRead(entry: QueryEntry): Promise<SnapshotResult> {
149
+ function startRead(entry: QueryEntry): PendingRead {
115
150
  // Cleared here rather than when the read lands: the read about to be issued is the one that
116
151
  // answers the staleness, so a second caller must join it instead of forcing another.
117
152
  entry.stale = false;
118
- const reading = readSnapshot(entry);
153
+ entry.generation += 1;
154
+ const reading: PendingRead = { generation: entry.generation, result: readSnapshot(entry) };
119
155
  entry.reading = reading;
120
156
  const done = (): void => {
121
157
  if (entry.reading === reading) entry.reading = null;
122
158
  };
123
- void reading.then(done, done);
159
+ void reading.result.then(done, done);
124
160
  return reading;
125
161
  }
126
162
 
package/src/rebase.ts CHANGED
@@ -9,7 +9,7 @@
9
9
  import { RebaseConflictError } from './errors';
10
10
  import type { Row } from './json';
11
11
  import type { LocalStore, LocalTx, TableMap } from './local-store';
12
- import { type ConflictStrategyName, type Frame, PROTOCOL_VERSION } from './sync-protocol';
12
+ import { type ConflictStrategyName, PROTOCOL_VERSION, type RebaseFrame } from './sync-protocol';
13
13
 
14
14
  export interface MergeArgs {
15
15
  /** Local row as the user last saw it, before any rollback. */
@@ -246,7 +246,12 @@ function numberAt(row: Row | null | undefined, field: string): number | null {
246
246
  return typeof value === 'number' ? value : null;
247
247
  }
248
248
 
249
- export function rebaseFrame(ack: ServerAck, strategy: ConflictStrategy): Frame {
249
+ /**
250
+ * `RebaseFrame`, not `Frame`: this builds exactly one member of the union and declaring the whole
251
+ * union threw that away, so every caller had to re-narrow a frame it had just constructed before it
252
+ * could read `strategy` or `row` back off it.
253
+ */
254
+ export function rebaseFrame(ack: ServerAck, strategy: ConflictStrategy): RebaseFrame {
250
255
  return {
251
256
  type: 'rebase',
252
257
  v: PROTOCOL_VERSION,