@syncular/server 0.69.0 → 0.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/context.d.ts CHANGED
@@ -73,6 +73,13 @@ export interface ServerLimits {
73
73
  readonly maxSubscriptionsPerRequest: number;
74
74
  /** Cap on concurrent registered-query watches per watch session. */
75
75
  readonly maxWatchesPerSession: number;
76
+ /**
77
+ * §4.6 byte budget of one live section: the stored bytes (payload, row id,
78
+ * scopes) of the changes a pull loads from the log. A section stops before
79
+ * the commit that would pass it; a commit whose changes for the scan set
80
+ * pass it alone resets the units it reaches, which bootstrap instead.
81
+ */
82
+ readonly maxLiveSectionBytes: number;
76
83
  }
77
84
  export declare const DEFAULT_LIMITS: ServerLimits;
78
85
  /** Receives applied commits for realtime fanout (§8.2). */
package/dist/context.js CHANGED
@@ -21,6 +21,8 @@ export const DEFAULT_LIMITS = {
21
21
  maxSubscriptionsPerRequest: 500,
22
22
  // Every commit reruns a session's affected watches one after another.
23
23
  maxWatchesPerSession: 100,
24
+ // The live section is buffered whole before the first unit section.
25
+ maxLiveSectionBytes: 16 * 1024 * 1024,
24
26
  };
25
27
  export function clockOf(ctx) {
26
28
  return ctx.clock ?? Date.now;
package/dist/handler.js CHANGED
@@ -597,19 +597,33 @@ async function* streamResponse(plan, ctx, schema, report) {
597
597
  // resets the live section), and the section is read again.
598
598
  let live;
599
599
  for (;;) {
600
- const scan = units.flatMap(({ plan: unit }, index) => {
600
+ const scanned = units.flatMap(({ plan: unit }, index) => {
601
601
  const afterSeq = bound(states[index]);
602
602
  return afterSeq === undefined
603
603
  ? []
604
- : [{ scopes: unit.scopes, afterSeq, tables: unit.pairs }];
604
+ : [{ index, scopes: unit.scopes, afterSeq, tables: unit.pairs }];
605
605
  });
606
- if (liveReset || scan.length === 0)
606
+ if (liveReset || scanned.length === 0)
607
607
  break;
608
608
  live = await ctx.storage.readLiveWindow(ctx.partition, {
609
- units: scan,
609
+ units: scanned,
610
610
  throughSeq: maxSeq,
611
611
  limitChanges: limits.limitCommits,
612
+ limitBytes: limitsOf(ctx).maxLiveSectionBytes,
612
613
  });
614
+ // §4.6: a commit past the byte budget on its own cannot be served
615
+ // from the log; the units it reaches reset and bootstrap.
616
+ for (const at of live.oversized?.units ?? []) {
617
+ const index = scanned[at].index;
618
+ if (states[index].kind === 'live')
619
+ liveReset = true;
620
+ else
621
+ states[index] = { kind: 'reset' };
622
+ }
623
+ if (live.oversized !== undefined) {
624
+ live = undefined;
625
+ continue;
626
+ }
613
627
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
614
628
  let expired = false;
615
629
  for (const [index, state] of states.entries()) {
@@ -86,6 +86,10 @@ export async function openLocalAuthority(options) {
86
86
  segments: new MemorySegmentStore(),
87
87
  resolveScopes: () => allowed,
88
88
  clock,
89
+ // §12.2: the client's own process serves its own commits; the
90
+ // live byte budget (§4.6) bounds a shared server's heap and does
91
+ // not apply here, as in the native authority.
92
+ limits: { maxLiveSectionBytes: Number.MAX_SAFE_INTEGER },
89
93
  ...(options.crdtMergers !== undefined
90
94
  ? { crdtMergers: options.crdtMergers }
91
95
  : {}),
@@ -2253,24 +2253,32 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2253
2253
  return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
2254
2254
  }
2255
2255
  readLiveWindow(partition, query) {
2256
- // One statement per page: one LATERAL range of
2256
+ // Two statements per page. The candidate read takes one LATERAL range of
2257
2257
  // `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
2258
- // the index, merged in commit order and LIMITed again, then LEFT JOINed
2259
- // to the commit meta and every change of the commit. A candidate whose
2260
- // commit vanished comes back with NULL meta and still advances the scan.
2258
+ // the index, merged in commit order and LIMITed again, then LEFT JOINs
2259
+ // the commit meta (NULL when the commit vanished, which still advances
2260
+ // the scan) and, per key, the byte size of the changes the key reads;
2261
+ // `octet_length` of a TOASTed payload reads its pointer, never the value.
2262
+ // The change read then loads only those changes of the commits the byte
2263
+ // budget admits.
2261
2264
  return readLiveWindowPages(query, async (keys, limit) => {
2262
- const { rows } = await this.#exec.query(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2263
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
2264
- FROM (SELECT DISTINCT s.commit_seq
2265
- FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
2266
- CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
2265
+ const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
2266
+ c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
2267
+ FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
2267
2268
  WHERE partition=$1 AND var=k.var AND value=k.value
2268
2269
  AND commit_seq>k.after AND commit_seq<=$3
2269
2270
  ORDER BY commit_seq LIMIT $4) s
2270
- ORDER BY s.commit_seq LIMIT $4) c
2271
- LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2272
- LEFT JOIN sync_changes ch ON ch.partition=$1 AND ch.commit_seq=c.commit_seq
2273
- ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
2271
+ ORDER BY s.commit_seq LIMIT $4)
2272
+ SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2273
+ z.key AS key, z.bytes AS bytes
2274
+ FROM c
2275
+ LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2276
+ LEFT JOIN LATERAL (SELECT k.i AS key,
2277
+ sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
2278
+ FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value
2279
+ WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
2280
+ GROUP BY k.i) z ON true
2281
+ ORDER BY c.commit_seq, z.key`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
2274
2282
  return rows.map((record) => ({
2275
2283
  commitSeq: asNumber(record.commit_seq),
2276
2284
  ...(record.actor_id === null
@@ -2281,9 +2289,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2281
2289
  createdAtMs: asNumber(record.created_at_ms),
2282
2290
  },
2283
2291
  }),
2284
- ...(record.tbl === null
2292
+ ...(record.key === null
2285
2293
  ? {}
2286
- : { change: toStoredChange({ ...record, tbl: record.tbl }) }),
2294
+ : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
2295
+ }));
2296
+ }, async (keys, after, through) => {
2297
+ const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
2298
+ SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
2299
+ ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
2300
+ FROM sync_changes ch
2301
+ WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
2302
+ AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value)
2303
+ ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through]);
2304
+ return rows.map((record) => ({
2305
+ commitSeq: asNumber(record.commit_seq),
2306
+ change: toStoredChange(record),
2287
2307
  }));
2288
2308
  });
2289
2309
  }
package/dist/realtime.js CHANGED
@@ -642,9 +642,19 @@ export class RealtimeSession {
642
642
  // the keys before, reconcile after (leaves on lost keys, snapshots on
643
643
  // gained keys).
644
644
  const previousKeys = this.scopeKeys;
645
+ const previous = this.registrations;
645
646
  this.registrations = registrations;
646
647
  this.scopeKeys = this.#hub.scopeKeysOf(registrations);
647
648
  this.reconcilePresence(previousKeys);
649
+ // §8.7: a host refresh that removes or narrows a registration wakes the
650
+ // session, so its client runs the round whose §3.3 revocation purge
651
+ // drops the rows it lost. A registration survives when the same unit
652
+ // and table come back with every previous scope value still allowed.
653
+ if (failClosed &&
654
+ previous.some((old) => !registrations.some((next) => next.id === old.id &&
655
+ next.table === old.table &&
656
+ Object.entries(next.effective).every(([variable, values]) => old.effective[variable]?.every((value) => values.includes(value)) ?? false))))
657
+ this.sendWake('catchup-required');
648
658
  }
649
659
  /** §8.7 protocol violation: drop the connection, fail loud. The
650
660
  * message is intentionally unsent — there is no error channel for an
@@ -788,24 +788,30 @@ export class SqliteFamilyStorage {
788
788
  return row?.seq ?? 0;
789
789
  }
790
790
  readLiveWindow(partition, query) {
791
- // One statement per page: the unit keys bind as one JSON array, and the
792
- // CROSS JOIN keeps them the outer loop, so each key seeks one range of
793
- // `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
794
- // missing). The candidates LEFT JOIN the commit meta and every change of
795
- // the commit; a candidate whose commit vanished comes back with NULL
796
- // meta and still advances the scan.
797
- return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
798
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
799
- FROM (SELECT DISTINCT s.commit_seq AS commit_seq
800
- FROM json_each(?2) AS k
801
- CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
802
- ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
803
- AND s.value=json_extract(k.value, '$.value')
804
- AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
805
- ORDER BY s.commit_seq LIMIT ?4) AS c
806
- LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
807
- LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
808
- ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
791
+ // Two statements per page. The candidate read binds the unit keys as one
792
+ // JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
793
+ // one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
794
+ // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
795
+ // when the commit vanished, which still advances the scan) and, per key,
796
+ // the byte size of the changes the key reads; `length()` of a blob reads
797
+ // the record header, never the payload. The change read then loads only
798
+ // those changes of the commits the byte budget admits.
799
+ return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
800
+ c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
801
+ FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
802
+ ON s.partition=?1 AND s.var=k.var AND s.value=k.value
803
+ AND s.commit_seq>k.after AND s.commit_seq<=?3
804
+ ORDER BY s.commit_seq LIMIT ?4)
805
+ SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
806
+ z.i AS key, z.bytes AS bytes
807
+ FROM c
808
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
809
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
810
+ sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
811
+ FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
812
+ CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
813
+ GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
814
+ ORDER BY c.commit_seq, z.i`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
809
815
  commitSeq: record.commit_seq,
810
816
  ...(record.actor_id !== null && record.created_at_ms !== null
811
817
  ? {
@@ -815,9 +821,18 @@ export class SqliteFamilyStorage {
815
821
  },
816
822
  }
817
823
  : {}),
818
- ...(record.tbl !== null
819
- ? { change: toStoredChange({ ...record, tbl: record.tbl }) }
824
+ ...(record.key !== null
825
+ ? { key: record.key, bytes: record.bytes ?? 0 }
820
826
  : {}),
827
+ })), async (keys, after, through) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
828
+ SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
829
+ ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
830
+ FROM sync_changes AS ch
831
+ WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
832
+ AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
833
+ ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through])).map((record) => ({
834
+ commitSeq: record.commit_seq,
835
+ change: toStoredChange(record),
821
836
  })));
822
837
  }
823
838
  async scanRows(partition, query) {
@@ -15,32 +15,52 @@ export declare function resolveIndexRowScan(table: CompiledTable, query: IndexRo
15
15
  * which reads outside the transaction.
16
16
  */
17
17
  export declare function transactionQuery(tx: StorageTransaction, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
18
- /** One index range of the live scan: a unit key and its exclusive lower bound. */
18
+ /**
19
+ * One index range of the live scan: a unit key, its exclusive lower bound,
20
+ * and the tables its units cover. `i` is the key's position in the list.
21
+ */
19
22
  export interface LiveKey {
23
+ readonly i: number;
20
24
  readonly var: string;
21
25
  readonly value: string;
22
26
  readonly after: number;
27
+ readonly tables: readonly string[];
23
28
  }
24
29
  /**
25
- * One row of a live-window page: a candidate commit joined to its metadata
26
- * (absent when the commit vanished) and one of its changes (absent when it
27
- * has none), in (commitSeq, change index) order.
30
+ * One row of a live-window candidate page: a candidate commit joined to its
31
+ * metadata (absent when the commit vanished) and, per key, the stored bytes
32
+ * (payload, row id, scopes) of its changes that key reads: a covered table
33
+ * and the key's value in the stored scopes. A commit with no such change
34
+ * comes back once without `key`. Rows are in (commitSeq, key) order.
28
35
  */
29
- export interface LiveWindowRow {
36
+ export interface LiveCandidateRow {
30
37
  readonly commitSeq: number;
31
38
  readonly meta?: {
32
39
  readonly actorId: string;
33
40
  readonly createdAtMs: number;
34
41
  };
35
- readonly change?: StoredChange;
42
+ readonly key?: number;
43
+ readonly bytes?: number;
36
44
  }
37
45
  /**
38
- * The §4.5 live scan over a backend's page read. Each distinct unit key
39
- * (the first variable, keys sorted) becomes one index range starting at the
40
- * lowest bound of the units that share it; `readPage` returns up to `limit`
41
- * candidate commits above every range's bound. A change is kept when a unit
42
- * covers its table, the pair's effective scopes match it, and its commit
43
- * lies above that unit's own bound. Every candidate advances the scan, so a
44
- * page of non-matching candidates never repeats.
46
+ * The §4.5 live scan over a backend's two reads. `readCandidates` returns
47
+ * up to `limit` candidate commits above every key range's bound with their
48
+ * per-key byte sizes, and no payload. `readChanges` loads the changes the
49
+ * keys read in a commit range `(after, through]`, in (commitSeq, change
50
+ * index) order. Each distinct unit key (the first variable, keys sorted)
51
+ * becomes one index range starting at the lowest bound of the units that
52
+ * share it. A change is kept when a unit covers its table, the pair's
53
+ * effective scopes match it, and its commit lies above that unit's own
54
+ * bound. Every candidate advances the scan, so a page of non-matching
55
+ * candidates never repeats.
56
+ *
57
+ * The sizes bound the loaded bytes by `limitBytes` before any payload is
58
+ * read: the section stops before the commit that would pass it, and a
59
+ * commit that passes it alone ends the read as `oversized`, naming the
60
+ * units of the keys that read the most of it, largest first, until the rest
61
+ * fits (§4.6).
45
62
  */
46
- export declare function readLiveWindowPages(query: LiveWindowQuery, readPage: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveWindowRow[]>): Promise<LiveWindow>;
63
+ export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], after: number, through: number) => Promise<readonly {
64
+ readonly commitSeq: number;
65
+ readonly change: StoredChange;
66
+ }[]>): Promise<LiveWindow>;
@@ -48,19 +48,31 @@ export async function transactionQuery(tx, query) {
48
48
  return tx.queryAuthoritative(query);
49
49
  }
50
50
  /**
51
- * The §4.5 live scan over a backend's page read. Each distinct unit key
52
- * (the first variable, keys sorted) becomes one index range starting at the
53
- * lowest bound of the units that share it; `readPage` returns up to `limit`
54
- * candidate commits above every range's bound. A change is kept when a unit
55
- * covers its table, the pair's effective scopes match it, and its commit
56
- * lies above that unit's own bound. Every candidate advances the scan, so a
57
- * page of non-matching candidates never repeats.
51
+ * The §4.5 live scan over a backend's two reads. `readCandidates` returns
52
+ * up to `limit` candidate commits above every key range's bound with their
53
+ * per-key byte sizes, and no payload. `readChanges` loads the changes the
54
+ * keys read in a commit range `(after, through]`, in (commitSeq, change
55
+ * index) order. Each distinct unit key (the first variable, keys sorted)
56
+ * becomes one index range starting at the lowest bound of the units that
57
+ * share it. A change is kept when a unit covers its table, the pair's
58
+ * effective scopes match it, and its commit lies above that unit's own
59
+ * bound. Every candidate advances the scan, so a page of non-matching
60
+ * candidates never repeats.
61
+ *
62
+ * The sizes bound the loaded bytes by `limitBytes` before any payload is
63
+ * read: the section stops before the commit that would pass it, and a
64
+ * commit that passes it alone ends the read as `oversized`, naming the
65
+ * units of the keys that read the most of it, largest first, until the rest
66
+ * fits (§4.6).
58
67
  */
59
- export async function readLiveWindowPages(query, readPage) {
68
+ export async function readLiveWindowPages(query, readCandidates, readChanges) {
60
69
  const bounds = new Map();
70
+ const unitKeys = [];
61
71
  for (const unit of query.units) {
62
- if (unit.tables.size === 0)
72
+ if (unit.tables.size === 0) {
73
+ unitKeys.push(undefined);
63
74
  continue;
75
+ }
64
76
  const variable = Object.keys(unit.scopes).sort()[0];
65
77
  const value = variable === undefined ? undefined : unit.scopes[variable];
66
78
  if (variable === undefined || value === undefined) {
@@ -68,47 +80,113 @@ export async function readLiveWindowPages(query, readPage) {
68
80
  }
69
81
  const id = JSON.stringify([variable, value]);
70
82
  const prior = bounds.get(id);
71
- if (prior === undefined || unit.afterSeq < prior.after) {
72
- bounds.set(id, { var: variable, value, after: unit.afterSeq });
73
- }
83
+ const tables = new Set([...(prior?.tables ?? []), ...unit.tables.keys()]);
84
+ const i = prior?.key.i ?? bounds.size;
85
+ bounds.set(id, {
86
+ key: {
87
+ i,
88
+ var: variable,
89
+ value,
90
+ after: Math.min(prior?.key.after ?? unit.afterSeq, unit.afterSeq),
91
+ tables: [...tables],
92
+ },
93
+ tables,
94
+ });
95
+ unitKeys.push(i);
74
96
  }
97
+ const keys = [...bounds.values()].map(({ key }) => key);
75
98
  const commits = [];
76
99
  const limit = Math.max(64, query.limitChanges);
77
100
  let delivered = 0;
101
+ let loaded = 0;
78
102
  let scanned = -1;
79
- while (bounds.size > 0) {
80
- const rows = await readPage([...bounds.values()].map((key) => ({
81
- ...key,
82
- after: Math.max(key.after, scanned),
83
- })), limit);
84
- let candidates = 0;
103
+ // Candidates sized but not loaded yet: (scanned, pending.at(-1)].
104
+ let pending = [];
105
+ const scanKeys = () => keys.map((key) => ({ ...key, after: Math.max(key.after, scanned) }));
106
+ // Loads the pending candidates; returns T when `limitChanges` cut there.
107
+ const load = async () => {
108
+ const last = pending.at(-1);
109
+ if (last === undefined)
110
+ return undefined;
111
+ const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
112
+ const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
113
+ pending = [];
114
+ scanned = last.commitSeq;
85
115
  for (let i = 0; i < rows.length;) {
86
- const head = rows[i];
87
- if (head === undefined)
88
- break;
89
- candidates += 1;
90
- scanned = head.commitSeq;
116
+ const seq = rows[i].commitSeq;
91
117
  const changes = [];
92
- for (; rows[i]?.commitSeq === head.commitSeq; i++) {
93
- const change = rows[i]?.change;
94
- if (change !== undefined &&
95
- query.units.some((unit) => {
96
- const effective = unit.tables.get(change.table);
97
- return (head.commitSeq > unit.afterSeq &&
98
- effective !== undefined &&
99
- matchesEffective(change.scopes, effective));
100
- })) {
118
+ for (; rows[i]?.commitSeq === seq; i++) {
119
+ const change = rows[i].change;
120
+ if (query.units.some((unit) => {
121
+ const effective = unit.tables.get(change.table);
122
+ return (seq > unit.afterSeq &&
123
+ effective !== undefined &&
124
+ matchesEffective(change.scopes, effective));
125
+ })) {
101
126
  changes.push(change);
102
127
  }
103
128
  }
104
- if (head.meta === undefined || changes.length === 0)
129
+ const meta = metas.get(seq);
130
+ if (meta === undefined || changes.length === 0)
105
131
  continue;
106
- commits.push({ commitSeq: head.commitSeq, ...head.meta, changes });
132
+ commits.push({ commitSeq: seq, ...meta, changes });
107
133
  delivered += changes.length;
108
- if (delivered >= query.limitChanges) {
109
- return { commits, throughSeq: head.commitSeq };
134
+ if (delivered >= query.limitChanges)
135
+ return seq;
136
+ }
137
+ return undefined;
138
+ };
139
+ while (keys.length > 0) {
140
+ const rows = await readCandidates(scanKeys(), limit);
141
+ let candidates = 0;
142
+ for (let i = 0; i < rows.length;) {
143
+ const head = rows[i];
144
+ candidates += 1;
145
+ let bytes = 0;
146
+ const sizes = [];
147
+ for (; rows[i]?.commitSeq === head.commitSeq; i++) {
148
+ const row = rows[i];
149
+ if (row.key === undefined)
150
+ continue;
151
+ bytes += row.bytes ?? 0;
152
+ sizes.push([row.key, row.bytes ?? 0]);
153
+ }
154
+ if (bytes > query.limitBytes) {
155
+ // The keys that read the most of it leave until the rest fits.
156
+ const reached = new Set();
157
+ let rest = bytes;
158
+ for (const [key, size] of sizes.sort((x, y) => y[1] - x[1])) {
159
+ if (rest <= query.limitBytes)
160
+ break;
161
+ reached.add(key);
162
+ rest -= size;
163
+ }
164
+ return {
165
+ commits: [],
166
+ throughSeq: query.throughSeq,
167
+ oversized: {
168
+ commitSeq: head.commitSeq,
169
+ units: query.units.flatMap((unit, index) => {
170
+ const key = unitKeys[index];
171
+ return key !== undefined &&
172
+ reached.has(key) &&
173
+ head.commitSeq > unit.afterSeq
174
+ ? [index]
175
+ : [];
176
+ }),
177
+ },
178
+ };
179
+ }
180
+ if (loaded + bytes > query.limitBytes) {
181
+ const cut = await load();
182
+ return { commits, throughSeq: cut ?? scanned };
110
183
  }
184
+ loaded += bytes;
185
+ pending.push({ commitSeq: head.commitSeq, meta: head.meta });
111
186
  }
187
+ const cut = await load();
188
+ if (cut !== undefined)
189
+ return { commits, throughSeq: cut };
112
190
  if (candidates < limit)
113
191
  break;
114
192
  }
package/dist/storage.d.ts CHANGED
@@ -366,16 +366,31 @@ export interface LiveWindowQuery {
366
366
  readonly throughSeq: number;
367
367
  /** The `limitCommits` budget: stop after the commit that reaches it. */
368
368
  readonly limitChanges: number;
369
+ /**
370
+ * `ServerLimits.maxLiveSectionBytes`: the stored bytes (payload, row id,
371
+ * scopes) of the changes a section loads. The section stops before the
372
+ * commit that would pass it; a commit that passes it alone is oversized.
373
+ */
374
+ readonly limitBytes: number;
369
375
  }
370
376
  /** The commits of one live section (§4.5). */
371
377
  export interface LiveWindow {
372
378
  /** Oldest first, each with its matching changes in change order. */
373
379
  readonly commits: readonly StoredCommit[];
374
380
  /**
375
- * `T`: `query.throughSeq`, or the last delivered `commitSeq` when the
376
- * budget cut the section.
381
+ * `T`: `query.throughSeq`, or the last scanned `commitSeq` when a budget
382
+ * cut the section.
377
383
  */
378
384
  readonly throughSeq: number;
385
+ /**
386
+ * A commit whose changes for the scan set pass `limitBytes` on their own
387
+ * (§4.6): `units` are the indexes into `query.units` it reaches. The
388
+ * section is unusable; the caller resets those units and reads again.
389
+ */
390
+ readonly oversized?: {
391
+ readonly commitSeq: number;
392
+ readonly units: readonly number[];
393
+ };
379
394
  }
380
395
  export interface RowScanQuery {
381
396
  readonly table: string;
@@ -806,9 +821,11 @@ export interface ServerStorage {
806
821
  * The §4.5 live section: every commit in `(min afterSeq, throughSeq]`
807
822
  * carrying a change that matches a unit of the scan set, oldest first,
808
823
  * each with its matching changes once, in change order, across tables.
809
- * Stops after the commit whose matching changes reach `limitChanges`
810
- * (never splitting a commit). Candidates come from the change-scope key
811
- * index, one index range per distinct unit key, never from a log scan.
824
+ * Stops after the commit whose matching changes reach `limitChanges`, or
825
+ * before the commit that would pass `limitBytes` (never splitting a
826
+ * commit). Candidates come from the change-scope key index, one index
827
+ * range per distinct unit key, never from a log scan, and only changes of
828
+ * a covered table carrying a unit key's value are read.
812
829
  */
813
830
  readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
814
831
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.69.0",
3
+ "version": "0.70.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.69.0"
71
+ "@syncular/core": "0.70.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/context.ts CHANGED
@@ -91,6 +91,13 @@ export interface ServerLimits {
91
91
  readonly maxSubscriptionsPerRequest: number;
92
92
  /** Cap on concurrent registered-query watches per watch session. */
93
93
  readonly maxWatchesPerSession: number;
94
+ /**
95
+ * §4.6 byte budget of one live section: the stored bytes (payload, row id,
96
+ * scopes) of the changes a pull loads from the log. A section stops before
97
+ * the commit that would pass it; a commit whose changes for the scan set
98
+ * pass it alone resets the units it reaches, which bootstrap instead.
99
+ */
100
+ readonly maxLiveSectionBytes: number;
94
101
  }
95
102
 
96
103
  export const DEFAULT_LIMITS: ServerLimits = {
@@ -103,6 +110,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
103
110
  maxSubscriptionsPerRequest: 500,
104
111
  // Every commit reruns a session's affected watches one after another.
105
112
  maxWatchesPerSession: 100,
113
+ // The live section is buffered whole before the first unit section.
114
+ maxLiveSectionBytes: 16 * 1024 * 1024,
106
115
  };
107
116
 
108
117
  /** Receives applied commits for realtime fanout (§8.2). */
package/src/handler.ts CHANGED
@@ -918,18 +918,30 @@ async function* streamResponse(
918
918
  // resets the live section), and the section is read again.
919
919
  let live: LiveWindow | undefined;
920
920
  for (;;) {
921
- const scan = units.flatMap(({ plan: unit }, index) => {
921
+ const scanned = units.flatMap(({ plan: unit }, index) => {
922
922
  const afterSeq = bound(states[index]!);
923
923
  return afterSeq === undefined
924
924
  ? []
925
- : [{ scopes: unit.scopes, afterSeq, tables: unit.pairs }];
925
+ : [{ index, scopes: unit.scopes, afterSeq, tables: unit.pairs }];
926
926
  });
927
- if (liveReset || scan.length === 0) break;
927
+ if (liveReset || scanned.length === 0) break;
928
928
  live = await ctx.storage.readLiveWindow(ctx.partition, {
929
- units: scan,
929
+ units: scanned,
930
930
  throughSeq: maxSeq,
931
931
  limitChanges: limits.limitCommits,
932
+ limitBytes: limitsOf(ctx).maxLiveSectionBytes,
932
933
  });
934
+ // §4.6: a commit past the byte budget on its own cannot be served
935
+ // from the log; the units it reaches reset and bootstrap.
936
+ for (const at of live.oversized?.units ?? []) {
937
+ const index = scanned[at]!.index;
938
+ if (states[index]!.kind === 'live') liveReset = true;
939
+ else states[index] = { kind: 'reset' };
940
+ }
941
+ if (live.oversized !== undefined) {
942
+ live = undefined;
943
+ continue;
944
+ }
933
945
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
934
946
  let expired = false;
935
947
  for (const [index, state] of states.entries()) {
@@ -215,6 +215,10 @@ export async function openLocalAuthority(
215
215
  segments: new MemorySegmentStore(),
216
216
  resolveScopes: () => allowed,
217
217
  clock,
218
+ // §12.2: the client's own process serves its own commits; the
219
+ // live byte budget (§4.6) bounds a shared server's heap and does
220
+ // not apply here, as in the native authority.
221
+ limits: { maxLiveSectionBytes: Number.MAX_SAFE_INTEGER },
218
222
  ...(options.crdtMergers !== undefined
219
223
  ? { crdtMergers: options.crdtMergers }
220
224
  : {}),
@@ -3314,52 +3314,77 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3314
3314
  partition: string,
3315
3315
  query: LiveWindowQuery,
3316
3316
  ): Promise<LiveWindow> {
3317
- // One statement per page: one LATERAL range of
3317
+ // Two statements per page. The candidate read takes one LATERAL range of
3318
3318
  // `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
3319
- // the index, merged in commit order and LIMITed again, then LEFT JOINed
3320
- // to the commit meta and every change of the commit. A candidate whose
3321
- // commit vanished comes back with NULL meta and still advances the scan.
3322
- return readLiveWindowPages(query, async (keys, limit) => {
3323
- const { rows } = await this.#exec.query<{
3324
- commit_seq: unknown;
3325
- actor_id: string | null;
3326
- created_at_ms: unknown;
3327
- tbl: string | null;
3328
- row_id: string;
3329
- op: unknown;
3330
- row_version: unknown;
3331
- scopes: unknown;
3332
- payload: unknown;
3333
- }>(
3334
- `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3335
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
3336
- FROM (SELECT DISTINCT s.commit_seq
3337
- FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
3338
- CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
3319
+ // the index, merged in commit order and LIMITed again, then LEFT JOINs
3320
+ // the commit meta (NULL when the commit vanished, which still advances
3321
+ // the scan) and, per key, the byte size of the changes the key reads;
3322
+ // `octet_length` of a TOASTed payload reads its pointer, never the value.
3323
+ // The change read then loads only those changes of the commits the byte
3324
+ // budget admits.
3325
+ return readLiveWindowPages(
3326
+ query,
3327
+ async (keys, limit) => {
3328
+ const { rows } = await this.#exec.query<{
3329
+ commit_seq: unknown;
3330
+ actor_id: string | null;
3331
+ created_at_ms: unknown;
3332
+ key: unknown;
3333
+ bytes: unknown;
3334
+ }>(
3335
+ `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
3336
+ c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
3337
+ FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
3339
3338
  WHERE partition=$1 AND var=k.var AND value=k.value
3340
3339
  AND commit_seq>k.after AND commit_seq<=$3
3341
3340
  ORDER BY commit_seq LIMIT $4) s
3342
- ORDER BY s.commit_seq LIMIT $4) c
3343
- LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
3344
- LEFT JOIN sync_changes ch ON ch.partition=$1 AND ch.commit_seq=c.commit_seq
3345
- ORDER BY c.commit_seq, ch.idx`,
3346
- [partition, JSON.stringify(keys), query.throughSeq, limit],
3347
- );
3348
- return rows.map((record) => ({
3349
- commitSeq: asNumber(record.commit_seq),
3350
- ...(record.actor_id === null
3351
- ? {}
3352
- : {
3353
- meta: {
3354
- actorId: record.actor_id,
3355
- createdAtMs: asNumber(record.created_at_ms),
3356
- },
3357
- }),
3358
- ...(record.tbl === null
3359
- ? {}
3360
- : { change: toStoredChange({ ...record, tbl: record.tbl }) }),
3361
- }));
3362
- });
3341
+ ORDER BY s.commit_seq LIMIT $4)
3342
+ SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3343
+ z.key AS key, z.bytes AS bytes
3344
+ FROM c
3345
+ LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
3346
+ LEFT JOIN LATERAL (SELECT k.i AS key,
3347
+ sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
3348
+ FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value
3349
+ WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
3350
+ GROUP BY k.i) z ON true
3351
+ ORDER BY c.commit_seq, z.key`,
3352
+ [partition, JSON.stringify(keys), query.throughSeq, limit],
3353
+ );
3354
+ return rows.map((record) => ({
3355
+ commitSeq: asNumber(record.commit_seq),
3356
+ ...(record.actor_id === null
3357
+ ? {}
3358
+ : {
3359
+ meta: {
3360
+ actorId: record.actor_id,
3361
+ createdAtMs: asNumber(record.created_at_ms),
3362
+ },
3363
+ }),
3364
+ ...(record.key === null
3365
+ ? {}
3366
+ : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
3367
+ }));
3368
+ },
3369
+ async (keys, after, through) => {
3370
+ const { rows } = await this.#exec.query<
3371
+ ChangeRecord & { commit_seq: unknown }
3372
+ >(
3373
+ `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
3374
+ SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
3375
+ ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
3376
+ FROM sync_changes ch
3377
+ WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
3378
+ AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value)
3379
+ ORDER BY ch.commit_seq, ch.idx`,
3380
+ [partition, JSON.stringify(keys), after, through],
3381
+ );
3382
+ return rows.map((record) => ({
3383
+ commitSeq: asNumber(record.commit_seq),
3384
+ change: toStoredChange(record),
3385
+ }));
3386
+ },
3387
+ );
3363
3388
  }
3364
3389
 
3365
3390
  async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
package/src/realtime.ts CHANGED
@@ -846,9 +846,32 @@ export class RealtimeSession {
846
846
  // the keys before, reconcile after (leaves on lost keys, snapshots on
847
847
  // gained keys).
848
848
  const previousKeys = this.scopeKeys;
849
+ const previous = this.registrations;
849
850
  this.registrations = registrations;
850
851
  this.scopeKeys = this.#hub.scopeKeysOf(registrations);
851
852
  this.reconcilePresence(previousKeys);
853
+ // §8.7: a host refresh that removes or narrows a registration wakes the
854
+ // session, so its client runs the round whose §3.3 revocation purge
855
+ // drops the rows it lost. A registration survives when the same unit
856
+ // and table come back with every previous scope value still allowed.
857
+ if (
858
+ failClosed &&
859
+ previous.some(
860
+ (old) =>
861
+ !registrations.some(
862
+ (next) =>
863
+ next.id === old.id &&
864
+ next.table === old.table &&
865
+ Object.entries(next.effective).every(
866
+ ([variable, values]) =>
867
+ old.effective[variable]?.every((value) =>
868
+ values.includes(value),
869
+ ) ?? false,
870
+ ),
871
+ ),
872
+ )
873
+ )
874
+ this.sendWake('catchup-required');
852
875
  }
853
876
 
854
877
  /** §8.7 protocol violation: drop the connection, fail loud. The
@@ -1199,53 +1199,73 @@ export abstract class SqliteFamilyStorage {
1199
1199
  partition: string,
1200
1200
  query: LiveWindowQuery,
1201
1201
  ): Promise<LiveWindow> {
1202
- // One statement per page: the unit keys bind as one JSON array, and the
1203
- // CROSS JOIN keeps them the outer loop, so each key seeks one range of
1204
- // `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
1205
- // missing). The candidates LEFT JOIN the commit meta and every change of
1206
- // the commit; a candidate whose commit vanished comes back with NULL
1207
- // meta and still advances the scan.
1208
- return readLiveWindowPages(query, async (keys, limit) =>
1209
- (
1210
- await this.guardedAll<{
1211
- commit_seq: number;
1212
- actor_id: string | null;
1213
- created_at_ms: number | null;
1214
- tbl: string | null;
1215
- row_id: string;
1216
- op: number;
1217
- row_version: number | null;
1218
- scopes: string;
1219
- payload: Uint8Array | null;
1220
- }>(
1221
- `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
1222
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
1223
- FROM (SELECT DISTINCT s.commit_seq AS commit_seq
1224
- FROM json_each(?2) AS k
1225
- CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
1226
- ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
1227
- AND s.value=json_extract(k.value, '$.value')
1228
- AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
1229
- ORDER BY s.commit_seq LIMIT ?4) AS c
1230
- LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
1231
- LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
1232
- ORDER BY c.commit_seq, ch.idx`,
1233
- [partition, JSON.stringify(keys), query.throughSeq, limit],
1234
- )
1235
- ).map((record) => ({
1236
- commitSeq: record.commit_seq,
1237
- ...(record.actor_id !== null && record.created_at_ms !== null
1238
- ? {
1239
- meta: {
1240
- actorId: record.actor_id,
1241
- createdAtMs: record.created_at_ms,
1242
- },
1243
- }
1244
- : {}),
1245
- ...(record.tbl !== null
1246
- ? { change: toStoredChange({ ...record, tbl: record.tbl }) }
1247
- : {}),
1248
- })),
1202
+ // Two statements per page. The candidate read binds the unit keys as one
1203
+ // JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
1204
+ // one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
1205
+ // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
1206
+ // when the commit vanished, which still advances the scan) and, per key,
1207
+ // the byte size of the changes the key reads; `length()` of a blob reads
1208
+ // the record header, never the payload. The change read then loads only
1209
+ // those changes of the commits the byte budget admits.
1210
+ return readLiveWindowPages(
1211
+ query,
1212
+ async (keys, limit) =>
1213
+ (
1214
+ await this.guardedAll<{
1215
+ commit_seq: number;
1216
+ actor_id: string | null;
1217
+ created_at_ms: number | null;
1218
+ key: number | null;
1219
+ bytes: number | null;
1220
+ }>(
1221
+ `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
1222
+ c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
1223
+ FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
1224
+ ON s.partition=?1 AND s.var=k.var AND s.value=k.value
1225
+ AND s.commit_seq>k.after AND s.commit_seq<=?3
1226
+ ORDER BY s.commit_seq LIMIT ?4)
1227
+ SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
1228
+ z.i AS key, z.bytes AS bytes
1229
+ FROM c
1230
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
1231
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
1232
+ sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
1233
+ FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
1234
+ CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
1235
+ GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
1236
+ ORDER BY c.commit_seq, z.i`,
1237
+ [partition, JSON.stringify(keys), query.throughSeq, limit],
1238
+ )
1239
+ ).map((record) => ({
1240
+ commitSeq: record.commit_seq,
1241
+ ...(record.actor_id !== null && record.created_at_ms !== null
1242
+ ? {
1243
+ meta: {
1244
+ actorId: record.actor_id,
1245
+ createdAtMs: record.created_at_ms,
1246
+ },
1247
+ }
1248
+ : {}),
1249
+ ...(record.key !== null
1250
+ ? { key: record.key, bytes: record.bytes ?? 0 }
1251
+ : {}),
1252
+ })),
1253
+ async (keys, after, through) =>
1254
+ (
1255
+ await this.guardedAll<SqliteChangeRecord & { commit_seq: number }>(
1256
+ `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
1257
+ SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
1258
+ ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
1259
+ FROM sync_changes AS ch
1260
+ WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
1261
+ AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
1262
+ ORDER BY ch.commit_seq, ch.idx`,
1263
+ [partition, JSON.stringify(keys), after, through],
1264
+ )
1265
+ ).map((record) => ({
1266
+ commitSeq: record.commit_seq,
1267
+ change: toStoredChange(record),
1268
+ })),
1249
1269
  );
1250
1270
  }
1251
1271
 
@@ -79,43 +79,71 @@ export async function transactionQuery(
79
79
  return tx.queryAuthoritative(query);
80
80
  }
81
81
 
82
- /** One index range of the live scan: a unit key and its exclusive lower bound. */
82
+ /**
83
+ * One index range of the live scan: a unit key, its exclusive lower bound,
84
+ * and the tables its units cover. `i` is the key's position in the list.
85
+ */
83
86
  export interface LiveKey {
87
+ readonly i: number;
84
88
  readonly var: string;
85
89
  readonly value: string;
86
90
  readonly after: number;
91
+ readonly tables: readonly string[];
87
92
  }
88
93
 
89
94
  /**
90
- * One row of a live-window page: a candidate commit joined to its metadata
91
- * (absent when the commit vanished) and one of its changes (absent when it
92
- * has none), in (commitSeq, change index) order.
95
+ * One row of a live-window candidate page: a candidate commit joined to its
96
+ * metadata (absent when the commit vanished) and, per key, the stored bytes
97
+ * (payload, row id, scopes) of its changes that key reads: a covered table
98
+ * and the key's value in the stored scopes. A commit with no such change
99
+ * comes back once without `key`. Rows are in (commitSeq, key) order.
93
100
  */
94
- export interface LiveWindowRow {
101
+ export interface LiveCandidateRow {
95
102
  readonly commitSeq: number;
96
103
  readonly meta?: { readonly actorId: string; readonly createdAtMs: number };
97
- readonly change?: StoredChange;
104
+ readonly key?: number;
105
+ readonly bytes?: number;
98
106
  }
99
107
 
100
108
  /**
101
- * The §4.5 live scan over a backend's page read. Each distinct unit key
102
- * (the first variable, keys sorted) becomes one index range starting at the
103
- * lowest bound of the units that share it; `readPage` returns up to `limit`
104
- * candidate commits above every range's bound. A change is kept when a unit
105
- * covers its table, the pair's effective scopes match it, and its commit
106
- * lies above that unit's own bound. Every candidate advances the scan, so a
107
- * page of non-matching candidates never repeats.
109
+ * The §4.5 live scan over a backend's two reads. `readCandidates` returns
110
+ * up to `limit` candidate commits above every key range's bound with their
111
+ * per-key byte sizes, and no payload. `readChanges` loads the changes the
112
+ * keys read in a commit range `(after, through]`, in (commitSeq, change
113
+ * index) order. Each distinct unit key (the first variable, keys sorted)
114
+ * becomes one index range starting at the lowest bound of the units that
115
+ * share it. A change is kept when a unit covers its table, the pair's
116
+ * effective scopes match it, and its commit lies above that unit's own
117
+ * bound. Every candidate advances the scan, so a page of non-matching
118
+ * candidates never repeats.
119
+ *
120
+ * The sizes bound the loaded bytes by `limitBytes` before any payload is
121
+ * read: the section stops before the commit that would pass it, and a
122
+ * commit that passes it alone ends the read as `oversized`, naming the
123
+ * units of the keys that read the most of it, largest first, until the rest
124
+ * fits (§4.6).
108
125
  */
109
126
  export async function readLiveWindowPages(
110
127
  query: LiveWindowQuery,
111
- readPage: (
128
+ readCandidates: (
112
129
  keys: readonly LiveKey[],
113
130
  limit: number,
114
- ) => Promise<readonly LiveWindowRow[]>,
131
+ ) => Promise<readonly LiveCandidateRow[]>,
132
+ readChanges: (
133
+ keys: readonly LiveKey[],
134
+ after: number,
135
+ through: number,
136
+ ) => Promise<
137
+ readonly { readonly commitSeq: number; readonly change: StoredChange }[]
138
+ >,
115
139
  ): Promise<LiveWindow> {
116
- const bounds = new Map<string, LiveKey>();
140
+ const bounds = new Map<string, { key: LiveKey; tables: Set<string> }>();
141
+ const unitKeys: (number | undefined)[] = [];
117
142
  for (const unit of query.units) {
118
- if (unit.tables.size === 0) continue;
143
+ if (unit.tables.size === 0) {
144
+ unitKeys.push(undefined);
145
+ continue;
146
+ }
119
147
  const variable = Object.keys(unit.scopes).sort()[0];
120
148
  const value = variable === undefined ? undefined : unit.scopes[variable];
121
149
  if (variable === undefined || value === undefined) {
@@ -123,39 +151,50 @@ export async function readLiveWindowPages(
123
151
  }
124
152
  const id = JSON.stringify([variable, value]);
125
153
  const prior = bounds.get(id);
126
- if (prior === undefined || unit.afterSeq < prior.after) {
127
- bounds.set(id, { var: variable, value, after: unit.afterSeq });
128
- }
154
+ const tables = new Set([...(prior?.tables ?? []), ...unit.tables.keys()]);
155
+ const i = prior?.key.i ?? bounds.size;
156
+ bounds.set(id, {
157
+ key: {
158
+ i,
159
+ var: variable,
160
+ value,
161
+ after: Math.min(prior?.key.after ?? unit.afterSeq, unit.afterSeq),
162
+ tables: [...tables],
163
+ },
164
+ tables,
165
+ });
166
+ unitKeys.push(i);
129
167
  }
168
+ const keys = [...bounds.values()].map(({ key }) => key);
130
169
  const commits: StoredCommit[] = [];
131
170
  const limit = Math.max(64, query.limitChanges);
132
171
  let delivered = 0;
172
+ let loaded = 0;
133
173
  let scanned = -1;
134
- while (bounds.size > 0) {
135
- const rows = await readPage(
136
- [...bounds.values()].map((key) => ({
137
- ...key,
138
- after: Math.max(key.after, scanned),
139
- })),
140
- limit,
141
- );
142
- let candidates = 0;
174
+ // Candidates sized but not loaded yet: (scanned, pending.at(-1)].
175
+ let pending: { commitSeq: number; meta?: LiveCandidateRow['meta'] }[] = [];
176
+ const scanKeys = () =>
177
+ keys.map((key) => ({ ...key, after: Math.max(key.after, scanned) }));
178
+ // Loads the pending candidates; returns T when `limitChanges` cut there.
179
+ const load = async (): Promise<number | undefined> => {
180
+ const last = pending.at(-1);
181
+ if (last === undefined) return undefined;
182
+ const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
183
+ const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
184
+ pending = [];
185
+ scanned = last.commitSeq;
143
186
  for (let i = 0; i < rows.length;) {
144
- const head = rows[i];
145
- if (head === undefined) break;
146
- candidates += 1;
147
- scanned = head.commitSeq;
187
+ const seq = rows[i]!.commitSeq;
148
188
  const changes: StoredChange[] = [];
149
- for (; rows[i]?.commitSeq === head.commitSeq; i++) {
150
- const change = rows[i]?.change;
189
+ for (; rows[i]?.commitSeq === seq; i++) {
190
+ const change = rows[i]!.change;
151
191
  if (
152
- change !== undefined &&
153
192
  query.units.some((unit) => {
154
193
  const effective: ScopeMap | undefined = unit.tables.get(
155
194
  change.table,
156
195
  );
157
196
  return (
158
- head.commitSeq > unit.afterSeq &&
197
+ seq > unit.afterSeq &&
159
198
  effective !== undefined &&
160
199
  matchesEffective(change.scopes, effective)
161
200
  );
@@ -164,13 +203,62 @@ export async function readLiveWindowPages(
164
203
  changes.push(change);
165
204
  }
166
205
  }
167
- if (head.meta === undefined || changes.length === 0) continue;
168
- commits.push({ commitSeq: head.commitSeq, ...head.meta, changes });
206
+ const meta = metas.get(seq);
207
+ if (meta === undefined || changes.length === 0) continue;
208
+ commits.push({ commitSeq: seq, ...meta, changes });
169
209
  delivered += changes.length;
170
- if (delivered >= query.limitChanges) {
171
- return { commits, throughSeq: head.commitSeq };
210
+ if (delivered >= query.limitChanges) return seq;
211
+ }
212
+ return undefined;
213
+ };
214
+ while (keys.length > 0) {
215
+ const rows = await readCandidates(scanKeys(), limit);
216
+ let candidates = 0;
217
+ for (let i = 0; i < rows.length;) {
218
+ const head = rows[i]!;
219
+ candidates += 1;
220
+ let bytes = 0;
221
+ const sizes: [key: number, bytes: number][] = [];
222
+ for (; rows[i]?.commitSeq === head.commitSeq; i++) {
223
+ const row = rows[i]!;
224
+ if (row.key === undefined) continue;
225
+ bytes += row.bytes ?? 0;
226
+ sizes.push([row.key, row.bytes ?? 0]);
227
+ }
228
+ if (bytes > query.limitBytes) {
229
+ // The keys that read the most of it leave until the rest fits.
230
+ const reached = new Set<number>();
231
+ let rest = bytes;
232
+ for (const [key, size] of sizes.sort((x, y) => y[1] - x[1])) {
233
+ if (rest <= query.limitBytes) break;
234
+ reached.add(key);
235
+ rest -= size;
236
+ }
237
+ return {
238
+ commits: [],
239
+ throughSeq: query.throughSeq,
240
+ oversized: {
241
+ commitSeq: head.commitSeq,
242
+ units: query.units.flatMap((unit, index) => {
243
+ const key = unitKeys[index];
244
+ return key !== undefined &&
245
+ reached.has(key) &&
246
+ head.commitSeq > unit.afterSeq
247
+ ? [index]
248
+ : [];
249
+ }),
250
+ },
251
+ };
252
+ }
253
+ if (loaded + bytes > query.limitBytes) {
254
+ const cut = await load();
255
+ return { commits, throughSeq: cut ?? scanned };
172
256
  }
257
+ loaded += bytes;
258
+ pending.push({ commitSeq: head.commitSeq, meta: head.meta });
173
259
  }
260
+ const cut = await load();
261
+ if (cut !== undefined) return { commits, throughSeq: cut };
174
262
  if (candidates < limit) break;
175
263
  }
176
264
  return { commits, throughSeq: query.throughSeq };
package/src/storage.ts CHANGED
@@ -496,6 +496,12 @@ export interface LiveWindowQuery {
496
496
  readonly throughSeq: number;
497
497
  /** The `limitCommits` budget: stop after the commit that reaches it. */
498
498
  readonly limitChanges: number;
499
+ /**
500
+ * `ServerLimits.maxLiveSectionBytes`: the stored bytes (payload, row id,
501
+ * scopes) of the changes a section loads. The section stops before the
502
+ * commit that would pass it; a commit that passes it alone is oversized.
503
+ */
504
+ readonly limitBytes: number;
499
505
  }
500
506
 
501
507
  /** The commits of one live section (§4.5). */
@@ -503,10 +509,19 @@ export interface LiveWindow {
503
509
  /** Oldest first, each with its matching changes in change order. */
504
510
  readonly commits: readonly StoredCommit[];
505
511
  /**
506
- * `T`: `query.throughSeq`, or the last delivered `commitSeq` when the
507
- * budget cut the section.
512
+ * `T`: `query.throughSeq`, or the last scanned `commitSeq` when a budget
513
+ * cut the section.
508
514
  */
509
515
  readonly throughSeq: number;
516
+ /**
517
+ * A commit whose changes for the scan set pass `limitBytes` on their own
518
+ * (§4.6): `units` are the indexes into `query.units` it reaches. The
519
+ * section is unusable; the caller resets those units and reads again.
520
+ */
521
+ readonly oversized?: {
522
+ readonly commitSeq: number;
523
+ readonly units: readonly number[];
524
+ };
510
525
  }
511
526
 
512
527
  export interface RowScanQuery {
@@ -1111,9 +1126,11 @@ export interface ServerStorage {
1111
1126
  * The §4.5 live section: every commit in `(min afterSeq, throughSeq]`
1112
1127
  * carrying a change that matches a unit of the scan set, oldest first,
1113
1128
  * each with its matching changes once, in change order, across tables.
1114
- * Stops after the commit whose matching changes reach `limitChanges`
1115
- * (never splitting a commit). Candidates come from the change-scope key
1116
- * index, one index range per distinct unit key, never from a log scan.
1129
+ * Stops after the commit whose matching changes reach `limitChanges`, or
1130
+ * before the commit that would pass `limitBytes` (never splitting a
1131
+ * commit). Candidates come from the change-scope key index, one index
1132
+ * range per distinct unit key, never from a log scan, and only changes of
1133
+ * a covered table carrying a unit key's value are read.
1117
1134
  */
1118
1135
  readLiveWindow(
1119
1136
  partition: string,