@syncular/server 0.98.0 → 0.99.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2254,29 +2254,56 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2254
2254
  readLiveWindow(partition, query) {
2255
2255
  // Two statements per page. The candidate read takes one LATERAL range of
2256
2256
  // `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
2257
- // the index, merged in commit order and LIMITed again, then LEFT JOINs
2258
- // the commit meta (NULL when the commit vanished, which still advances
2259
- // the scan) and, per key, the byte size of the changes the key reads;
2260
- // `octet_length` of a TOASTed payload reads its pointer, never the value.
2261
- // The change read then seeks only the commits the byte budget admits.
2262
- return readLiveWindowPages(query, async (keys, limit) => {
2263
- const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
2264
- c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
2265
- FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
2266
- WHERE partition=$1 AND var=k.var AND value=k.value
2267
- AND commit_seq>k.after AND commit_seq<=$3
2268
- ORDER BY commit_seq LIMIT $4) s
2269
- ORDER BY s.commit_seq LIMIT $4)
2270
- SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2271
- z.key AS key, z.bytes AS bytes, z.changes AS changes
2272
- FROM c
2273
- LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2274
- LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
2275
- sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
2276
- FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
2277
- WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
2278
- GROUP BY k.i) z ON true
2279
- ORDER BY c.commit_seq, z.key`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
2257
+ // the index, merged in commit order and LIMITed again into one array.
2258
+ // `walk` then sizes the candidates one at a time and stops after the one
2259
+ // that reaches the budget: per key, the count and byte size of the
2260
+ // commit's changes the key reads. Each change's scopes expand once and
2261
+ // hash-join the keys on (var, value); `octet_length` of a TOASTed payload
2262
+ // reads its pointer, never the value. Every walked commit LEFT JOINs its
2263
+ // meta (NULL when the commit vanished, which still advances the scan).
2264
+ // The change read then seeks only the commits the byte budget admits,
2265
+ // matching each change to the keys by the same join.
2266
+ return readLiveWindowPages(query, async (keys, limit, budget) => {
2267
+ const { rows } = await this.#exec.query(`WITH RECURSIVE k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
2268
+ c AS MATERIALIZED (SELECT coalesce(array_agg(d.commit_seq ORDER BY d.commit_seq), '{}') AS seqs
2269
+ FROM (SELECT DISTINCT s.commit_seq
2270
+ FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
2271
+ WHERE partition=$1 AND var=k.var AND value=k.value
2272
+ AND commit_seq>k.after AND commit_seq<=$3
2273
+ ORDER BY commit_seq LIMIT $4) s
2274
+ ORDER BY s.commit_seq LIMIT $4) d),
2275
+ km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
2276
+ FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(i, after, tables)) AS vals FROM k GROUP BY var) x),
2277
+ walk(n, commit_seq, changes, bytes, sizes) AS (
2278
+ SELECT 0, NULL::bigint, 0::numeric, 0::numeric, NULL::jsonb
2279
+ UNION ALL
2280
+ SELECT w.n+1, c.seqs[w.n+1], w.changes+z.changes, w.bytes+z.bytes, z.sizes
2281
+ FROM walk w CROSS JOIN c CROSS JOIN km CROSS JOIN LATERAL (
2282
+ SELECT coalesce(sum(g.changes), 0) AS changes, coalesce(sum(g.bytes), 0) AS bytes,
2283
+ jsonb_agg(jsonb_build_array(g.key, g.bytes, g.changes)) AS sizes
2284
+ FROM (SELECT (x.kv->>0)::int AS key, count(*) AS changes,
2285
+ sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
2286
+ FROM sync_changes ch
2287
+ CROSS JOIN LATERAL jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
2288
+ CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
2289
+ WHERE ch.partition=$1 AND ch.commit_seq=c.seqs[w.n+1]
2290
+ AND ch.commit_seq>(x.kv->>1)::bigint AND x.kv->2 @> to_jsonb(ch.tbl)
2291
+ GROUP BY 1) g) z
2292
+ WHERE w.n<cardinality(c.seqs) AND w.changes<$5 AND w.bytes<=$6)
2293
+ SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2294
+ (z->>0)::int AS key, (z->>1)::bigint AS bytes, (z->>2)::bigint AS changes
2295
+ FROM walk w
2296
+ LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=w.commit_seq
2297
+ LEFT JOIN LATERAL jsonb_array_elements(w.sizes) z ON true
2298
+ WHERE w.n>0
2299
+ ORDER BY w.commit_seq, key`, [
2300
+ partition,
2301
+ JSON.stringify(keys),
2302
+ query.throughSeq,
2303
+ limit,
2304
+ budget.changes,
2305
+ budget.bytes,
2306
+ ]);
2280
2307
  return rows.map((record) => ({
2281
2308
  commitSeq: asNumber(record.commit_seq),
2282
2309
  ...(record.actor_id === null
@@ -2296,12 +2323,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2296
2323
  }),
2297
2324
  }));
2298
2325
  }, async (keys, commitSeqs) => {
2299
- const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
2326
+ const { rows } = await this.#exec.query(`WITH km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
2327
+ FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(after, tables)) AS vals
2328
+ FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint, tables jsonb) GROUP BY var) x)
2300
2329
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
2301
2330
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
2302
- FROM sync_changes ch
2331
+ FROM sync_changes ch CROSS JOIN km
2303
2332
  WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
2304
- AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
2333
+ AND EXISTS (SELECT 1 FROM jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
2334
+ CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
2335
+ WHERE ch.commit_seq>(x.kv->>0)::bigint AND x.kv->1 @> to_jsonb(ch.tbl))
2305
2336
  ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)]);
2306
2337
  return rows.map((record) => ({
2307
2338
  commitSeq: asNumber(record.commit_seq),
@@ -646,28 +646,52 @@ export class SqliteFamilyStorage {
646
646
  // Two statements per page. The candidate read binds the unit keys as one
647
647
  // JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
648
648
  // one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
649
- // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
650
- // when the commit vanished, which still advances the scan) and, per key,
651
- // the byte size of the changes the key reads; `length()` of a blob reads
652
- // the record header, never the payload. The change read then seeks only
653
- // the commits the byte budget admits, through the primary key, with the
654
- // key list materialized once per statement.
655
- return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
656
- c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
649
+ // the index is missing). `walk` then sizes the candidates one at a time,
650
+ // in commit order, and stops after the one that reaches the budget: per
651
+ // key, the count and byte size of the commit's changes the key reads.
652
+ // Each change's scopes expand once and join the keys on (var, value)
653
+ // through an automatic index; `length()` of a blob reads the record
654
+ // header, never the payload. Every walked commit LEFT JOINs its meta
655
+ // (NULL when the commit vanished, which still advances the scan). The
656
+ // change read then seeks only the commits the byte budget admits,
657
+ // through the primary key, matching each change to the keys by the same
658
+ // join.
659
+ return readLiveWindowPages(query, async (keys, limit, budget) => (await this.guardedAll(`WITH RECURSIVE k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
660
+ c AS MATERIALIZED (SELECT row_number() OVER (ORDER BY commit_seq) AS n, commit_seq FROM (SELECT DISTINCT s.commit_seq AS commit_seq
657
661
  FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
658
662
  ON s.partition=?1 AND s.var=k.var AND s.value=k.value
659
663
  AND s.commit_seq>k.after AND s.commit_seq<=?3
660
- ORDER BY s.commit_seq LIMIT ?4)
661
- SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
662
- z.i AS key, z.bytes AS bytes, z.changes AS changes
663
- FROM c
664
- LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
665
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
666
- sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
667
- FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
668
- CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
669
- GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
670
- ORDER BY c.commit_seq, z.i`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
664
+ ORDER BY s.commit_seq LIMIT ?4)),
665
+ walk(n, commit_seq, sizes, changes, bytes) AS (
666
+ SELECT 0, NULL, '[]', 0, 0
667
+ UNION ALL
668
+ SELECT c.n, c.commit_seq,
669
+ (SELECT json_group_array(json_array(g.i, g.bytes, g.changes)) FROM (SELECT k.i AS i, count(*) AS changes,
670
+ sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
671
+ FROM sync_changes AS ch CROSS JOIN json_each(ch.scopes) AS e
672
+ CROSS JOIN k ON k.var=e.key AND k.value=e.value
673
+ WHERE ch.partition=?1 AND ch.commit_seq=c.commit_seq
674
+ AND ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables))
675
+ GROUP BY k.i) AS g),
676
+ w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)),
677
+ w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes))
678
+ FROM walk AS w JOIN c ON c.n=w.n+1
679
+ WHERE w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)) < ?5
680
+ AND w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes)) <= ?6)
681
+ SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
682
+ z.value ->> 0 AS key, z.value ->> 1 AS bytes, z.value ->> 2 AS changes
683
+ FROM walk AS w
684
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=w.commit_seq
685
+ LEFT JOIN json_each(w.sizes) AS z
686
+ WHERE w.n>0
687
+ ORDER BY w.commit_seq, key`, [
688
+ partition,
689
+ JSON.stringify(keys),
690
+ query.throughSeq,
691
+ limit,
692
+ budget.changes,
693
+ budget.bytes,
694
+ ])).map((record) => ({
671
695
  commitSeq: record.commit_seq,
672
696
  ...(record.actor_id !== null && record.created_at_ms !== null
673
697
  ? {
@@ -688,7 +712,8 @@ export class SqliteFamilyStorage {
688
712
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
689
713
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
690
714
  FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
691
- WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
715
+ WHERE EXISTS (SELECT 1 FROM json_each(ch.scopes) AS e CROSS JOIN k ON k.var=e.key AND k.value=e.value
716
+ WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)))
692
717
  ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)])).map((record) => ({
693
718
  commitSeq: record.commit_seq,
694
719
  change: toStoredChange(record),
@@ -74,12 +74,16 @@ export interface LiveCandidateRow {
74
74
  readonly changes?: number;
75
75
  }
76
76
  /**
77
- * The §4.5 live scan over a backend's two reads. `readCandidates` returns
78
- * up to `limit` candidate commits above every key range's bound with their
79
- * per-key byte sizes, and no payload. `readChanges` loads the changes the
80
- * keys read in the given candidate commits, in (commitSeq, change index)
81
- * order; it never visits a commit outside that list, so the cost of a page
82
- * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
77
+ * The §4.5 live scan over a backend's two reads. `readCandidates` takes up
78
+ * to `limit` candidate commits above every key range's bound and sizes them
79
+ * in commit order, without payload, until their change count reaches
80
+ * `budget.changes` or their byte size passes `budget.bytes`. It returns the
81
+ * sized candidates with their per-key sizes, the one that reached the budget
82
+ * last, so a page sizes the commits it delivers and one more. `readChanges`
83
+ * loads the changes the keys read in the given candidate commits, in
84
+ * (commitSeq, change index) order; it never visits a commit outside that
85
+ * list, so the cost of a page follows its candidates and not the length of
86
+ * the change log below them. Each distinct unit key (the first variable, keys sorted)
83
87
  * becomes one index range starting at the lowest bound of the units that
84
88
  * share it. A change is kept when a unit covers its table, the pair's
85
89
  * effective scopes match it, and its commit lies above that unit's own
@@ -97,7 +101,10 @@ export interface LiveCandidateRow {
97
101
  * units of the keys that read the most of it, largest first, until the rest
98
102
  * fits (§4.6).
99
103
  */
100
- export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
104
+ export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number, budget: {
105
+ readonly changes: number;
106
+ readonly bytes: number;
107
+ }) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
101
108
  readonly commitSeq: number;
102
109
  readonly change: StoredChange;
103
110
  }[]>): Promise<LiveWindow>;
@@ -101,12 +101,16 @@ export async function transactionQuery(tx, query) {
101
101
  return tx.queryAuthoritative(query);
102
102
  }
103
103
  /**
104
- * The §4.5 live scan over a backend's two reads. `readCandidates` returns
105
- * up to `limit` candidate commits above every key range's bound with their
106
- * per-key byte sizes, and no payload. `readChanges` loads the changes the
107
- * keys read in the given candidate commits, in (commitSeq, change index)
108
- * order; it never visits a commit outside that list, so the cost of a page
109
- * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
104
+ * The §4.5 live scan over a backend's two reads. `readCandidates` takes up
105
+ * to `limit` candidate commits above every key range's bound and sizes them
106
+ * in commit order, without payload, until their change count reaches
107
+ * `budget.changes` or their byte size passes `budget.bytes`. It returns the
108
+ * sized candidates with their per-key sizes, the one that reached the budget
109
+ * last, so a page sizes the commits it delivers and one more. `readChanges`
110
+ * loads the changes the keys read in the given candidate commits, in
111
+ * (commitSeq, change index) order; it never visits a commit outside that
112
+ * list, so the cost of a page follows its candidates and not the length of
113
+ * the change log below them. Each distinct unit key (the first variable, keys sorted)
110
114
  * becomes one index range starting at the lowest bound of the units that
111
115
  * share it. A change is kept when a unit covers its table, the pair's
112
116
  * effective scopes match it, and its commit lies above that unit's own
@@ -199,8 +203,13 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
199
203
  return undefined;
200
204
  };
201
205
  while (keys.length > 0) {
202
- const rows = await readCandidates(scanKeys(), limit);
206
+ const rows = await readCandidates(scanKeys(), limit, {
207
+ changes: query.limitChanges - delivered,
208
+ bytes: query.limitBytes - loaded,
209
+ });
203
210
  let candidates = 0;
211
+ // The change budget ended the read, possibly before `limit` candidates.
212
+ let reached = false;
204
213
  for (let i = 0; i < rows.length;) {
205
214
  const head = rows[i];
206
215
  candidates += 1;
@@ -249,6 +258,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
249
258
  bound += changes;
250
259
  pending.push({ commitSeq: head.commitSeq, meta: head.meta });
251
260
  if (delivered + bound >= query.limitChanges) {
261
+ reached = true;
252
262
  const cut = await load();
253
263
  if (cut !== undefined)
254
264
  return { commits, throughSeq: cut };
@@ -257,7 +267,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
257
267
  const cut = await load();
258
268
  if (cut !== undefined)
259
269
  return { commits, throughSeq: cut };
260
- if (candidates < limit)
270
+ if (candidates < limit && !reached)
261
271
  break;
262
272
  }
263
273
  return { commits, throughSeq: query.throughSeq };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.98.0",
3
+ "version": "0.99.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.98.0"
71
+ "@syncular/core": "0.99.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
@@ -3328,14 +3328,18 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3328
3328
  ): Promise<LiveWindow> {
3329
3329
  // Two statements per page. The candidate read takes one LATERAL range of
3330
3330
  // `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
3331
- // the index, merged in commit order and LIMITed again, then LEFT JOINs
3332
- // the commit meta (NULL when the commit vanished, which still advances
3333
- // the scan) and, per key, the byte size of the changes the key reads;
3334
- // `octet_length` of a TOASTed payload reads its pointer, never the value.
3335
- // The change read then seeks only the commits the byte budget admits.
3331
+ // the index, merged in commit order and LIMITed again into one array.
3332
+ // `walk` then sizes the candidates one at a time and stops after the one
3333
+ // that reaches the budget: per key, the count and byte size of the
3334
+ // commit's changes the key reads. Each change's scopes expand once and
3335
+ // hash-join the keys on (var, value); `octet_length` of a TOASTed payload
3336
+ // reads its pointer, never the value. Every walked commit LEFT JOINs its
3337
+ // meta (NULL when the commit vanished, which still advances the scan).
3338
+ // The change read then seeks only the commits the byte budget admits,
3339
+ // matching each change to the keys by the same join.
3336
3340
  return readLiveWindowPages(
3337
3341
  query,
3338
- async (keys, limit) => {
3342
+ async (keys, limit, budget) => {
3339
3343
  const { rows } = await this.#exec.query<{
3340
3344
  commit_seq: unknown;
3341
3345
  actor_id: string | null;
@@ -3344,24 +3348,47 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3344
3348
  bytes: unknown;
3345
3349
  changes: unknown;
3346
3350
  }>(
3347
- `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
3348
- c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
3349
- FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
3350
- WHERE partition=$1 AND var=k.var AND value=k.value
3351
- AND commit_seq>k.after AND commit_seq<=$3
3352
- ORDER BY commit_seq LIMIT $4) s
3353
- ORDER BY s.commit_seq LIMIT $4)
3354
- SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3355
- z.key AS key, z.bytes AS bytes, z.changes AS changes
3356
- FROM c
3357
- LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
3358
- LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
3359
- sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
3360
- FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
3361
- WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
3362
- GROUP BY k.i) z ON true
3363
- ORDER BY c.commit_seq, z.key`,
3364
- [partition, JSON.stringify(keys), query.throughSeq, limit],
3351
+ `WITH RECURSIVE k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
3352
+ c AS MATERIALIZED (SELECT coalesce(array_agg(d.commit_seq ORDER BY d.commit_seq), '{}') AS seqs
3353
+ FROM (SELECT DISTINCT s.commit_seq
3354
+ FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
3355
+ WHERE partition=$1 AND var=k.var AND value=k.value
3356
+ AND commit_seq>k.after AND commit_seq<=$3
3357
+ ORDER BY commit_seq LIMIT $4) s
3358
+ ORDER BY s.commit_seq LIMIT $4) d),
3359
+ km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
3360
+ FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(i, after, tables)) AS vals FROM k GROUP BY var) x),
3361
+ walk(n, commit_seq, changes, bytes, sizes) AS (
3362
+ SELECT 0, NULL::bigint, 0::numeric, 0::numeric, NULL::jsonb
3363
+ UNION ALL
3364
+ SELECT w.n+1, c.seqs[w.n+1], w.changes+z.changes, w.bytes+z.bytes, z.sizes
3365
+ FROM walk w CROSS JOIN c CROSS JOIN km CROSS JOIN LATERAL (
3366
+ SELECT coalesce(sum(g.changes), 0) AS changes, coalesce(sum(g.bytes), 0) AS bytes,
3367
+ jsonb_agg(jsonb_build_array(g.key, g.bytes, g.changes)) AS sizes
3368
+ FROM (SELECT (x.kv->>0)::int AS key, count(*) AS changes,
3369
+ sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
3370
+ FROM sync_changes ch
3371
+ CROSS JOIN LATERAL jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
3372
+ CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
3373
+ WHERE ch.partition=$1 AND ch.commit_seq=c.seqs[w.n+1]
3374
+ AND ch.commit_seq>(x.kv->>1)::bigint AND x.kv->2 @> to_jsonb(ch.tbl)
3375
+ GROUP BY 1) g) z
3376
+ WHERE w.n<cardinality(c.seqs) AND w.changes<$5 AND w.bytes<=$6)
3377
+ SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3378
+ (z->>0)::int AS key, (z->>1)::bigint AS bytes, (z->>2)::bigint AS changes
3379
+ FROM walk w
3380
+ LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=w.commit_seq
3381
+ LEFT JOIN LATERAL jsonb_array_elements(w.sizes) z ON true
3382
+ WHERE w.n>0
3383
+ ORDER BY w.commit_seq, key`,
3384
+ [
3385
+ partition,
3386
+ JSON.stringify(keys),
3387
+ query.throughSeq,
3388
+ limit,
3389
+ budget.changes,
3390
+ budget.bytes,
3391
+ ],
3365
3392
  );
3366
3393
  return rows.map((record) => ({
3367
3394
  commitSeq: asNumber(record.commit_seq),
@@ -3386,12 +3413,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3386
3413
  const { rows } = await this.#exec.query<
3387
3414
  StoredChangeRecord & { commit_seq: unknown }
3388
3415
  >(
3389
- `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
3416
+ `WITH km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
3417
+ FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(after, tables)) AS vals
3418
+ FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint, tables jsonb) GROUP BY var) x)
3390
3419
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
3391
3420
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
3392
- FROM sync_changes ch
3421
+ FROM sync_changes ch CROSS JOIN km
3393
3422
  WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
3394
- AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
3423
+ AND EXISTS (SELECT 1 FROM jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
3424
+ CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
3425
+ WHERE ch.commit_seq>(x.kv->>0)::bigint AND x.kv->1 @> to_jsonb(ch.tbl))
3395
3426
  ORDER BY ch.commit_seq, ch.idx`,
3396
3427
  [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
3397
3428
  );
@@ -1023,15 +1023,19 @@ export abstract class SqliteFamilyStorage {
1023
1023
  // Two statements per page. The candidate read binds the unit keys as one
1024
1024
  // JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
1025
1025
  // one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
1026
- // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
1027
- // when the commit vanished, which still advances the scan) and, per key,
1028
- // the byte size of the changes the key reads; `length()` of a blob reads
1029
- // the record header, never the payload. The change read then seeks only
1030
- // the commits the byte budget admits, through the primary key, with the
1031
- // key list materialized once per statement.
1026
+ // the index is missing). `walk` then sizes the candidates one at a time,
1027
+ // in commit order, and stops after the one that reaches the budget: per
1028
+ // key, the count and byte size of the commit's changes the key reads.
1029
+ // Each change's scopes expand once and join the keys on (var, value)
1030
+ // through an automatic index; `length()` of a blob reads the record
1031
+ // header, never the payload. Every walked commit LEFT JOINs its meta
1032
+ // (NULL when the commit vanished, which still advances the scan). The
1033
+ // change read then seeks only the commits the byte budget admits,
1034
+ // through the primary key, matching each change to the keys by the same
1035
+ // join.
1032
1036
  return readLiveWindowPages(
1033
1037
  query,
1034
- async (keys, limit) =>
1038
+ async (keys, limit, budget) =>
1035
1039
  (
1036
1040
  await this.guardedAll<{
1037
1041
  commit_seq: number;
@@ -1041,23 +1045,43 @@ export abstract class SqliteFamilyStorage {
1041
1045
  bytes: number | null;
1042
1046
  changes: number | null;
1043
1047
  }>(
1044
- `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
1045
- c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
1048
+ `WITH RECURSIVE k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
1049
+ c AS MATERIALIZED (SELECT row_number() OVER (ORDER BY commit_seq) AS n, commit_seq FROM (SELECT DISTINCT s.commit_seq AS commit_seq
1046
1050
  FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
1047
1051
  ON s.partition=?1 AND s.var=k.var AND s.value=k.value
1048
1052
  AND s.commit_seq>k.after AND s.commit_seq<=?3
1049
- ORDER BY s.commit_seq LIMIT ?4)
1050
- SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
1051
- z.i AS key, z.bytes AS bytes, z.changes AS changes
1052
- FROM c
1053
- LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
1054
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
1055
- sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
1056
- FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
1057
- CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
1058
- GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
1059
- ORDER BY c.commit_seq, z.i`,
1060
- [partition, JSON.stringify(keys), query.throughSeq, limit],
1053
+ ORDER BY s.commit_seq LIMIT ?4)),
1054
+ walk(n, commit_seq, sizes, changes, bytes) AS (
1055
+ SELECT 0, NULL, '[]', 0, 0
1056
+ UNION ALL
1057
+ SELECT c.n, c.commit_seq,
1058
+ (SELECT json_group_array(json_array(g.i, g.bytes, g.changes)) FROM (SELECT k.i AS i, count(*) AS changes,
1059
+ sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
1060
+ FROM sync_changes AS ch CROSS JOIN json_each(ch.scopes) AS e
1061
+ CROSS JOIN k ON k.var=e.key AND k.value=e.value
1062
+ WHERE ch.partition=?1 AND ch.commit_seq=c.commit_seq
1063
+ AND ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables))
1064
+ GROUP BY k.i) AS g),
1065
+ w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)),
1066
+ w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes))
1067
+ FROM walk AS w JOIN c ON c.n=w.n+1
1068
+ WHERE w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)) < ?5
1069
+ AND w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes)) <= ?6)
1070
+ SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
1071
+ z.value ->> 0 AS key, z.value ->> 1 AS bytes, z.value ->> 2 AS changes
1072
+ FROM walk AS w
1073
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=w.commit_seq
1074
+ LEFT JOIN json_each(w.sizes) AS z
1075
+ WHERE w.n>0
1076
+ ORDER BY w.commit_seq, key`,
1077
+ [
1078
+ partition,
1079
+ JSON.stringify(keys),
1080
+ query.throughSeq,
1081
+ limit,
1082
+ budget.changes,
1083
+ budget.bytes,
1084
+ ],
1061
1085
  )
1062
1086
  ).map((record) => ({
1063
1087
  commitSeq: record.commit_seq,
@@ -1084,7 +1108,8 @@ export abstract class SqliteFamilyStorage {
1084
1108
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
1085
1109
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
1086
1110
  FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
1087
- WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
1111
+ WHERE EXISTS (SELECT 1 FROM json_each(ch.scopes) AS e CROSS JOIN k ON k.var=e.key AND k.value=e.value
1112
+ WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)))
1088
1113
  ORDER BY ch.commit_seq, ch.idx`,
1089
1114
  [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
1090
1115
  )
@@ -177,12 +177,16 @@ export interface LiveCandidateRow {
177
177
  }
178
178
 
179
179
  /**
180
- * The §4.5 live scan over a backend's two reads. `readCandidates` returns
181
- * up to `limit` candidate commits above every key range's bound with their
182
- * per-key byte sizes, and no payload. `readChanges` loads the changes the
183
- * keys read in the given candidate commits, in (commitSeq, change index)
184
- * order; it never visits a commit outside that list, so the cost of a page
185
- * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
180
+ * The §4.5 live scan over a backend's two reads. `readCandidates` takes up
181
+ * to `limit` candidate commits above every key range's bound and sizes them
182
+ * in commit order, without payload, until their change count reaches
183
+ * `budget.changes` or their byte size passes `budget.bytes`. It returns the
184
+ * sized candidates with their per-key sizes, the one that reached the budget
185
+ * last, so a page sizes the commits it delivers and one more. `readChanges`
186
+ * loads the changes the keys read in the given candidate commits, in
187
+ * (commitSeq, change index) order; it never visits a commit outside that
188
+ * list, so the cost of a page follows its candidates and not the length of
189
+ * the change log below them. Each distinct unit key (the first variable, keys sorted)
186
190
  * becomes one index range starting at the lowest bound of the units that
187
191
  * share it. A change is kept when a unit covers its table, the pair's
188
192
  * effective scopes match it, and its commit lies above that unit's own
@@ -205,6 +209,7 @@ export async function readLiveWindowPages(
205
209
  readCandidates: (
206
210
  keys: readonly LiveKey[],
207
211
  limit: number,
212
+ budget: { readonly changes: number; readonly bytes: number },
208
213
  ) => Promise<readonly LiveCandidateRow[]>,
209
214
  readChanges: (
210
215
  keys: readonly LiveKey[],
@@ -294,8 +299,13 @@ export async function readLiveWindowPages(
294
299
  return undefined;
295
300
  };
296
301
  while (keys.length > 0) {
297
- const rows = await readCandidates(scanKeys(), limit);
302
+ const rows = await readCandidates(scanKeys(), limit, {
303
+ changes: query.limitChanges - delivered,
304
+ bytes: query.limitBytes - loaded,
305
+ });
298
306
  let candidates = 0;
307
+ // The change budget ended the read, possibly before `limit` candidates.
308
+ let reached = false;
299
309
  for (let i = 0; i < rows.length;) {
300
310
  const head = rows[i]!;
301
311
  candidates += 1;
@@ -342,13 +352,14 @@ export async function readLiveWindowPages(
342
352
  bound += changes;
343
353
  pending.push({ commitSeq: head.commitSeq, meta: head.meta });
344
354
  if (delivered + bound >= query.limitChanges) {
355
+ reached = true;
345
356
  const cut = await load();
346
357
  if (cut !== undefined) return { commits, throughSeq: cut };
347
358
  }
348
359
  }
349
360
  const cut = await load();
350
361
  if (cut !== undefined) return { commits, throughSeq: cut };
351
- if (candidates < limit) break;
362
+ if (candidates < limit && !reached) break;
352
363
  }
353
364
  return { commits, throughSeq: query.throughSeq };
354
365
  }