@syncular/server 0.82.2 → 0.83.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2404,8 +2404,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2404
2404
  // the commit meta (NULL when the commit vanished, which still advances
2405
2405
  // the scan) and, per key, the byte size of the changes the key reads;
2406
2406
  // `octet_length` of a TOASTed payload reads its pointer, never the value.
2407
- // The change read then loads only those changes of the commits the byte
2408
- // budget admits.
2407
+ // The change read then seeks only the commits the byte budget admits.
2409
2408
  return readLiveWindowPages(query, async (keys, limit) => {
2410
2409
  const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
2411
2410
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
@@ -2438,14 +2437,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2438
2437
  ? {}
2439
2438
  : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
2440
2439
  }));
2441
- }, async (keys, after, through) => {
2440
+ }, async (keys, commitSeqs) => {
2442
2441
  const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
2443
2442
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
2444
2443
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
2445
2444
  FROM sync_changes ch
2446
- WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
2445
+ WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
2447
2446
  AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
2448
- ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through]);
2447
+ ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)]);
2449
2448
  return rows.map((record) => ({
2450
2449
  commitSeq: asNumber(record.commit_seq),
2451
2450
  change: toStoredChange(record),
@@ -796,8 +796,9 @@ export class SqliteFamilyStorage {
796
796
  // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
797
797
  // when the commit vanished, which still advances the scan) and, per key,
798
798
  // the byte size of the changes the key reads; `length()` of a blob reads
799
- // the record header, never the payload. The change read then loads only
800
- // those changes of the commits the byte budget admits.
799
+ // the record header, never the payload. The change read then seeks only
800
+ // the commits the byte budget admits, through the primary key, with the
801
+ // key list materialized once per statement.
801
802
  return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
802
803
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
803
804
  FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
@@ -826,13 +827,12 @@ export class SqliteFamilyStorage {
826
827
  ...(record.key !== null
827
828
  ? { key: record.key, bytes: record.bytes ?? 0 }
828
829
  : {}),
829
- })), async (keys, after, through) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
830
+ })), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
830
831
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
831
832
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
832
- FROM sync_changes AS ch
833
- WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
834
- AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
835
- ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through])).map((record) => ({
833
+ FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
834
+ WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
835
+ ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)])).map((record) => ({
836
836
  commitSeq: record.commit_seq,
837
837
  change: toStoredChange(record),
838
838
  })));
@@ -72,8 +72,9 @@ export interface LiveCandidateRow {
72
72
  * The §4.5 live scan over a backend's two reads. `readCandidates` returns
73
73
  * up to `limit` candidate commits above every key range's bound with their
74
74
  * per-key byte sizes, and no payload. `readChanges` loads the changes the
75
- * keys read in a commit range `(after, through]`, in (commitSeq, change
76
- * index) order. Each distinct unit key (the first variable, keys sorted)
75
+ * keys read in the given candidate commits, in (commitSeq, change index)
76
+ * order; it never visits a commit outside that list, so the cost of a page
77
+ * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
77
78
  * becomes one index range starting at the lowest bound of the units that
78
79
  * share it. A change is kept when a unit covers its table, the pair's
79
80
  * effective scopes match it, and its commit lies above that unit's own
@@ -86,7 +87,7 @@ export interface LiveCandidateRow {
86
87
  * units of the keys that read the most of it, largest first, until the rest
87
88
  * fits (§4.6).
88
89
  */
89
- export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], after: number, through: number) => Promise<readonly {
90
+ export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
90
91
  readonly commitSeq: number;
91
92
  readonly change: StoredChange;
92
93
  }[]>): Promise<LiveWindow>;
@@ -101,8 +101,9 @@ export async function transactionQuery(tx, query) {
101
101
  * The §4.5 live scan over a backend's two reads. `readCandidates` returns
102
102
  * up to `limit` candidate commits above every key range's bound with their
103
103
  * per-key byte sizes, and no payload. `readChanges` loads the changes the
104
- * keys read in a commit range `(after, through]`, in (commitSeq, change
105
- * index) order. Each distinct unit key (the first variable, keys sorted)
104
+ * keys read in the given candidate commits, in (commitSeq, change index)
105
+ * order; it never visits a commit outside that list, so the cost of a page
106
+ * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
106
107
  * becomes one index range starting at the lowest bound of the units that
107
108
  * share it. A change is kept when a unit covers its table, the pair's
108
109
  * effective scopes match it, and its commit lies above that unit's own
@@ -159,7 +160,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
159
160
  if (last === undefined)
160
161
  return undefined;
161
162
  const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
162
- const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
163
+ const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
163
164
  pending = [];
164
165
  scanned = last.commitSeq;
165
166
  for (let i = 0; i < rows.length;) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.82.2",
3
+ "version": "0.83.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.82.2"
71
+ "@syncular/core": "0.83.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
@@ -3551,8 +3551,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3551
3551
  // the commit meta (NULL when the commit vanished, which still advances
3552
3552
  // the scan) and, per key, the byte size of the changes the key reads;
3553
3553
  // `octet_length` of a TOASTed payload reads its pointer, never the value.
3554
- // The change read then loads only those changes of the commits the byte
3555
- // budget admits.
3554
+ // The change read then seeks only the commits the byte budget admits.
3556
3555
  return readLiveWindowPages(
3557
3556
  query,
3558
3557
  async (keys, limit) => {
@@ -3597,7 +3596,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3597
3596
  : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
3598
3597
  }));
3599
3598
  },
3600
- async (keys, after, through) => {
3599
+ async (keys, commitSeqs) => {
3601
3600
  const { rows } = await this.#exec.query<
3602
3601
  ChangeRecord & { commit_seq: unknown }
3603
3602
  >(
@@ -3605,10 +3604,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3605
3604
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
3606
3605
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
3607
3606
  FROM sync_changes ch
3608
- WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
3607
+ WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
3609
3608
  AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
3610
3609
  ORDER BY ch.commit_seq, ch.idx`,
3611
- [partition, JSON.stringify(keys), after, through],
3610
+ [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
3612
3611
  );
3613
3612
  return rows.map((record) => ({
3614
3613
  commitSeq: asNumber(record.commit_seq),
@@ -1208,8 +1208,9 @@ export abstract class SqliteFamilyStorage {
1208
1208
  // the index is missing). Each candidate LEFT JOINs its commit meta (NULL
1209
1209
  // when the commit vanished, which still advances the scan) and, per key,
1210
1210
  // the byte size of the changes the key reads; `length()` of a blob reads
1211
- // the record header, never the payload. The change read then loads only
1212
- // those changes of the commits the byte budget admits.
1211
+ // the record header, never the payload. The change read then seeks only
1212
+ // the commits the byte budget admits, through the primary key, with the
1213
+ // key list materialized once per statement.
1213
1214
  return readLiveWindowPages(
1214
1215
  query,
1215
1216
  async (keys, limit) =>
@@ -1253,17 +1254,16 @@ export abstract class SqliteFamilyStorage {
1253
1254
  ? { key: record.key, bytes: record.bytes ?? 0 }
1254
1255
  : {}),
1255
1256
  })),
1256
- async (keys, after, through) =>
1257
+ async (keys, commitSeqs) =>
1257
1258
  (
1258
1259
  await this.guardedAll<SqliteChangeRecord & { commit_seq: number }>(
1259
- `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
1260
+ `WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
1260
1261
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
1261
1262
  ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
1262
- FROM sync_changes AS ch
1263
- WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
1264
- AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
1263
+ FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
1264
+ WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
1265
1265
  ORDER BY ch.commit_seq, ch.idx`,
1266
- [partition, JSON.stringify(keys), after, through],
1266
+ [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
1267
1267
  )
1268
1268
  ).map((record) => ({
1269
1269
  commitSeq: record.commit_seq,
@@ -175,8 +175,9 @@ export interface LiveCandidateRow {
175
175
  * The §4.5 live scan over a backend's two reads. `readCandidates` returns
176
176
  * up to `limit` candidate commits above every key range's bound with their
177
177
  * per-key byte sizes, and no payload. `readChanges` loads the changes the
178
- * keys read in a commit range `(after, through]`, in (commitSeq, change
179
- * index) order. Each distinct unit key (the first variable, keys sorted)
178
+ * keys read in the given candidate commits, in (commitSeq, change index)
179
+ * order; it never visits a commit outside that list, so the cost of a page
180
+ * follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
180
181
  * becomes one index range starting at the lowest bound of the units that
181
182
  * share it. A change is kept when a unit covers its table, the pair's
182
183
  * effective scopes match it, and its commit lies above that unit's own
@@ -197,8 +198,7 @@ export async function readLiveWindowPages(
197
198
  ) => Promise<readonly LiveCandidateRow[]>,
198
199
  readChanges: (
199
200
  keys: readonly LiveKey[],
200
- after: number,
201
- through: number,
201
+ commitSeqs: readonly number[],
202
202
  ) => Promise<
203
203
  readonly { readonly commitSeq: number; readonly change: StoredChange }[]
204
204
  >,
@@ -246,7 +246,10 @@ export async function readLiveWindowPages(
246
246
  const last = pending.at(-1);
247
247
  if (last === undefined) return undefined;
248
248
  const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
249
- const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
249
+ const rows = await readChanges(
250
+ scanKeys(),
251
+ pending.map((c) => c.commitSeq),
252
+ );
250
253
  pending = [];
251
254
  scanned = last.commitSeq;
252
255
  for (let i = 0; i < rows.length;) {