@syncular/server 0.38.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -67,9 +67,10 @@ import {
67
67
  migratePayload,
68
68
  parseLayouts,
69
69
  quoteIdent,
70
+ tableColumnNames,
70
71
  retiredTableNames,
71
72
  rewritePlan,
72
- rewriteRowSql,
73
+ columnSqlType,
73
74
  rewriteValues,
74
75
  SCHEMA_META_DDL_POSTGRES,
75
76
  SYNC_COLUMN_VERSIONS_COLUMN,
@@ -153,7 +154,7 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
153
154
  * their table.
154
155
  *
155
156
  * `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
156
- * scope index: `writeRowOn` and `deleteRow` replace a row's scope entries
157
+ * scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
157
158
  * with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
158
159
  * column of the inverted PRIMARY KEY, so without this index that predicate
159
160
  * scans the table's whole scope range on every row written (a bulk import
@@ -645,8 +646,30 @@ async function rewriteRowsOn(
645
646
  oldLayout: readonly StoredColumnLayout[] | undefined,
646
647
  ): Promise<void> {
647
648
  const select = selectRowsForRewriteSql(table, 'postgres');
648
- const update = rewriteRowSql(table, 'postgres');
649
- const BATCH = 500;
649
+ // One UPDATE per page: behind a network hop (Hyperdrive, a managed
650
+ // database) every statement is a round trip, and the bump holds the
651
+ // migration lock for its whole duration. The page's rows bind as one
652
+ // VALUES list with typed placeholders (`rewriteValues` order: projection
653
+ // columns, payload, partition, row id).
654
+ const types = [
655
+ ...(table.materialize
656
+ ? table.columns.map((column) => columnSqlType(column, 'postgres'))
657
+ : []),
658
+ 'BYTEA',
659
+ 'TEXT',
660
+ 'TEXT',
661
+ ];
662
+ const projected = table.materialize ? table.columns : [];
663
+ const sets = [
664
+ ...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
665
+ `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
666
+ ].join(', ');
667
+ const aliases = types.map((_, i) => `c${i}`).join(', ');
668
+ const partitionAlias = `c${projected.length + 1}`;
669
+ const rowIdAlias = `c${projected.length + 2}`;
670
+ // A page binds types.length parameters per row; PGlite corrupts its state
671
+ // above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
672
+ const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
650
673
  let afterPartition = '';
651
674
  let afterRowId = '';
652
675
  for (;;) {
@@ -654,21 +677,30 @@ async function rewriteRowsOn(
654
677
  partition: string;
655
678
  row_id: string;
656
679
  payload: unknown;
657
- }>(select, [afterPartition, afterRowId, BATCH]);
680
+ }>(select, [afterPartition, afterRowId, batch]);
658
681
  if (rows.length === 0) break;
659
- for (const row of rows) {
682
+ const params: unknown[] = [];
683
+ const values = rows.map((row) => {
660
684
  const bytes = asBytes(row.payload);
661
685
  const payload =
662
686
  oldLayout !== undefined
663
687
  ? migratePayload(oldLayout, table, bytes)
664
688
  : bytes;
665
- await q.query(
666
- update,
667
- rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
689
+ const base = params.length;
690
+ params.push(
691
+ ...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
668
692
  );
669
- }
693
+ return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
694
+ });
695
+ await q.query(
696
+ `UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
697
+ FROM (VALUES ${values.join(',')}) AS v(${aliases})
698
+ WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
699
+ AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
700
+ params,
701
+ );
670
702
  const last = rows[rows.length - 1];
671
- if (last === undefined || rows.length < BATCH) break;
703
+ if (last === undefined || rows.length < batch) break;
672
704
  afterPartition = last.partition;
673
705
  afterRowId = last.row_id;
674
706
  }
@@ -709,37 +741,70 @@ async function scanRowsByIndexOn(
709
741
  return rows.map(toStoredRow);
710
742
  }
711
743
 
712
- async function writeRowOn(
744
+ /**
745
+ * Write rows of one table in one statement per chunk: the upsert, and the
746
+ * scope index synced to each row's scope map (entries outside it deleted,
747
+ * missing ones inserted). Row ids are distinct, so no index entry is touched
748
+ * twice and the writable CTEs cannot conflict. The scopes bind as text
749
+ * before the JSONB cast (see `appendCommit`). Every chunk is handed to `q`
750
+ * before the first one settles, so a statement issued after this call
751
+ * queues behind all of them on the transaction FIFO.
752
+ */
753
+ function writeRowsOn(
713
754
  q: PgQueryable,
714
755
  compiled: CompiledTable,
715
756
  partition: string,
716
- row: StoredRow,
717
- ): Promise<void> {
718
- // One statement per row write: the upsert, and the scope index synced to
719
- // the row's scope map (entries outside it deleted, missing ones inserted).
720
- // No index entry is touched twice, so the writable CTEs cannot conflict.
721
- // The scopes bind as text before the JSONB cast (see `appendCommit`).
722
- const params = upsertValues(compiled, partition, row, 'postgres');
723
- const n = params.length;
724
- await q.query(
725
- `WITH upserted AS (${upsertSql(compiled, 'postgres')}),
726
- scope AS (SELECT key, value FROM jsonb_each_text($${n + 1}::text::jsonb)),
727
- removed AS (
728
- DELETE FROM sync_row_scopes
729
- WHERE partition=$${n + 2} AND tbl=$${n + 3} AND row_id=$${n + 4}
730
- AND (var, value) NOT IN (SELECT key, value FROM scope)
731
- )
732
- INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
733
- SELECT $${n + 2}, $${n + 3}, key, value, $${n + 4} FROM scope
734
- ON CONFLICT DO NOTHING`,
735
- [
736
- ...params,
737
- JSON.stringify(row.scopes),
738
- partition,
739
- compiled.name,
740
- row.rowId,
741
- ],
742
- );
757
+ rows: readonly {
758
+ readonly row: StoredRow;
759
+ /** `upsertValues` of the row, computed when it was staged. */
760
+ readonly values: readonly unknown[];
761
+ }[],
762
+ ): Promise<unknown> {
763
+ const width = tableColumnNames(compiled).length;
764
+ // PGlite corrupts its state above 32,767 bound parameters (see
765
+ // PG_CHANGES_PER_STATEMENT).
766
+ const chunk = Math.max(1, Math.floor(30_000 / width));
767
+ const writes: Promise<unknown>[] = [];
768
+ for (let first = 0; first < rows.length; first += chunk) {
769
+ const slice = rows.slice(first, first + chunk);
770
+ const params = slice.flatMap((entry) => entry.values);
771
+ const n = params.length;
772
+ writes.push(
773
+ q.query(
774
+ `WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
775
+ batch AS (
776
+ SELECT row_id, scopes
777
+ FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
778
+ ),
779
+ scope AS (
780
+ SELECT batch.row_id, entry.key, entry.value
781
+ FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
782
+ ),
783
+ removed AS (
784
+ DELETE FROM sync_row_scopes existing
785
+ USING batch
786
+ WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
787
+ AND existing.row_id=batch.row_id
788
+ AND NOT EXISTS (
789
+ SELECT 1 FROM scope
790
+ WHERE scope.row_id=existing.row_id
791
+ AND scope.key=existing.var AND scope.value=existing.value)
792
+ )
793
+ INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
794
+ SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
795
+ ON CONFLICT DO NOTHING`,
796
+ [
797
+ ...params,
798
+ JSON.stringify(
799
+ slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
800
+ ),
801
+ partition,
802
+ compiled.name,
803
+ ],
804
+ ),
805
+ );
806
+ }
807
+ return Promise.all(writes);
743
808
  }
744
809
 
745
810
  async function getPushResultOn(
@@ -788,6 +853,8 @@ async function queryAuthoritativeOn(
788
853
 
789
854
  class PostgresTransaction implements StorageTransaction {
790
855
  #client: PgQueryable;
856
+ /** The FIFO without the staged-write prelude (see the constructor). */
857
+ #serial: PgQueryable;
791
858
  #partition: string;
792
859
  /** Storage-owned relation rewrite, bound to this transaction's partition. */
793
860
  readonly #prepareQuery: (
@@ -805,6 +872,25 @@ class PostgresTransaction implements StorageTransaction {
805
872
  */
806
873
  readonly #prefetchedRows = new Map<string, StoredRow | null>();
807
874
  readonly #prefetchedTombstones = new Map<string, number | null>();
875
+ /**
876
+ * Upserts staged per table (row id → row and its push operation index).
877
+ * Every statement of this transaction writes them first (see the
878
+ * constructor), so reads and later writes observe them; a commit's rows
879
+ * reach the database as one statement per table instead of one per row.
880
+ * Tables with a unique index write immediately: a violation must name its
881
+ * operation.
882
+ */
883
+ readonly #staged = new Map<
884
+ CompiledTable,
885
+ Map<
886
+ string,
887
+ {
888
+ readonly row: StoredRow;
889
+ readonly values: readonly unknown[];
890
+ readonly opIndex?: number;
891
+ }
892
+ >
893
+ >();
808
894
  /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
809
895
  #resolve: () => void;
810
896
  #reject: (error: unknown) => void;
@@ -825,7 +911,7 @@ class PostgresTransaction implements StorageTransaction {
825
911
  // issued through this transaction is chained on one FIFO — the same
826
912
  // shape as `SqliteServerStorage.queryAuthoritative`.
827
913
  let tail: Promise<void> = Promise.resolve();
828
- this.#client = {
914
+ const serial: PgQueryable = {
829
915
  query: <Row>(text: string, params?: readonly unknown[]) => {
830
916
  const run = tail.then(() => client.query<Row>(text, params));
831
917
  tail = run.then(
@@ -835,6 +921,23 @@ class PostgresTransaction implements StorageTransaction {
835
921
  return run;
836
922
  },
837
923
  };
924
+ this.#serial = serial;
925
+ this.#client = {
926
+ query: async <Row>(text: string, params?: readonly unknown[]) => {
927
+ // Queue the staged writes ahead of this statement synchronously, so
928
+ // a concurrently issued statement cannot overtake them.
929
+ const staged = this.#writeStaged();
930
+ const run = serial.query<Row>(text, params);
931
+ try {
932
+ await staged;
933
+ } catch (error) {
934
+ // The aborted transaction fails the queued statement too.
935
+ run.catch(() => undefined);
936
+ throw error;
937
+ }
938
+ return run;
939
+ },
940
+ };
838
941
  this.#partition = partition;
839
942
  this.#writerVersion = writerVersion;
840
943
  this.#resolveTable = resolveTable;
@@ -847,6 +950,27 @@ class PostgresTransaction implements StorageTransaction {
847
950
  if (!this.#open) throw new Error('transaction already finished');
848
951
  }
849
952
 
953
+ #writeStaged(): Promise<unknown> {
954
+ if (this.#staged.size === 0) return Promise.resolve();
955
+ const staged = [...this.#staged];
956
+ this.#staged.clear();
957
+ return Promise.all(
958
+ staged.map(([table, rows]) => {
959
+ const entries = [...rows.values()];
960
+ return writeRowsOn(this.#serial, table, this.#partition, entries).catch(
961
+ (error: unknown) => {
962
+ // Staged tables have no unique index, so a violation is a column
963
+ // constraint; it names the first staged operation of the table.
964
+ if (isPostgresConstraintError(error)) {
965
+ throw new StorageConstraintError(error, entries[0]?.opIndex);
966
+ }
967
+ throw error;
968
+ },
969
+ );
970
+ }),
971
+ );
972
+ }
973
+
850
974
  getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
851
975
  this.#assertOpen();
852
976
  const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
@@ -1039,6 +1163,7 @@ class PostgresTransaction implements StorageTransaction {
1039
1163
  if (!this.#pushApplySavepoint) {
1040
1164
  throw new Error('push rejection requires its apply savepoint');
1041
1165
  }
1166
+ this.#staged.clear();
1042
1167
  await this.#client.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
1043
1168
  await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
1044
1169
  this.#pushApplySavepoint = false;
@@ -1053,13 +1178,28 @@ class PostgresTransaction implements StorageTransaction {
1053
1178
  ): Promise<void> {
1054
1179
  this.#assertOpen();
1055
1180
  this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
1056
- try {
1057
- await writeRowOn(
1058
- this.#client,
1059
- this.#resolveTable(table),
1060
- this.#partition,
1181
+ const compiled = this.#resolveTable(table);
1182
+ // Decodes the payload: an undecodable row fails here, at its operation.
1183
+ const values = upsertValues(compiled, this.#partition, row, 'postgres');
1184
+ if (!compiled.indexes.some((index) => index.unique === true)) {
1185
+ let rows = this.#staged.get(compiled);
1186
+ if (rows === undefined) {
1187
+ rows = new Map();
1188
+ this.#staged.set(compiled, rows);
1189
+ }
1190
+ // A later upsert of the same row in this transaction replaces it.
1191
+ rows.delete(row.rowId);
1192
+ rows.set(row.rowId, {
1061
1193
  row,
1062
- );
1194
+ values,
1195
+ ...(context === undefined ? {} : { opIndex: context.opIndex }),
1196
+ });
1197
+ return;
1198
+ }
1199
+ try {
1200
+ await writeRowsOn(this.#client, compiled, this.#partition, [
1201
+ { row, values },
1202
+ ]);
1063
1203
  } catch (error) {
1064
1204
  if (isPostgresConstraintError(error)) {
1065
1205
  throw new StorageConstraintError(error, context?.opIndex);
@@ -1232,6 +1372,7 @@ class PostgresTransaction implements StorageTransaction {
1232
1372
 
1233
1373
  async commit(): Promise<void> {
1234
1374
  this.#assertOpen();
1375
+ await this.#writeStaged();
1235
1376
  this.#open = false;
1236
1377
  // Signal the transaction wrapper to COMMIT; wait for it to actually land.
1237
1378
  this.#resolve();
@@ -1240,6 +1381,7 @@ class PostgresTransaction implements StorageTransaction {
1240
1381
 
1241
1382
  async rollback(): Promise<void> {
1242
1383
  if (!this.#open) return;
1384
+ this.#staged.clear();
1243
1385
  this.#open = false;
1244
1386
  this.#reject(new RollbackSignal());
1245
1387
  // Swallow — a rollback is a normal outcome, not an error to the caller.
@@ -1280,11 +1422,11 @@ export class PostgresServerStorage implements ServerStorage {
1280
1422
  #schemaVersion: number | undefined;
1281
1423
  /**
1282
1424
  * Page reads waiting for the end of the current microtask turn, keyed by
1283
- * statement and partition. A pull starts the window read (or first
1425
+ * batch stream and partition. A pull starts the window read (or first
1284
1426
  * snapshot page) of every subscription before awaiting any of them, so the
1285
- * window reads of all tables leave as ONE set-based statement (snapshot
1286
- * pages as one per table, whose SQL names the table) instead of one round
1287
- * trip per subscription or table (SYNCULAR-PULL-ROUNDTRIPS-001).
1427
+ * reads of all tables leave as ONE set-based statement per stream instead
1428
+ * of one round trip per subscription or table
1429
+ * (SYNCULAR-PULL-ROUNDTRIPS-001).
1288
1430
  */
1289
1431
  readonly #pageBatches = new Map<string, PendingPage[]>();
1290
1432
 
@@ -1293,37 +1435,42 @@ export class PostgresServerStorage implements ServerStorage {
1293
1435
  }
1294
1436
 
1295
1437
  /**
1296
- * Queue one page read into the batch for (`sql`, `partition`). The batch
1297
- * runs as a single statement with the queued specs bound as a JSON array
1298
- * (`$2`); each spec gets its position as `i`, and the result rows are
1299
- * split back by their `sub` column.
1438
+ * Queue one page read into the batch for (`key`, `partition`). `sql` is the
1439
+ * statement every batch of that stream shares, or a builder over the queued
1440
+ * specs when the statement names what the batch contains (the snapshot scan
1441
+ * joins one branch per table present). The batch runs as a single statement
1442
+ * with the queued specs bound as a JSON array (`$2`); each spec gets its
1443
+ * position as `i`, and the result rows are split back by their `sub`
1444
+ * column.
1300
1445
  */
1301
1446
  #batchedPage<Row>(
1302
- sql: string,
1447
+ key: string,
1448
+ sql: string | ((specs: readonly Record<string, unknown>[]) => string),
1303
1449
  partition: string,
1304
1450
  spec: Record<string, unknown>,
1305
1451
  ): Promise<Row[]> {
1306
- const key = `${sql}\u0000${partition}`;
1307
- let batch = this.#pageBatches.get(key);
1452
+ const batchKey = `${key}\u0000${partition}`;
1453
+ let batch = this.#pageBatches.get(batchKey);
1308
1454
  if (batch === undefined) {
1309
1455
  const queued: PendingPage[] = [];
1310
1456
  batch = queued;
1311
- this.#pageBatches.set(key, queued);
1457
+ this.#pageBatches.set(batchKey, queued);
1312
1458
  queueMicrotask(() => {
1313
- this.#pageBatches.delete(key);
1314
- const specs = JSON.stringify(
1315
- queued.map((entry, i) => ({ ...entry.spec, i })),
1316
- );
1317
- this.#exec.query<PgRow>(sql, [partition, specs]).then(
1318
- ({ rows }) => {
1319
- const grouped = queued.map((): PgRow[] => []);
1320
- for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
1321
- queued.forEach((entry, i) => entry.resolve(grouped[i] ?? []));
1322
- },
1323
- (error: unknown) => {
1324
- for (const entry of queued) entry.reject(error);
1325
- },
1326
- );
1459
+ this.#pageBatches.delete(batchKey);
1460
+ const specs = queued.map((entry, i) => ({ ...entry.spec, i }));
1461
+ const statement = typeof sql === 'string' ? sql : sql(specs);
1462
+ this.#exec
1463
+ .query<PgRow>(statement, [partition, JSON.stringify(specs)])
1464
+ .then(
1465
+ ({ rows }) => {
1466
+ const grouped = queued.map((): PgRow[] => []);
1467
+ for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
1468
+ queued.forEach((entry, i) => entry.resolve(grouped[i] ?? []));
1469
+ },
1470
+ (error: unknown) => {
1471
+ for (const entry of queued) entry.reject(error);
1472
+ },
1473
+ );
1327
1474
  });
1328
1475
  }
1329
1476
  const target = batch;
@@ -1538,10 +1685,6 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1538
1685
  // backfill for flipped-on materialization) — all inside one
1539
1686
  // transaction (Postgres DDL is transactional — a failed bump leaves
1540
1687
  // no half-state).
1541
- const layouts = parseLayouts(
1542
- typeof marker.layouts === 'string' ? marker.layouts : undefined,
1543
- );
1544
- const retiredTables = retiredTableNames(schema, layouts);
1545
1688
  await this.#exec.transaction(async (client) => {
1546
1689
  // RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
1547
1690
  // statement, before any read, DDL, rewrite, marker or checkpoint
@@ -1554,6 +1697,31 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1554
1697
  // when it takes this lock, so a writer already inside completes while
1555
1698
  // the migration waits.
1556
1699
  await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
1700
+ // The probe ran before the lock. A concurrent `ensureSchema` (another
1701
+ // isolate, the Durable Object's first request) may have bumped while
1702
+ // this one waited; re-running the rewrite would decode its new-layout
1703
+ // payloads with the old layout. The locked marker decides.
1704
+ const { rows: lockedMarker } = await client.query<{
1705
+ schema_version: unknown;
1706
+ layouts: unknown;
1707
+ }>('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
1708
+ const lockedVersion =
1709
+ lockedMarker[0]?.schema_version === null ||
1710
+ lockedMarker[0]?.schema_version === undefined
1711
+ ? undefined
1712
+ : asNumber(lockedMarker[0].schema_version);
1713
+ if (lockedVersion !== undefined && lockedVersion > schema.version) {
1714
+ throw new Error(
1715
+ `stored schema version ${lockedVersion} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
1716
+ );
1717
+ }
1718
+ if (lockedVersion === schema.version) return;
1719
+ const layouts = parseLayouts(
1720
+ typeof lockedMarker[0]?.layouts === 'string'
1721
+ ? lockedMarker[0].layouts
1722
+ : undefined,
1723
+ );
1724
+ const retiredTables = retiredTableNames(schema, layouts);
1557
1725
  const existing = new Map<string, ReadonlySet<string>>();
1558
1726
  const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
1559
1727
  for (const table of schema.tables.values()) {
@@ -1620,13 +1788,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1620
1788
  );
1621
1789
  }
1622
1790
  }
1791
+ if (retiredTables.length > 0) {
1792
+ // Neither index leads with `tbl`: one scan per table, not one per
1793
+ // retired table.
1794
+ await client.query(
1795
+ `DELETE FROM sync_row_scopes
1796
+ WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
1797
+ [JSON.stringify(retiredTables)],
1798
+ );
1799
+ await client.query(
1800
+ `DELETE FROM sync_blob_refs
1801
+ WHERE tbl IN (SELECT jsonb_array_elements_text($1::text::jsonb))`,
1802
+ [JSON.stringify(retiredTables)],
1803
+ );
1804
+ }
1623
1805
  for (const tableName of retiredTables) {
1624
- await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
1625
- tableName,
1626
- ]);
1627
- await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
1628
- tableName,
1629
- ]);
1630
1806
  await client.query(dropTableDdl(tableName));
1631
1807
  }
1632
1808
  for (const statement of schemaDdl(
@@ -2359,6 +2535,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2359
2535
  if (firstVariable === undefined || values.length === 0) return 0;
2360
2536
  // Concurrent reads of a pull share one statement across tables.
2361
2537
  const [row] = await this.#batchedPage<{ seq: unknown }>(
2538
+ 'latestScopedChange',
2362
2539
  LATEST_SCOPED_CHANGE_BATCH_SQL,
2363
2540
  partition,
2364
2541
  { tbl: query.table, var: firstVariable, vals: values },
@@ -2388,6 +2565,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2388
2565
  const batchSize = Math.max(64, query.limitChanges);
2389
2566
  while (deliveredChanges < query.limitChanges) {
2390
2567
  const records = await this.#batchedPage<CommitWindowRecord>(
2568
+ 'commitWindow',
2391
2569
  COMMIT_WINDOW_BATCH_SQL,
2392
2570
  partition,
2393
2571
  {
@@ -2456,21 +2634,30 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2456
2634
  if (firstValues.length === 0) return [];
2457
2635
  // Candidates via the inverted index (ordered + LIMITed at the covering
2458
2636
  // PK) LEFT JOINed to the row table — one round trip per page, never one
2459
- // per row, and pages of concurrent scans on the same table share one
2460
- // statement (see `scanRowBatchSql`). Exact multi-variable verification
2461
- // against the stored scope map happens below.
2462
- const sql = scanRowBatchSql(this.table(query.table));
2637
+ // per row, and concurrent scans across tables share one statement (see
2638
+ // `scanRowBatchSql`). Exact multi-variable verification against the
2639
+ // stored scope map happens below.
2463
2640
  const rows: StoredRow[] = [];
2464
2641
  let afterRowId = query.afterRowId ?? '';
2465
2642
  const batchSize = Math.max(64, query.limit);
2466
2643
  while (rows.length < query.limit) {
2467
- const records = await this.#batchedPage<RowRecord>(sql, partition, {
2468
- tbl: query.table,
2469
- var: firstVariable,
2470
- vals: firstValues,
2471
- after: afterRowId,
2472
- lim: batchSize,
2473
- });
2644
+ const records = await this.#batchedPage<RowRecord>(
2645
+ 'scanRows',
2646
+ (specs) =>
2647
+ scanRowBatchSql(
2648
+ [...new Set(specs.map((spec) => String(spec.tbl)))].map((table) =>
2649
+ this.table(table),
2650
+ ),
2651
+ ),
2652
+ partition,
2653
+ {
2654
+ tbl: query.table,
2655
+ var: firstVariable,
2656
+ vals: firstValues,
2657
+ after: afterRowId,
2658
+ lim: batchSize,
2659
+ },
2660
+ );
2474
2661
  if (records.length === 0) break;
2475
2662
  for (const record of records) {
2476
2663
  afterRowId = record.row_id;
package/src/pull.ts CHANGED
@@ -484,8 +484,8 @@ async function* bootstrapSegments(
484
484
  /**
485
485
  * The storage reads a section needs before its first frame. A pull starts
486
486
  * them for every subscription before awaiting any, so a storage that batches
487
- * concurrent page reads (PostgreSQL, D1) answers all subscriptions of one
488
- * table with one statement, and the horizon re-check runs once per pull.
487
+ * concurrent page reads (PostgreSQL, D1) answers the first pages of every
488
+ * subscription in one statement, and the horizon re-check runs once per pull.
489
489
  */
490
490
  export interface PreparedSection {
491
491
  /** §4.5 window read for an incremental cursor; empty when none was read. */
@@ -499,7 +499,11 @@ export interface PreparedSection {
499
499
  readonly firstPage: StoredRow[] | undefined;
500
500
  /** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
501
501
  readonly pin: number | undefined;
502
- /** §5.3 stored image for a sqlite-lane bootstrap's reuse key, if any. */
502
+ /**
503
+ * §5.3 stored image for a sqlite-lane bootstrap's reuse key. Read only when
504
+ * the eligibility probe shows a snapshot larger than one rows page, since a
505
+ * smaller table can never match a stored image.
506
+ */
503
507
  readonly image: SegmentRecord | undefined;
504
508
  }
505
509
 
@@ -574,21 +578,30 @@ export async function prepareSections(
574
578
  const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
575
579
  return { asOf: pin, pin };
576
580
  });
577
- const lane = pinned.then(async ({ asOf, pin }) => ({
578
- pin,
579
- image: await ctx.segments.find(
580
- {
581
- partition: ctx.partition,
582
- logEpoch,
583
- table: plan.table.name,
584
- schemaVersion: schema.version,
585
- mediaType: 'sqlite',
586
- scopeDigest: await scopeDigest(plan.effective),
587
- asOfCommitSeq: asOf,
588
- },
589
- clockOf(ctx)(),
590
- ),
591
- }));
581
+ const lane = pinned.then(async ({ asOf, pin }) => {
582
+ // §5.3 eligibility: a stored image only matches a snapshot larger than
583
+ // one rows page, so the probe decides before the reuse lookup; a remote
584
+ // `find` on a small table would be a wasted round trip.
585
+ const probe = await firstPage;
586
+ if (probe.length <= limits.limitSnapshotRows) {
587
+ return { pin, image: undefined };
588
+ }
589
+ return {
590
+ pin,
591
+ image: await ctx.segments.find(
592
+ {
593
+ partition: ctx.partition,
594
+ logEpoch,
595
+ table: plan.table.name,
596
+ schemaVersion: schema.version,
597
+ mediaType: 'sqlite',
598
+ scopeDigest: await scopeDigest(plan.effective),
599
+ asOfCommitSeq: asOf,
600
+ },
601
+ clockOf(ctx)(),
602
+ ),
603
+ };
604
+ });
592
605
  return { firstPage, lane };
593
606
  });
594
607
  const settled = await Promise.all(