@syncular/server 0.15.48 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +66 -9
  2. package/dist/admin.js +1 -5
  3. package/dist/authoritative-query.d.ts +14 -6
  4. package/dist/authoritative-query.js +60 -82
  5. package/dist/d1-storage.d.ts +13 -1
  6. package/dist/d1-storage.js +468 -121
  7. package/dist/operations.d.ts +2 -0
  8. package/dist/operations.js +23 -4
  9. package/dist/postgres-storage.d.ts +6 -1
  10. package/dist/postgres-storage.js +80 -24
  11. package/dist/prune.d.ts +3 -1
  12. package/dist/prune.js +18 -14
  13. package/dist/pull.js +79 -39
  14. package/dist/push.js +5 -5
  15. package/dist/realtime.d.ts +1 -1
  16. package/dist/realtime.js +16 -10
  17. package/dist/relational-rows.d.ts +7 -1
  18. package/dist/relational-rows.js +17 -2
  19. package/dist/sqlite-bun.js +2 -2
  20. package/dist/sqlite-image.d.ts +3 -3
  21. package/dist/sqlite-image.js +10 -6
  22. package/dist/sqlite-node.js +2 -2
  23. package/dist/sqlite-storage.d.ts +6 -1
  24. package/dist/sqlite-storage.js +93 -32
  25. package/dist/storage-errors.d.ts +1 -1
  26. package/dist/storage-errors.js +7 -0
  27. package/dist/storage.d.ts +29 -5
  28. package/package.json +2 -2
  29. package/src/admin.ts +4 -6
  30. package/src/authoritative-query.ts +89 -94
  31. package/src/d1-storage.ts +641 -159
  32. package/src/operations.ts +31 -3
  33. package/src/postgres-storage.ts +146 -46
  34. package/src/prune.ts +29 -15
  35. package/src/pull.ts +102 -49
  36. package/src/push.ts +5 -5
  37. package/src/realtime.ts +20 -9
  38. package/src/relational-rows.ts +18 -2
  39. package/src/sqlite-bun.ts +2 -2
  40. package/src/sqlite-image.ts +26 -15
  41. package/src/sqlite-node.ts +2 -2
  42. package/src/sqlite-storage.ts +131 -37
  43. package/src/storage-errors.ts +22 -1
  44. package/src/storage.ts +50 -5
package/src/operations.ts CHANGED
@@ -1,3 +1,7 @@
1
+ import {
2
+ type AuthoritativeRelationPlan,
3
+ validateAuthoritativeRelationPlan,
4
+ } from './authoritative-query';
1
5
  import {
2
6
  decodeRow,
3
7
  decodeRemoteOperationRequest,
@@ -42,6 +46,7 @@ export interface AuthoritativeQueryDescriptor<Params = undefined> {
42
46
  readonly hasParams: boolean;
43
47
  readonly sql: string;
44
48
  readonly tables: readonly string[];
49
+ readonly relationPlans: readonly AuthoritativeRelationPlan[];
45
50
  readonly resultColumns: readonly {
46
51
  readonly name: string;
47
52
  readonly type:
@@ -280,6 +285,9 @@ export function registerRemoteQuery<Params>(
280
285
  options: RemoteQueryOptions<Params>,
281
286
  ): RegisteredRemoteQuery {
282
287
  if (
288
+ !Array.isArray(descriptor.relationPlans) ||
289
+ !descriptor.relationPlans.some((plan) => plan.sql === descriptor.sql) ||
290
+ descriptor.relationPlans.some((plan) => !Array.isArray(plan.relations)) ||
283
291
  descriptor.id.length === 0 ||
284
292
  new Set(descriptor.tables).size !== descriptor.tables.length ||
285
293
  !Array.isArray(descriptor.resultColumns) ||
@@ -288,9 +296,12 @@ export function registerRemoteQuery<Params>(
288
296
  descriptor.resultColumns.length
289
297
  ) {
290
298
  throw new Error(
291
- 'remote query requires a non-empty id and unique tables and result columns',
299
+ 'remote query requires generated relation plans, a non-empty id, and unique tables and result columns; regenerate queries',
292
300
  );
293
301
  }
302
+ for (const plan of descriptor.relationPlans) {
303
+ validateAuthoritativeRelationPlan(plan, descriptor.tables);
304
+ }
294
305
  if (
295
306
  !Number.isSafeInteger(options.maxRows) ||
296
307
  options.maxRows < 1 ||
@@ -394,12 +405,29 @@ export function registerRemoteQuery<Params>(
394
405
  'configured storage does not implement authoritative queries',
395
406
  );
396
407
  }
397
- await ctx.storage.ensureSchema(schema);
398
408
  const selectedSql = descriptor.sqlFor?.(params) ?? descriptor.sql;
409
+ const plan = descriptor.relationPlans.find(
410
+ (candidate) => candidate.sql === selectedSql,
411
+ );
412
+ if (plan === undefined) {
413
+ throw syncError(
414
+ 'operation.invalid_request',
415
+ 'selected SQL has no generated relation plan; regenerate queries',
416
+ );
417
+ }
418
+ await ctx.storage.ensureSchema(schema);
419
+ const prefix = 'SELECT * FROM (';
399
420
  let result;
400
421
  try {
401
422
  result = await ctx.storage.queryAuthoritative(ctx.partition, {
402
- sql: `SELECT * FROM (${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
423
+ plan: {
424
+ sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
425
+ relations: plan.relations.map((relation) => ({
426
+ ...relation,
427
+ start: relation.start + prefix.length,
428
+ end: relation.end + prefix.length,
429
+ })),
430
+ },
403
431
  params: [...descriptor.bind(params), options.maxRows + 1],
404
432
  tables: descriptor.tables,
405
433
  });
@@ -1,3 +1,6 @@
1
+ import { validateCommitPruneQuery } from './prune';
2
+ import { StorageQueryError } from './storage-errors';
3
+ import type { CommitPruneQuery, CommitPruneResult } from './storage';
1
4
  /**
2
5
  * Postgres server storage: the production database path.
3
6
  *
@@ -223,6 +226,28 @@ interface SerializedResult {
223
226
  details?: import('@syncular/core').RejectionDetails;
224
227
  }
225
228
 
229
+ async function lockPartitionOn(
230
+ client: PgQueryable,
231
+ partition: string,
232
+ ): Promise<void> {
233
+ const locked = await client.query(
234
+ 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
235
+ [partition],
236
+ );
237
+ if (locked.rows.length > 0) return;
238
+ // The first writer initializes the partition. Concurrent initializers may
239
+ // wait on this insert, so acquire the row lock again before applying writes.
240
+ await client.query(
241
+ `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
242
+ ON CONFLICT (partition) DO NOTHING`,
243
+ [partition],
244
+ );
245
+ await client.query(
246
+ 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
247
+ [partition],
248
+ );
249
+ }
250
+
226
251
  function toBase64(bytes: Uint8Array): string {
227
252
  return Buffer.from(bytes).toString('base64');
228
253
  }
@@ -649,15 +674,7 @@ class PostgresTransaction implements StorageTransaction {
649
674
 
650
675
  async lockPartitionForPush(): Promise<void> {
651
676
  this.#assertOpen();
652
- await this.#client.query(
653
- `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
654
- ON CONFLICT (partition) DO NOTHING`,
655
- [this.#partition],
656
- );
657
- await this.#client.query(
658
- 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
659
- [this.#partition],
660
- );
677
+ await lockPartitionOn(this.#client, this.#partition);
661
678
  await this.#client.query('SAVEPOINT syncular_push_candidate');
662
679
  this.#pushApplySavepoint = true;
663
680
  }
@@ -743,32 +760,40 @@ class PostgresTransaction implements StorageTransaction {
743
760
  // Allocate the next dense commitSeq under a per-partition row lock: the
744
761
  // UPDATE … RETURNING serializes concurrent pushes to this partition and
745
762
  // never leaves a gap on rollback (see the file header).
746
- const { rows } = await q.query<{ max_commit_seq: unknown }>(
747
- `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
748
- ON CONFLICT (partition) DO UPDATE
749
- SET max_commit_seq = sync_partitions.max_commit_seq + 1
750
- RETURNING max_commit_seq`,
751
- [p],
752
- );
753
- const commitSeq = asNumber(rows[0]?.max_commit_seq);
754
- await q.query(
755
- `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
756
- VALUES ($1,$2,$3,$4,$5,$6)`,
763
+ const { rows } = await q.query<{ commit_seq: unknown }>(
764
+ `WITH allocated AS (
765
+ INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
766
+ ON CONFLICT (partition) DO UPDATE
767
+ SET max_commit_seq = sync_partitions.max_commit_seq + 1
768
+ RETURNING max_commit_seq
769
+ )
770
+ INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
771
+ SELECT $1, max_commit_seq, $2, $3, $4, $5 FROM allocated
772
+ RETURNING commit_seq`,
757
773
  [
758
774
  p,
759
- commitSeq,
760
775
  commit.clientId,
761
776
  commit.clientCommitId,
762
777
  commit.actorId,
763
778
  commit.createdAtMs,
764
779
  ],
765
780
  );
781
+ const commitSeq = asNumber(rows[0]?.commit_seq);
766
782
  for (let idx = 0; idx < commit.changes.length; idx++) {
767
783
  const change = commit.changes[idx];
768
784
  if (change === undefined) continue;
785
+ // Bind serialized scopes as text before parsing JSONB. Drivers that
786
+ // encode JSONB parameters would otherwise store this string as a scalar.
769
787
  await q.query(
770
- `INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
771
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
788
+ `WITH inserted AS (
789
+ INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
790
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8::text::jsonb,$9)
791
+ RETURNING partition, tbl, commit_seq, scopes
792
+ )
793
+ INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
794
+ SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
795
+ FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
796
+ ON CONFLICT DO NOTHING`,
772
797
  [
773
798
  p,
774
799
  commitSeq,
@@ -781,13 +806,6 @@ class PostgresTransaction implements StorageTransaction {
781
806
  change.payload ?? null,
782
807
  ],
783
808
  );
784
- for (const [variable, value] of Object.entries(change.scopes)) {
785
- await q.query(
786
- `INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
787
- VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
788
- [p, change.table, variable, value, commitSeq],
789
- );
790
- }
791
809
  }
792
810
  return commitSeq;
793
811
  }
@@ -976,6 +994,9 @@ export class PostgresServerStorage implements ServerStorage {
976
994
  await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
977
995
  tableName,
978
996
  ]);
997
+ await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
998
+ tableName,
999
+ ]);
979
1000
  await client.query(dropTableDdl(tableName));
980
1001
  }
981
1002
  for (const statement of schemaDdl(
@@ -1047,6 +1068,7 @@ export class PostgresServerStorage implements ServerStorage {
1047
1068
  ): Promise<PartitionRegistryEntry> {
1048
1069
  if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
1049
1070
  await this.#exec.transaction(async (client) => {
1071
+ await lockPartitionOn(client, partition);
1050
1072
  await client.query(
1051
1073
  `INSERT INTO sync_partition_registry(
1052
1074
  partition, log_epoch, epoch_required, last_authenticated_at_ms
@@ -1148,7 +1170,7 @@ export class PostgresServerStorage implements ServerStorage {
1148
1170
  }
1149
1171
  const prepared = bindAuthoritativePartition(
1150
1172
  prepareAuthoritativeQuery(
1151
- query.sql,
1173
+ query.plan,
1152
1174
  query.params,
1153
1175
  query.tables,
1154
1176
  this.#tables,
@@ -1177,6 +1199,14 @@ export class PostgresServerStorage implements ServerStorage {
1177
1199
  });
1178
1200
  }
1179
1201
 
1202
+ async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
1203
+ const { rows } = await this.#exec.query<{ log_epoch: string }>(
1204
+ 'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
1205
+ [partition],
1206
+ );
1207
+ return rows[0]?.log_epoch;
1208
+ }
1209
+
1180
1210
  async getHorizonSeq(partition: string): Promise<number> {
1181
1211
  const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
1182
1212
  'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
@@ -1188,25 +1218,52 @@ export class PostgresServerStorage implements ServerStorage {
1188
1218
  async setHorizonSeq(partition: string, seq: number): Promise<void> {
1189
1219
  await this.#exec.query(
1190
1220
  `INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
1191
- ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`,
1221
+ ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`,
1192
1222
  [partition, seq],
1193
1223
  );
1194
1224
  }
1195
1225
 
1196
- async pruneCommitsThrough(partition: string, seq: number): Promise<number> {
1197
- const removed = await this.#exec.query(
1198
- 'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
1199
- [partition, seq],
1200
- );
1201
- await this.#exec.query(
1202
- 'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
1203
- [partition, seq],
1204
- );
1205
- await this.#exec.query(
1206
- 'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
1207
- [partition, seq],
1208
- );
1209
- return removed.rowCount;
1226
+ async pruneCommitsThrough(
1227
+ partition: string,
1228
+ query: CommitPruneQuery,
1229
+ ): Promise<CommitPruneResult> {
1230
+ validateCommitPruneQuery(query);
1231
+ return this.#exec.transaction(async (client) => {
1232
+ await lockPartitionOn(client, partition);
1233
+ const epoch = await client.query<{ log_epoch: string }>(
1234
+ 'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE',
1235
+ [partition],
1236
+ );
1237
+ if (epoch.rows[0]?.log_epoch !== query.logEpoch)
1238
+ throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
1239
+ const previous = await client.query<{ horizon_seq: unknown }>(
1240
+ 'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
1241
+ [partition],
1242
+ );
1243
+ const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
1244
+ const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
1245
+ await client.query(
1246
+ 'UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1',
1247
+ [partition, horizonSeq],
1248
+ );
1249
+ const removed = await client.query(
1250
+ 'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
1251
+ [partition, horizonSeq],
1252
+ );
1253
+ await client.query(
1254
+ 'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
1255
+ [partition, horizonSeq],
1256
+ );
1257
+ await client.query(
1258
+ 'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
1259
+ [partition, horizonSeq],
1260
+ );
1261
+ return {
1262
+ previousHorizonSeq,
1263
+ horizonSeq,
1264
+ removedCommits: removed.rowCount,
1265
+ };
1266
+ });
1210
1267
  }
1211
1268
 
1212
1269
  async getCommitSeqBefore(
@@ -1630,6 +1687,49 @@ export class PostgresServerStorage implements ServerStorage {
1630
1687
  );
1631
1688
  }
1632
1689
 
1690
+ async advanceClientCursor(
1691
+ partition: string,
1692
+ clientId: string,
1693
+ actorId: string,
1694
+ logEpoch: string,
1695
+ cursor: number,
1696
+ updatedAtMs: number,
1697
+ ): Promise<void> {
1698
+ await this.#exec.query(
1699
+ `UPDATE sync_clients
1700
+ SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
1701
+ WHERE partition=$3 AND client_id=$4 AND actor_id=$5
1702
+ AND EXISTS (SELECT 1 FROM sync_partition_registry
1703
+ WHERE partition=sync_clients.partition AND log_epoch=$6)`,
1704
+ [cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
1705
+ );
1706
+ }
1707
+
1708
+ async updateClientCursor(
1709
+ partition: string,
1710
+ clientId: string,
1711
+ cursor: number,
1712
+ updatedAtMs: number,
1713
+ ): Promise<void> {
1714
+ await this.#exec.query(
1715
+ `UPDATE sync_clients
1716
+ SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
1717
+ WHERE partition=$1 AND client_id=$2`,
1718
+ [partition, clientId, cursor, updatedAtMs],
1719
+ );
1720
+ }
1721
+
1722
+ async getActiveClientCursorFloor(
1723
+ partition: string,
1724
+ cutoffMs: number,
1725
+ ): Promise<number | null> {
1726
+ const { rows } = await this.#exec.query<{ cursor: unknown }>(
1727
+ 'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
1728
+ [partition, cutoffMs],
1729
+ );
1730
+ return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
1731
+ }
1732
+
1633
1733
  async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
1634
1734
  const { rows } = await this.#exec.query<{
1635
1735
  client_id: string;
package/src/prune.ts CHANGED
@@ -7,7 +7,20 @@
7
7
  * least the newest `minRetainedCommits` commits are always retained.
8
8
  */
9
9
  import { emitEvent, type SyncularServerEvents } from './events';
10
- import type { ServerStorage } from './storage';
10
+ import type { CommitPruneQuery, ServerStorage } from './storage';
11
+ import { StorageQueryError } from './storage-errors';
12
+
13
+ /** Shared validation for the built-in atomic pruning adapters. */
14
+ export function validateCommitPruneQuery(query: CommitPruneQuery): void {
15
+ if (
16
+ !Number.isSafeInteger(query.throughSeq) ||
17
+ query.throughSeq < 0 ||
18
+ typeof query.logEpoch !== 'string' ||
19
+ query.logEpoch.length === 0
20
+ ) {
21
+ throw new StorageQueryError('sync.storage.invalid_prune_cursor');
22
+ }
23
+ }
11
24
 
12
25
  export interface RetentionPolicy {
13
26
  /** Active window for laggard cursors (default 14 days). */
@@ -37,28 +50,29 @@ export interface PruneOptions {
37
50
  export async function pruneCommitLog(options: PruneOptions): Promise<number> {
38
51
  const { storage, partition, nowMs } = options;
39
52
  const policy = { ...DEFAULT_RETENTION, ...options.retention };
53
+ const logEpoch = await storage.getPartitionLogEpoch(partition);
54
+ if (logEpoch === undefined)
55
+ throw new StorageQueryError('sync.storage.partition_unregistered');
40
56
  const maxSeq = await storage.getMaxCommitSeq(partition);
41
- const cursors = await storage.listClientCursors(partition);
42
- const activeCursors = cursors
43
- .filter((c) => c.updatedAtMs >= nowMs - policy.activeWindowMs)
44
- .map((c) => c.cursor);
45
57
  const cursorFloor =
46
- activeCursors.length > 0
47
- ? Math.min(...activeCursors)
48
- : Number.MAX_SAFE_INTEGER;
58
+ (await storage.getActiveClientCursorFloor(
59
+ partition,
60
+ nowMs - policy.activeWindowMs,
61
+ )) ?? Number.MAX_SAFE_INTEGER;
49
62
  const forcedSeq = await storage.getCommitSeqBefore(
50
63
  partition,
51
64
  nowMs - policy.ageForceMs,
52
65
  );
53
66
  const retainFloor = maxSeq - policy.minRetainedCommits;
54
67
  const target = Math.min(Math.max(cursorFloor, forcedSeq), retainFloor);
55
- const current = await storage.getHorizonSeq(partition);
56
- const horizon = Math.max(current, Math.max(0, target));
57
- let removedCommits = 0;
58
- if (horizon > current) {
59
- await storage.setHorizonSeq(partition, horizon);
60
- removedCommits = await storage.pruneCommitsThrough(partition, horizon);
61
- }
68
+ const {
69
+ previousHorizonSeq: current,
70
+ horizonSeq: horizon,
71
+ removedCommits,
72
+ } = await storage.pruneCommitsThrough(partition, {
73
+ logEpoch,
74
+ throughSeq: Math.max(0, target),
75
+ });
62
76
  const events = options.events;
63
77
  if (events !== undefined) {
64
78
  emitEvent(events, {
package/src/pull.ts CHANGED
@@ -17,10 +17,16 @@ import { clockOf, limitsOf } from './context';
17
17
  import type { PullSegmentSummary } from './events';
18
18
  import type { CompiledSchema, CompiledTable } from './schema';
19
19
  import { scopeDigest } from './scopes';
20
- import type { SegmentRecord } from './segment-store';
20
+ import type { SegmentRecord, SegmentStore } from './segment-store';
21
21
  import { issueSegmentUrl } from './signed-url';
22
22
  import type { SqliteImageBuilder } from './sqlite-image';
23
- import type { StoredCommit, StoredRow } from './storage';
23
+ import type { ServerStorage, StoredCommit, StoredRow } from './storage';
24
+
25
+ // One artifact build per owning storage pair and complete immutable identity.
26
+ const imageBuilds = new WeakMap<
27
+ ServerStorage,
28
+ WeakMap<SegmentStore, Map<string, Promise<SegmentRecord>>>
29
+ >();
24
30
 
25
31
  /**
26
32
  * Resolve the §5.3 image builder: the host-injected one if present, else the
@@ -260,57 +266,89 @@ async function* sqliteImageSegment(
260
266
  // support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
261
267
  const buildImage = await resolveImageBuilder(ctx);
262
268
  if (buildImage === undefined) return false;
263
- // The probe rows are the snapshot's first page — keep them and scan on
264
- // from the probe's cursor instead of re-reading the whole prefix (the
265
- // scan is keyset-ordered by rowId, so the concatenation is exactly the
266
- // rows a single full scan would return).
267
- const rows: StoredRow[] = [...probe];
268
- let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
269
- for (;;) {
270
- const scanned: StoredRow[] = await storage.scanRows(partition, {
271
- table: plan.table.name,
272
- scopeFilter: plan.effective,
273
- afterRowId,
274
- limit: 50_000,
275
- });
276
- rows.push(...scanned);
277
- const last = scanned[scanned.length - 1];
278
- if (scanned.length < 50_000 || last === undefined) break;
279
- afterRowId = last.rowId;
269
+ let stores = imageBuilds.get(storage);
270
+ if (stores === undefined) {
271
+ stores = new WeakMap();
272
+ imageBuilds.set(storage, stores);
273
+ }
274
+ let builds = stores.get(segments);
275
+ if (builds === undefined) {
276
+ builds = new Map();
277
+ stores.set(segments, builds);
280
278
  }
281
- const bytes = buildImage({
282
- table: plan.table,
279
+ const identity = {
280
+ partition,
281
+ logEpoch,
282
+ table: plan.table.name,
283
283
  schemaVersion: schema.version,
284
- asOfCommitSeq: asOf,
284
+ mediaType: 'sqlite' as const,
285
285
  scopeDigest: digest,
286
- rows,
287
- });
288
- const record = await segments.put(
289
- {
290
- partition,
291
- logEpoch,
292
- table: plan.table.name,
293
- schemaVersion: schema.version,
294
- mediaType: 'sqlite',
295
- scopeDigest: digest,
296
- asOfCommitSeq: asOf,
297
- rowCount: rows.length,
298
- rowCursor: null,
299
- nextRowCursor: null,
300
- },
301
- bytes,
302
- now,
303
- );
286
+ asOfCommitSeq: asOf,
287
+ };
288
+ const key = JSON.stringify(identity);
289
+ let building = builds.get(key);
290
+ let builtHere = false;
291
+ if (building === undefined) {
292
+ building = (async () => {
293
+ // Another request can finish while this one's eligibility probe awaits.
294
+ const cached = await segments.find(identity, clockOf(ctx)());
295
+ if (cached !== undefined) return cached;
296
+ builtHere = true;
297
+ let rowCount = 0;
298
+ const bytes = await buildImage({
299
+ table: plan.table,
300
+ schemaVersion: schema.version,
301
+ asOfCommitSeq: asOf,
302
+ scopeDigest: digest,
303
+ rowBatches: (async function* () {
304
+ rowCount += probe.length;
305
+ yield probe;
306
+ let afterRowId = probe[probe.length - 1]!.rowId;
307
+ for (;;) {
308
+ const rows = await storage.scanRows(partition, {
309
+ table: plan.table.name,
310
+ scopeFilter: plan.effective,
311
+ afterRowId,
312
+ limit: 5_000,
313
+ });
314
+ rowCount += rows.length;
315
+ yield rows;
316
+ const last = rows[rows.length - 1];
317
+ if (rows.length < 5_000 || last === undefined) break;
318
+ afterRowId = last.rowId;
319
+ }
320
+ })(),
321
+ });
322
+ return segments.put(
323
+ { ...identity, rowCount, rowCursor: null, nextRowCursor: null },
324
+ bytes,
325
+ clockOf(ctx)(),
326
+ );
327
+ })();
328
+ builds.set(key, building);
329
+ }
330
+ let record: SegmentRecord;
331
+ try {
332
+ record = await building;
333
+ } finally {
334
+ if (builds.get(key) === building) builds.delete(key);
335
+ }
304
336
  trace?.segments.push({
305
337
  mediaType: 'sqlite',
306
338
  delivery: 'ref',
307
- origin: 'built',
339
+ origin: builtHere ? 'built' : 'reused',
308
340
  bytes: record.byteLength,
309
341
  rows: record.rowCount,
310
342
  });
311
343
  yield segmentRefFrame(
312
344
  record,
313
- await signedUrlFields(ctx, limits, record.segmentId, digest, now),
345
+ await signedUrlFields(
346
+ ctx,
347
+ limits,
348
+ record.segmentId,
349
+ digest,
350
+ clockOf(ctx)(),
351
+ ),
314
352
  );
315
353
  return true;
316
354
  }
@@ -463,6 +501,28 @@ export async function* subscriptionSection(
463
501
 
464
502
  const token = parseBootstrapToken(sub.bootstrapState, sub.table);
465
503
 
504
+ let commits: StoredCommit[] = [];
505
+ if (
506
+ token === undefined &&
507
+ sub.cursor >= 0 &&
508
+ sub.cursor >= horizonSeq &&
509
+ sub.cursor <= maxSeq
510
+ ) {
511
+ commits = await ctx.storage.readCommitWindow(ctx.partition, {
512
+ table: sub.table,
513
+ scopeFilter: plan.effective,
514
+ afterSeq: sub.cursor,
515
+ throughSeq: maxSeq,
516
+ limitChanges: limits.limitCommits + 1,
517
+ });
518
+ // Validate continuity before committing to an active section. A prune
519
+ // during a paged read can otherwise make an incomplete window look empty.
520
+ horizonSeq = Math.max(
521
+ horizonSeq,
522
+ await ctx.storage.getHorizonSeq(ctx.partition),
523
+ );
524
+ }
525
+
466
526
  // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
467
527
  // cannot compute deltas — answer `reset` and echo the cursor.
468
528
  if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
@@ -536,13 +596,6 @@ export async function* subscriptionSection(
536
596
  effectiveScopes: plan.effective,
537
597
  bootstrap: false,
538
598
  };
539
- const commits = await ctx.storage.readCommitWindow(ctx.partition, {
540
- table: sub.table,
541
- scopeFilter: plan.effective,
542
- afterSeq: sub.cursor,
543
- throughSeq: maxSeq,
544
- limitChanges: limits.limitCommits + 1,
545
- });
546
599
  let delivered = 0;
547
600
  let deliveredCommits = 0;
548
601
  let lastDeliveredSeq = sub.cursor;
package/src/push.ts CHANGED
@@ -185,7 +185,7 @@ async function runValidator(
185
185
  return errorRecord(
186
186
  opIndex,
187
187
  'sync.constraint_violation',
188
- `write validator for table ${JSON.stringify(table.name)} threw: ${error instanceof Error ? error.message : String(error)}`,
188
+ 'write validator failed',
189
189
  );
190
190
  }
191
191
  return undefined;
@@ -220,7 +220,7 @@ async function mergeCrdtColumns(
220
220
  return errorRecord(
221
221
  opIndex,
222
222
  'sync.crdt_merge_failed',
223
- `no CRDT merger registered for crdtType ${JSON.stringify(crdtType)} (§5.10.2)`,
223
+ 'no CRDT merger registered',
224
224
  );
225
225
  }
226
226
  const storedRaw = storedValues?.[index];
@@ -228,11 +228,11 @@ async function mergeCrdtColumns(
228
228
  let merged: Uint8Array;
229
229
  try {
230
230
  merged = await merger(stored, incoming);
231
- } catch (error) {
231
+ } catch {
232
232
  return errorRecord(
233
233
  opIndex,
234
234
  'sync.crdt_merge_failed',
235
- `CRDT merger for ${JSON.stringify(crdtType)} threw: ${error instanceof Error ? error.message : String(error)}`,
235
+ 'CRDT merger failed',
236
236
  );
237
237
  }
238
238
  values[index] = merged;
@@ -732,7 +732,7 @@ async function runCommitValidator(
732
732
  return errorRecord(
733
733
  operations[0]?.opIndex ?? 0,
734
734
  'sync.constraint_violation',
735
- `whole-commit validator threw: ${error instanceof Error ? error.message : String(error)}`,
735
+ 'whole-commit validator failed',
736
736
  );
737
737
  }
738
738
  return undefined;