@syncular/server 0.26.4 → 0.26.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/content-encoding.d.ts +17 -0
  2. package/dist/content-encoding.js +22 -0
  3. package/dist/context.d.ts +9 -0
  4. package/dist/d1-storage.d.ts +46 -0
  5. package/dist/d1-storage.js +12 -1
  6. package/dist/errors.d.ts +19 -0
  7. package/dist/errors.js +29 -0
  8. package/dist/handler.js +1 -1
  9. package/dist/index.d.ts +1 -1
  10. package/dist/index.js +1 -1
  11. package/dist/operations-realtime.js +4 -1
  12. package/dist/operations.js +6 -1
  13. package/dist/postgres-storage.d.ts +42 -0
  14. package/dist/postgres-storage.js +110 -45
  15. package/dist/pull.d.ts +46 -6
  16. package/dist/pull.js +129 -34
  17. package/dist/push.js +19 -2
  18. package/dist/realtime.d.ts +3 -0
  19. package/dist/realtime.js +32 -26
  20. package/dist/relational-rows.d.ts +19 -0
  21. package/dist/relational-rows.js +38 -0
  22. package/dist/s3-segment-store.d.ts +3 -1
  23. package/dist/s3-segment-store.js +16 -0
  24. package/dist/segment-download.d.ts +16 -0
  25. package/dist/segment-download.js +71 -21
  26. package/dist/segment-store.d.ts +16 -0
  27. package/dist/segment-store.js +19 -0
  28. package/dist/sqlite-storage.d.ts +5 -0
  29. package/dist/sqlite-storage.js +18 -1
  30. package/dist/storage.d.ts +20 -0
  31. package/package.json +2 -2
  32. package/src/content-encoding.ts +35 -0
  33. package/src/context.ts +9 -0
  34. package/src/d1-storage.ts +20 -1
  35. package/src/errors.ts +56 -0
  36. package/src/handler.ts +2 -0
  37. package/src/index.ts +1 -0
  38. package/src/operations-realtime.ts +4 -1
  39. package/src/operations.ts +6 -1
  40. package/src/postgres-storage.ts +145 -60
  41. package/src/pull.ts +212 -39
  42. package/src/push.ts +18 -2
  43. package/src/realtime.ts +45 -32
  44. package/src/relational-rows.ts +43 -0
  45. package/src/s3-segment-store.ts +15 -0
  46. package/src/segment-download.ts +129 -21
  47. package/src/segment-store.ts +36 -0
  48. package/src/sqlite-storage.ts +27 -0
  49. package/src/storage.ts +21 -0
@@ -40,7 +40,7 @@ import { serveNotReadyError } from './readiness';
40
40
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
41
41
  * does not tolerate). Cross-partition pushes never contend.
42
42
  */
43
- import type { PushOperationResult } from '@syncular/core';
43
+ import type { PushOperationResult, ScopeMap } from '@syncular/core';
44
44
  import {
45
45
  bindAuthoritativePartition,
46
46
  type BoundAuthoritativeQuery,
@@ -59,6 +59,7 @@ import {
59
59
  assertPhysicalColumns,
60
60
  assertStoredLayouts,
61
61
  COMMIT_WINDOW_BATCH_SQL,
62
+ LATEST_SCOPED_CHANGE_BATCH_SQL,
62
63
  deleteRowSql,
63
64
  dropTableDdl,
64
65
  indexRowPageStatement,
@@ -76,6 +77,7 @@ import {
76
77
  schemaDdl,
77
78
  selectRowScopesSql,
78
79
  selectRowSql,
80
+ prefetchRowsSql,
79
81
  selectRowsForRewriteSql,
80
82
  upsertSql,
81
83
  upsertValues,
@@ -697,21 +699,31 @@ async function writeRowOn(
697
699
  partition: string,
698
700
  row: StoredRow,
699
701
  ): Promise<void> {
702
+ // One statement per row write: the upsert, and the scope index synced to
703
+ // the row's scope map (entries outside it deleted, missing ones inserted).
704
+ // No index entry is touched twice, so the writable CTEs cannot conflict.
705
+ // The scopes bind as text before the JSONB cast (see `appendCommit`).
706
+ const params = upsertValues(compiled, partition, row, 'postgres');
707
+ const n = params.length;
700
708
  await q.query(
701
- upsertSql(compiled, 'postgres'),
702
- upsertValues(compiled, partition, row, 'postgres'),
703
- );
704
- await q.query(
705
- 'DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$2 AND row_id=$3',
706
- [partition, compiled.name, row.rowId],
709
+ `WITH upserted AS (${upsertSql(compiled, 'postgres')}),
710
+ scope AS (SELECT key, value FROM jsonb_each_text($${n + 1}::text::jsonb)),
711
+ removed AS (
712
+ DELETE FROM sync_row_scopes
713
+ WHERE partition=$${n + 2} AND tbl=$${n + 3} AND row_id=$${n + 4}
714
+ AND (var, value) NOT IN (SELECT key, value FROM scope)
715
+ )
716
+ INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
717
+ SELECT $${n + 2}, $${n + 3}, key, value, $${n + 4} FROM scope
718
+ ON CONFLICT DO NOTHING`,
719
+ [
720
+ ...params,
721
+ JSON.stringify(row.scopes),
722
+ partition,
723
+ compiled.name,
724
+ row.rowId,
725
+ ],
707
726
  );
708
- for (const [variable, value] of Object.entries(row.scopes)) {
709
- await q.query(
710
- `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
711
- VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
712
- [partition, compiled.name, variable, value, row.rowId],
713
- );
714
- }
715
727
  }
716
728
 
717
729
  async function getPushResultOn(
@@ -770,6 +782,13 @@ class PostgresTransaction implements StorageTransaction {
770
782
  #resolveTable: (name: string) => CompiledTable;
771
783
  #open = true;
772
784
  #pushApplySavepoint = false;
785
+ /**
786
+ * `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
787
+ * (or tombstone) read as absent. A write in this transaction drops the
788
+ * key, so the next read goes to the database again.
789
+ */
790
+ readonly #prefetchedRows = new Map<string, StoredRow | null>();
791
+ readonly #prefetchedTombstones = new Map<string, number | null>();
773
792
  /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
774
793
  #resolve: () => void;
775
794
  #reject: (error: unknown) => void;
@@ -814,6 +833,10 @@ class PostgresTransaction implements StorageTransaction {
814
833
 
815
834
  getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
816
835
  this.#assertOpen();
836
+ const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
837
+ if (prefetched !== undefined) {
838
+ return Promise.resolve(prefetched ?? undefined);
839
+ }
817
840
  return getRowOn(
818
841
  this.#client,
819
842
  this.#resolveTable(table),
@@ -822,11 +845,47 @@ class PostgresTransaction implements StorageTransaction {
822
845
  );
823
846
  }
824
847
 
848
+ async prefetchRows(table: string, rowIds: readonly string[]): Promise<void> {
849
+ this.#assertOpen();
850
+ const ids = [...new Set(rowIds)].filter(
851
+ (rowId) => !this.#prefetchedRows.has(`${table}\u0000${rowId}`),
852
+ );
853
+ if (ids.length === 0) return;
854
+ const { rows } = await this.#client.query<
855
+ Omit<RowRecord, 'row_id'> & {
856
+ lookup_id: string;
857
+ row_id: string | null;
858
+ tombstone_seq: unknown;
859
+ }
860
+ >(prefetchRowsSql(this.#resolveTable(table)), [
861
+ this.#partition,
862
+ JSON.stringify(ids),
863
+ table,
864
+ ]);
865
+ for (const record of rows) {
866
+ const key = `${table}\u0000${record.lookup_id}`;
867
+ this.#prefetchedRows.set(
868
+ key,
869
+ record.row_id === null
870
+ ? null
871
+ : toStoredRow({ ...record, row_id: record.row_id }),
872
+ );
873
+ this.#prefetchedTombstones.set(
874
+ key,
875
+ record.tombstone_seq === null || record.tombstone_seq === undefined
876
+ ? null
877
+ : asNumber(record.tombstone_seq),
878
+ );
879
+ }
880
+ }
881
+
825
882
  async getTombstoneSeq(
826
883
  table: string,
827
884
  rowId: string,
828
885
  ): Promise<number | undefined> {
829
886
  this.#assertOpen();
887
+ const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
888
+ if (prefetched !== undefined) return prefetched ?? undefined;
830
889
  const { rows } = await this.#client.query<{ commit_seq: unknown }>(
831
890
  'SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
832
891
  [this.#partition, table, rowId],
@@ -837,6 +896,7 @@ class PostgresTransaction implements StorageTransaction {
837
896
 
838
897
  async clearTombstone(table: string, rowId: string): Promise<void> {
839
898
  this.#assertOpen();
899
+ this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
840
900
  await this.#client.query(
841
901
  'DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
842
902
  [this.#partition, table, rowId],
@@ -976,6 +1036,7 @@ class PostgresTransaction implements StorageTransaction {
976
1036
  context?: { readonly opIndex: number },
977
1037
  ): Promise<void> {
978
1038
  this.#assertOpen();
1039
+ this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
979
1040
  try {
980
1041
  await writeRowOn(
981
1042
  this.#client,
@@ -993,18 +1054,16 @@ class PostgresTransaction implements StorageTransaction {
993
1054
 
994
1055
  async deleteRow(table: string, rowId: string): Promise<void> {
995
1056
  this.#assertOpen();
1057
+ this.#prefetchedRows.delete(`${table}\u0000${rowId}`);
1058
+ // One statement: the row, its scope-index entries, and (§5.9.4: a
1059
+ // deleted row references no blobs) its blob references.
996
1060
  await this.#client.query(
997
- deleteRowSql(this.#resolveTable(table), 'postgres'),
998
- [this.#partition, rowId],
999
- );
1000
- await this.#client.query(
1001
- 'DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$2 AND row_id=$3',
1002
- [this.#partition, table, rowId],
1003
- );
1004
- // §5.9.4: a deleted row references no blobs.
1005
- await this.#client.query(
1006
- 'DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$2 AND row_id=$3',
1007
- [this.#partition, table, rowId],
1061
+ `WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
1062
+ unscoped AS (
1063
+ DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
1064
+ )
1065
+ DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`,
1066
+ [this.#partition, rowId, table],
1008
1067
  );
1009
1068
  }
1010
1069
 
@@ -1030,6 +1089,8 @@ class PostgresTransaction implements StorageTransaction {
1030
1089
 
1031
1090
  async appendCommit(commit: NewCommit): Promise<number> {
1032
1091
  this.#assertOpen();
1092
+ // The commit writes its delete tombstones below.
1093
+ this.#prefetchedTombstones.clear();
1033
1094
  const q = this.#client;
1034
1095
  const p = this.#partition;
1035
1096
  // Allocate the next dense commitSeq under a per-partition row lock: the
@@ -1055,42 +1116,46 @@ class PostgresTransaction implements StorageTransaction {
1055
1116
  ],
1056
1117
  );
1057
1118
  const commitSeq = asNumber(rows[0]?.commit_seq);
1058
- for (let idx = 0; idx < commit.changes.length; idx++) {
1059
- const change = commit.changes[idx];
1060
- if (change === undefined) continue;
1061
- // Bind serialized scopes as text before parsing JSONB. Drivers that
1062
- // encode JSONB parameters would otherwise store this string as a scalar.
1063
- // One statement per change: scopes AND the §5 delete tombstone ride the
1064
- // same writable-CTE chain.
1065
- await q.query(
1066
- `WITH inserted AS (
1067
- INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1068
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8::text::jsonb,$9)
1069
- RETURNING partition, tbl, row_id, op, commit_seq, scopes
1070
- ),
1071
- scoped AS (
1072
- INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
1073
- SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
1074
- FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
1075
- ON CONFLICT DO NOTHING
1076
- )
1077
- INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
1078
- SELECT partition, tbl, row_id, commit_seq FROM inserted WHERE op = 2
1079
- ON CONFLICT (partition, tbl, row_id)
1080
- DO UPDATE SET commit_seq = excluded.commit_seq`,
1081
- [
1082
- p,
1083
- commitSeq,
1084
- idx,
1085
- change.table,
1086
- change.rowId,
1087
- change.op === 'upsert' ? 1 : 2,
1088
- change.rowVersion ?? null,
1089
- JSON.stringify(change.scopes),
1090
- change.payload ?? null,
1091
- ],
1119
+ if (commit.changes.length === 0) return commitSeq;
1120
+ // One statement for the whole commit: its changes, their inverted scope
1121
+ // entries, and the §5 delete tombstones ride one writable-CTE chain.
1122
+ // Serialized scopes bind as text before parsing JSONB; drivers that
1123
+ // encode JSONB parameters would otherwise store the string as a scalar.
1124
+ // A commit can delete one row twice, so the tombstone insert keeps one
1125
+ // row per key (every candidate carries the same commit_seq).
1126
+ const params: unknown[] = [p, commitSeq];
1127
+ const values = commit.changes.map((change, idx) => {
1128
+ const base = params.length;
1129
+ params.push(
1130
+ idx,
1131
+ change.table,
1132
+ change.rowId,
1133
+ change.op === 'upsert' ? 1 : 2,
1134
+ change.rowVersion ?? null,
1135
+ JSON.stringify(change.scopes),
1136
+ change.payload ?? null,
1092
1137
  );
1093
- }
1138
+ return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
1139
+ });
1140
+ await q.query(
1141
+ `WITH inserted AS (
1142
+ INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1143
+ VALUES ${values.join(',')}
1144
+ RETURNING partition, tbl, row_id, op, commit_seq, scopes
1145
+ ),
1146
+ scoped AS (
1147
+ INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
1148
+ SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
1149
+ FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
1150
+ ON CONFLICT DO NOTHING
1151
+ )
1152
+ INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
1153
+ SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
1154
+ FROM inserted WHERE op = 2
1155
+ ON CONFLICT (partition, tbl, row_id)
1156
+ DO UPDATE SET commit_seq = excluded.commit_seq`,
1157
+ params,
1158
+ );
1094
1159
  return commitSeq;
1095
1160
  }
1096
1161
 
@@ -2142,6 +2207,26 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
2142
2207
  };
2143
2208
  }
2144
2209
 
2210
+ async latestScopedChangeSeq(
2211
+ partition: string,
2212
+ query: { readonly table: string; readonly scopeFilter: ScopeMap },
2213
+ ): Promise<number> {
2214
+ const firstVariable = Object.keys(query.scopeFilter).sort()[0];
2215
+ const values =
2216
+ firstVariable === undefined
2217
+ ? []
2218
+ : (query.scopeFilter[firstVariable] ?? []);
2219
+ if (firstVariable === undefined || values.length === 0) return 0;
2220
+ // Concurrent reads of one table in a pull share one statement.
2221
+ const [row] = await this.#batchedPage<{ seq: unknown }>(
2222
+ LATEST_SCOPED_CHANGE_BATCH_SQL,
2223
+ partition,
2224
+ query.table,
2225
+ { var: firstVariable, vals: values },
2226
+ );
2227
+ return row === undefined ? 0 : asNumber(row.seq);
2228
+ }
2229
+
2145
2230
  async readCommitWindow(
2146
2231
  partition: string,
2147
2232
  query: CommitWindowQuery,
package/src/pull.ts CHANGED
@@ -9,13 +9,18 @@ import {
9
9
  type PullHeaderFrame,
10
10
  type ResponseFrame,
11
11
  type ScopeMap,
12
+ type SegmentRefFrame,
12
13
  type SegmentRow,
13
14
  type SubscriptionFrame,
14
15
  } from '@syncular/core';
15
- import type { SyncRequestContext } from './context';
16
+ import type { SyncRequestContext, SyncServerConfig } from './context';
16
17
  import { clockOf, limitsOf } from './context';
17
18
  import type { PullSegmentSummary } from './events';
18
- import type { CompiledSchema, CompiledTable } from './schema';
19
+ import {
20
+ type CompiledSchema,
21
+ type CompiledTable,
22
+ compileSchema,
23
+ } from './schema';
19
24
  import { scopeDigest } from './scopes';
20
25
  import type { SegmentRecord, SegmentStore } from './segment-store';
21
26
  import { issueSegmentUrl } from './signed-url';
@@ -244,21 +249,11 @@ async function* sqliteImageSegment(
244
249
  digest: string,
245
250
  trace: PullSectionTrace | undefined,
246
251
  logEpoch: string,
252
+ existing: SegmentRecord | undefined,
253
+ probe: StoredRow[],
247
254
  ): AsyncGenerator<ResponseFrame, boolean> {
248
255
  const { storage, segments, partition } = ctx;
249
256
  const now = clockOf(ctx)();
250
- const existing = await segments.find(
251
- {
252
- partition,
253
- logEpoch,
254
- table: plan.table.name,
255
- schemaVersion: schema.version,
256
- mediaType: 'sqlite',
257
- scopeDigest: digest,
258
- asOfCommitSeq: asOf,
259
- },
260
- now,
261
- );
262
257
  if (existing !== undefined) {
263
258
  trace?.segments.push({
264
259
  mediaType: 'sqlite',
@@ -276,12 +271,7 @@ async function* sqliteImageSegment(
276
271
  }
277
272
  // Eligibility probe (§5.3): image only when the snapshot exceeds one
278
273
  // rows page — smaller tables stay on the (typically inline) rows lane.
279
- const probe = await storage.scanRows(partition, {
280
- table: plan.table.name,
281
- scopeFilter: plan.effective,
282
- afterRowId: null,
283
- limit: limits.limitSnapshotRows + 1,
284
- });
274
+ // The probe is the rows lane's first page, read by `prepareSections`.
285
275
  if (probe.length <= limits.limitSnapshotRows) return false;
286
276
  // §5.3: building an image needs a SQLite engine. The host injects the
287
277
  // builder through `sqliteImageBuilder`; when omitted we default to the
@@ -389,7 +379,8 @@ async function* bootstrapSegments(
389
379
  startRowCursor: string | null,
390
380
  trace: PullSectionTrace | undefined,
391
381
  logEpoch: string,
392
- firstPage: StoredRow[] | undefined,
382
+ firstPage: StoredRow[],
383
+ image: SegmentRecord | undefined,
393
384
  ): AsyncGenerator<
394
385
  ResponseFrame,
395
386
  { complete: boolean; rowCursor: string | null }
@@ -408,13 +399,15 @@ async function* bootstrapSegments(
408
399
  digest,
409
400
  trace,
410
401
  logEpoch,
402
+ image,
403
+ firstPage,
411
404
  );
412
405
  if (imaged) return { complete: true, rowCursor: null };
413
406
  }
414
407
  let rowCursor = startRowCursor;
415
408
  for (let page = 0; page < limits.maxSnapshotPages; page++) {
416
409
  const scanned =
417
- page === 0 && firstPage !== undefined
410
+ page === 0
418
411
  ? firstPage
419
412
  : await storage.scanRows(partition, {
420
413
  table: plan.table.name,
@@ -492,27 +485,45 @@ async function* bootstrapSegments(
492
485
  /**
493
486
  * The storage reads a section needs before its first frame. A pull starts
494
487
  * them for every subscription before awaiting any, so a storage that batches
495
- * concurrent page reads (PostgreSQL) answers all subscriptions of one table
496
- * with one statement, and the horizon re-check runs once per pull.
488
+ * concurrent page reads (PostgreSQL, D1) answers all subscriptions of one
489
+ * table with one statement, and the horizon re-check runs once per pull.
497
490
  */
498
491
  export interface PreparedSection {
499
492
  /** §4.5 window read for an incremental cursor; empty when none was read. */
500
493
  readonly commits: StoredCommit[];
501
494
  /** Horizon to judge the cursor against, re-read after any window read. */
502
495
  readonly horizonSeq: number;
503
- /** First rows-lane snapshot page, read when the sqlite lane is excluded. */
496
+ /**
497
+ * A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
498
+ * resume cursor), which is also the §5.3 image eligibility probe.
499
+ */
504
500
  readonly firstPage: StoredRow[] | undefined;
501
+ /** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
502
+ readonly pin: number | undefined;
503
+ /** §5.3 stored image for a sqlite-lane bootstrap's reuse key, if any. */
504
+ readonly image: SegmentRecord | undefined;
505
+ }
506
+
507
+ interface PendingSection {
508
+ readonly commits?: Promise<StoredCommit[]>;
509
+ readonly firstPage?: Promise<StoredRow[]>;
510
+ readonly lane?: Promise<{
511
+ readonly pin: number | undefined;
512
+ readonly image: SegmentRecord | undefined;
513
+ }>;
505
514
  }
506
515
 
507
516
  /** Run the pre-frame storage reads of every subscription concurrently. */
508
517
  export async function prepareSections(
509
518
  ctx: SyncRequestContext,
519
+ schema: CompiledSchema,
510
520
  limits: PullLimits,
511
521
  plans: readonly SubscriptionPlan[],
512
522
  maxSeq: number,
513
523
  horizonSeq: number,
524
+ logEpoch: string,
514
525
  ): Promise<PreparedSection[]> {
515
- const reads = plans.map((plan) => {
526
+ const reads = plans.map((plan): PendingSection => {
516
527
  const sub = plan.frame;
517
528
  if (plan.status === 'revoked') return {};
518
529
  const token = parseBootstrapToken(sub.bootstrapState, sub.table);
@@ -537,24 +548,55 @@ export async function prepareSections(
537
548
  return {};
538
549
  }
539
550
  // Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
540
- const startCursor =
551
+ const resume =
541
552
  token !== undefined && token.asOfCommitSeq >= horizonSeq
542
- ? token.rowCursor
543
- : null;
544
- if (sqliteLaneEligible(limits, plan, startCursor)) return {};
545
- return {
546
- firstPage: ctx.storage.scanRows(ctx.partition, {
547
- table: plan.table.name,
548
- scopeFilter: plan.effective,
549
- afterRowId: startCursor,
550
- limit: limits.limitSnapshotRows + 1,
551
- }),
552
- };
553
+ ? token
554
+ : undefined;
555
+ const startCursor = resume?.rowCursor ?? null;
556
+ const firstPage = ctx.storage.scanRows(ctx.partition, {
557
+ table: plan.table.name,
558
+ scopeFilter: plan.effective,
559
+ afterRowId: startCursor,
560
+ limit: limits.limitSnapshotRows + 1,
561
+ });
562
+ if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
563
+ // §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
564
+ // scope (never below the horizon, never above maxCommitSeq), so a stored
565
+ // image stays the reuse key's match until this scope changes (§5.3).
566
+ const pinned =
567
+ resume !== undefined
568
+ ? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
569
+ : ctx.storage
570
+ .latestScopedChangeSeq(ctx.partition, {
571
+ table: plan.table.name,
572
+ scopeFilter: plan.effective,
573
+ })
574
+ .then((latest) => {
575
+ const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
576
+ return { asOf: pin, pin };
577
+ });
578
+ const lane = pinned.then(async ({ asOf, pin }) => ({
579
+ pin,
580
+ image: await ctx.segments.find(
581
+ {
582
+ partition: ctx.partition,
583
+ logEpoch,
584
+ table: plan.table.name,
585
+ schemaVersion: schema.version,
586
+ mediaType: 'sqlite',
587
+ scopeDigest: await scopeDigest(plan.effective),
588
+ asOfCommitSeq: asOf,
589
+ },
590
+ clockOf(ctx)(),
591
+ ),
592
+ }));
593
+ return { firstPage, lane };
553
594
  });
554
595
  const settled = await Promise.all(
555
596
  reads.map(async (read) => ({
556
597
  commits: read.commits === undefined ? undefined : await read.commits,
557
598
  firstPage: await read.firstPage,
599
+ lane: await read.lane,
558
600
  })),
559
601
  );
560
602
  // Validate continuity before committing to an active section. A prune
@@ -566,6 +608,8 @@ export async function prepareSections(
566
608
  commits: read.commits ?? [],
567
609
  horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
568
610
  firstPage: read.firstPage,
611
+ pin: read.lane?.pin,
612
+ image: read.lane?.image,
569
613
  }));
570
614
  }
571
615
 
@@ -583,12 +627,17 @@ export async function* subscriptionSection(
583
627
  trace?: PullSectionTrace,
584
628
  logEpoch?: string,
585
629
  ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
630
+ if (logEpoch === undefined || logEpoch.length === 0) {
631
+ throw new Error('subscriptionSection requires a non-empty log epoch');
632
+ }
586
633
  const [prepared] = await prepareSections(
587
634
  ctx,
635
+ schema,
588
636
  limits,
589
637
  [plan],
590
638
  maxSeq,
591
639
  horizonSeq,
640
+ logEpoch,
592
641
  );
593
642
  if (prepared === undefined) throw new Error('prepareSections lost a plan');
594
643
  return yield* preparedSection(
@@ -660,8 +709,11 @@ export async function* preparedSection(
660
709
  token !== undefined && token.asOfCommitSeq >= horizonSeq
661
710
  ? token
662
711
  : undefined;
663
- const asOf = resume?.asOfCommitSeq ?? maxSeq;
712
+ const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
664
713
  const startCursor = resume?.rowCursor ?? null;
714
+ if (prepared.firstPage === undefined) {
715
+ throw new Error('prepareSections did not read the bootstrap page');
716
+ }
665
717
  yield {
666
718
  type: 'SUB_START',
667
719
  id: sub.id,
@@ -680,6 +732,7 @@ export async function* preparedSection(
680
732
  trace,
681
733
  logEpoch,
682
734
  prepared.firstPage,
735
+ prepared.image,
683
736
  );
684
737
  if (outcome.complete) {
685
738
  yield { type: 'SUB_END', nextCursor: asOf };
@@ -740,3 +793,123 @@ export async function* preparedSection(
740
793
  yield { type: 'SUB_END', nextCursor };
741
794
  return { nextCursor, active: true };
742
795
  }
796
+
797
+ export interface PublishSqliteImageInput {
798
+ /** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
799
+ readonly config: SyncServerConfig;
800
+ readonly partition: string;
801
+ readonly table: string;
802
+ /**
803
+ * The effective scopes the subscribing clients hold for `table` (§3.2):
804
+ * the reuse key carries their digest, so publish once per distinct scope
805
+ * set that clients subscribe with.
806
+ */
807
+ readonly scopes: ScopeMap;
808
+ }
809
+
810
+ export interface PublishedSqliteImage {
811
+ readonly segmentId: string;
812
+ /** The §4.7 scope pin the image is stored under. */
813
+ readonly asOfCommitSeq: number;
814
+ readonly scopeDigest: string;
815
+ readonly rowCount: number;
816
+ readonly byteLength: number;
817
+ /** `reused` when an unexpired image for the same reuse key already existed. */
818
+ readonly origin: 'built' | 'reused';
819
+ }
820
+
821
+ /**
822
+ * Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
823
+ * up, from a host with a SQLite engine (a Bun process with the production
824
+ * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
825
+ * reuse key, the build), so the stored identity is exactly the one a
826
+ * serving host without a builder finds. The image stays current until a
827
+ * change reaches the scope; its lifetime is the segment store's TTL at
828
+ * publication.
829
+ */
830
+ export async function publishSqliteImage(
831
+ input: PublishSqliteImageInput,
832
+ ): Promise<PublishedSqliteImage> {
833
+ const { config, partition } = input;
834
+ if (config.sqliteImageBuilder === undefined) {
835
+ throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
836
+ }
837
+ const schema = compileSchema(config.schema);
838
+ const table = schema.tables.get(input.table);
839
+ if (table === undefined) {
840
+ throw new Error('publishSqliteImage: the schema has no such table');
841
+ }
842
+ await config.storage.ensureSchema(schema, config.checkpoints);
843
+ const logEpoch = await config.storage.getPartitionLogEpoch(partition);
844
+ if (logEpoch === undefined) {
845
+ throw new Error('publishSqliteImage: the partition has no log epoch');
846
+ }
847
+ const ctx: SyncRequestContext = {
848
+ ...config,
849
+ partition,
850
+ actorId: 'syncular.image-publisher',
851
+ };
852
+ const plan: SubscriptionPlan = {
853
+ frame: {
854
+ type: 'SUBSCRIPTION',
855
+ id: 'image',
856
+ table: input.table,
857
+ scopes: input.scopes,
858
+ cursor: -1,
859
+ },
860
+ table,
861
+ status: 'active',
862
+ effective: input.scopes,
863
+ };
864
+ // One-row pages make every table above one row image-eligible (§5.3).
865
+ const limits: PullLimits = {
866
+ limitCommits: 1,
867
+ limitSnapshotRows: 1,
868
+ maxSnapshotPages: 1,
869
+ accept: ACCEPT_SQLITE,
870
+ };
871
+ const maxSeq = await config.storage.getMaxCommitSeq(partition);
872
+ const horizonSeq = await config.storage.getHorizonSeq(partition);
873
+ const trace: PullSectionTrace = { segments: [] };
874
+ const [prepared] = await prepareSections(
875
+ ctx,
876
+ schema,
877
+ limits,
878
+ [plan],
879
+ maxSeq,
880
+ horizonSeq,
881
+ logEpoch,
882
+ );
883
+ if (prepared === undefined) throw new Error('prepareSections lost a plan');
884
+ let ref: SegmentRefFrame | undefined;
885
+ for await (const frame of preparedSection(
886
+ ctx,
887
+ schema,
888
+ limits,
889
+ plan,
890
+ maxSeq,
891
+ prepared,
892
+ trace,
893
+ logEpoch,
894
+ )) {
895
+ if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
896
+ ref = frame;
897
+ }
898
+ }
899
+ const image = trace.segments.find(
900
+ (segment) => segment.mediaType === 'sqlite',
901
+ );
902
+ if (ref === undefined || image === undefined) {
903
+ throw new Error(
904
+ 'publishSqliteImage: the scope holds at most one row, which pulls serve inline',
905
+ );
906
+ }
907
+ return {
908
+ segmentId: ref.segmentId,
909
+ asOfCommitSeq: ref.asOfCommitSeq,
910
+ scopeDigest: ref.scopeDigest,
911
+ rowCount: ref.rowCount,
912
+ byteLength: ref.byteLength,
913
+ origin: image.origin === 'reused' ? 'reused' : 'built',
914
+ };
915
+ }