@syncular/server 0.15.47 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +48 -5
  2. package/dist/admin.d.ts +1 -5
  3. package/dist/admin.js +3 -12
  4. package/dist/authoritative-query.d.ts +14 -6
  5. package/dist/authoritative-query.js +60 -82
  6. package/dist/blob-handlers.js +4 -1
  7. package/dist/context.d.ts +3 -1
  8. package/dist/context.js +4 -0
  9. package/dist/d1-storage.d.ts +8 -2
  10. package/dist/d1-storage.js +134 -26
  11. package/dist/errors.js +6 -0
  12. package/dist/events.d.ts +2 -1
  13. package/dist/frame-bytes.d.ts +2 -2
  14. package/dist/frame-bytes.js +33 -14
  15. package/dist/handler.js +58 -14
  16. package/dist/index.d.ts +1 -0
  17. package/dist/index.js +1 -0
  18. package/dist/operations.d.ts +2 -0
  19. package/dist/operations.js +25 -5
  20. package/dist/postgres-storage.d.ts +9 -3
  21. package/dist/postgres-storage.js +110 -20
  22. package/dist/prune.d.ts +3 -1
  23. package/dist/prune.js +18 -14
  24. package/dist/pull.d.ts +1 -1
  25. package/dist/pull.js +74 -37
  26. package/dist/push.js +5 -5
  27. package/dist/realtime.d.ts +4 -1
  28. package/dist/realtime.js +33 -9
  29. package/dist/restore.d.ts +13 -0
  30. package/dist/restore.js +13 -0
  31. package/dist/s3-segment-store.js +10 -1
  32. package/dist/seed.js +36 -4
  33. package/dist/segment-download.js +5 -2
  34. package/dist/segment-store.d.ts +3 -0
  35. package/dist/segment-store.js +1 -0
  36. package/dist/sqlite-bun-driver.d.ts +2 -1
  37. package/dist/sqlite-bun-driver.js +5 -2
  38. package/dist/sqlite-bun.js +2 -2
  39. package/dist/sqlite-dialect.d.ts +1 -1
  40. package/dist/sqlite-dialect.js +7 -0
  41. package/dist/sqlite-image.d.ts +3 -3
  42. package/dist/sqlite-image.js +10 -6
  43. package/dist/sqlite-node.js +2 -2
  44. package/dist/sqlite-segment-store.js +14 -5
  45. package/dist/sqlite-storage.d.ts +8 -2
  46. package/dist/sqlite-storage.js +147 -36
  47. package/dist/storage-errors.d.ts +1 -1
  48. package/dist/storage-errors.js +3 -0
  49. package/dist/storage.d.ts +38 -10
  50. package/package.json +2 -2
  51. package/src/admin.ts +7 -15
  52. package/src/authoritative-query.ts +89 -94
  53. package/src/blob-handlers.ts +8 -1
  54. package/src/context.ts +16 -1
  55. package/src/d1-storage.ts +193 -29
  56. package/src/errors.ts +6 -0
  57. package/src/events.ts +2 -1
  58. package/src/frame-bytes.ts +40 -14
  59. package/src/handler.ts +102 -29
  60. package/src/index.ts +1 -0
  61. package/src/operations.ts +37 -4
  62. package/src/postgres-storage.ts +184 -38
  63. package/src/prune.ts +29 -15
  64. package/src/pull.ts +90 -41
  65. package/src/push.ts +5 -5
  66. package/src/realtime.ts +46 -7
  67. package/src/restore.ts +28 -0
  68. package/src/s3-segment-store.ts +10 -1
  69. package/src/seed.ts +46 -4
  70. package/src/segment-download.ts +11 -2
  71. package/src/segment-store.ts +4 -0
  72. package/src/sqlite-bun-driver.ts +6 -2
  73. package/src/sqlite-bun.ts +2 -2
  74. package/src/sqlite-dialect.ts +7 -0
  75. package/src/sqlite-image.ts +26 -15
  76. package/src/sqlite-node.ts +2 -2
  77. package/src/sqlite-segment-store.ts +18 -4
  78. package/src/sqlite-storage.ts +203 -39
  79. package/src/storage-errors.ts +10 -1
  80. package/src/storage.ts +57 -10
@@ -1,3 +1,6 @@
1
+ import { validateCommitPruneQuery } from './prune';
2
+ import { StorageQueryError } from './storage-errors';
3
+ import type { CommitPruneQuery, CommitPruneResult } from './storage';
1
4
  /**
2
5
  * Postgres server storage: the production database path.
3
6
  *
@@ -85,6 +88,7 @@ import type {
85
88
  IndexRowScanQuery,
86
89
  NewCommit,
87
90
  NewReaction,
91
+ PartitionRegistryEntry,
88
92
  PrunedReactionCounts,
89
93
  ReactionClaimQuery,
90
94
  ReactionFailure,
@@ -136,6 +140,12 @@ CREATE TABLE IF NOT EXISTS sync_partitions(
136
140
  max_commit_seq BIGINT NOT NULL DEFAULT 0,
137
141
  horizon_seq BIGINT NOT NULL DEFAULT 0
138
142
  );
143
+ CREATE TABLE IF NOT EXISTS sync_partition_registry(
144
+ partition TEXT PRIMARY KEY,
145
+ log_epoch TEXT NOT NULL,
146
+ epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
147
+ last_authenticated_at_ms BIGINT NOT NULL
148
+ );
139
149
  CREATE TABLE IF NOT EXISTS sync_row_scopes(
140
150
  partition TEXT NOT NULL, tbl TEXT NOT NULL,
141
151
  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
@@ -189,10 +199,13 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
189
199
  ON sync_reactions(partition, status, available_at_ms, idempotency_key);
190
200
  CREATE TABLE IF NOT EXISTS sync_clients(
191
201
  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
202
+ wire_version INTEGER NOT NULL DEFAULT 1,
192
203
  cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
193
204
  updated_at_ms BIGINT NOT NULL,
194
205
  PRIMARY KEY(partition, client_id)
195
206
  );
207
+ ALTER TABLE sync_clients
208
+ ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
196
209
  CREATE TABLE IF NOT EXISTS sync_blob_refs(
197
210
  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
198
211
  blob_id TEXT NOT NULL,
@@ -213,6 +226,21 @@ interface SerializedResult {
213
226
  details?: import('@syncular/core').RejectionDetails;
214
227
  }
215
228
 
229
+ async function lockPartitionOn(
230
+ client: PgQueryable,
231
+ partition: string,
232
+ ): Promise<void> {
233
+ await client.query(
234
+ `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
235
+ ON CONFLICT (partition) DO NOTHING`,
236
+ [partition],
237
+ );
238
+ await client.query(
239
+ 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
240
+ [partition],
241
+ );
242
+ }
243
+
216
244
  function toBase64(bytes: Uint8Array): string {
217
245
  return Buffer.from(bytes).toString('base64');
218
246
  }
@@ -639,15 +667,7 @@ class PostgresTransaction implements StorageTransaction {
639
667
 
640
668
  async lockPartitionForPush(): Promise<void> {
641
669
  this.#assertOpen();
642
- await this.#client.query(
643
- `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
644
- ON CONFLICT (partition) DO NOTHING`,
645
- [this.#partition],
646
- );
647
- await this.#client.query(
648
- 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
649
- [this.#partition],
650
- );
670
+ await lockPartitionOn(this.#client, this.#partition);
651
671
  await this.#client.query('SAVEPOINT syncular_push_candidate');
652
672
  this.#pushApplySavepoint = true;
653
673
  }
@@ -998,6 +1018,87 @@ export class PostgresServerStorage implements ServerStorage {
998
1018
  this.#schemaVersion = schema.version;
999
1019
  }
1000
1020
 
1021
+ async touchPartition(
1022
+ partition: string,
1023
+ authenticatedAtMs: number,
1024
+ initialLogEpoch: string,
1025
+ ): Promise<PartitionRegistryEntry> {
1026
+ if (initialLogEpoch.length === 0) {
1027
+ throw new Error('initial log epoch must be non-empty');
1028
+ }
1029
+ const { rows } = await this.#exec.query<{
1030
+ log_epoch: string;
1031
+ epoch_required: boolean;
1032
+ last_authenticated_at_ms: unknown;
1033
+ }>(
1034
+ `INSERT INTO sync_partition_registry(
1035
+ partition, log_epoch, last_authenticated_at_ms
1036
+ ) VALUES ($1,$2,$3)
1037
+ ON CONFLICT(partition) DO UPDATE SET
1038
+ last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
1039
+ RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
1040
+ [partition, initialLogEpoch, authenticatedAtMs],
1041
+ );
1042
+ const row = rows[0];
1043
+ if (row === undefined)
1044
+ throw new Error('partition registry write did not persist');
1045
+ return {
1046
+ partition,
1047
+ logEpoch: row.log_epoch,
1048
+ epochRequired: row.epoch_required,
1049
+ lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
1050
+ };
1051
+ }
1052
+
1053
+ async rotatePartitionLogEpoch(
1054
+ partition: string,
1055
+ logEpoch: string,
1056
+ authenticatedAtMs: number,
1057
+ ): Promise<PartitionRegistryEntry> {
1058
+ if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
1059
+ await this.#exec.transaction(async (client) => {
1060
+ await lockPartitionOn(client, partition);
1061
+ await client.query(
1062
+ `INSERT INTO sync_partition_registry(
1063
+ partition, log_epoch, epoch_required, last_authenticated_at_ms
1064
+ ) VALUES ($1,$2,TRUE,$3)
1065
+ ON CONFLICT(partition) DO UPDATE SET
1066
+ log_epoch=EXCLUDED.log_epoch,
1067
+ epoch_required=TRUE,
1068
+ last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms`,
1069
+ [partition, logEpoch, authenticatedAtMs],
1070
+ );
1071
+ await client.query('DELETE FROM sync_clients WHERE partition=$1', [
1072
+ partition,
1073
+ ]);
1074
+ });
1075
+ return {
1076
+ partition,
1077
+ logEpoch,
1078
+ epochRequired: true,
1079
+ lastAuthenticatedAtMs: authenticatedAtMs,
1080
+ };
1081
+ }
1082
+
1083
+ async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
1084
+ const { rows } = await this.#exec.query<{
1085
+ partition: string;
1086
+ log_epoch: string;
1087
+ epoch_required: boolean;
1088
+ last_authenticated_at_ms: unknown;
1089
+ }>(
1090
+ `SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
1091
+ FROM sync_partition_registry ORDER BY partition`,
1092
+ [],
1093
+ );
1094
+ return rows.map((row) => ({
1095
+ partition: row.partition,
1096
+ logEpoch: row.log_epoch,
1097
+ epochRequired: row.epoch_required,
1098
+ lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
1099
+ }));
1100
+ }
1101
+
1001
1102
  /**
1002
1103
  * Open a real Postgres transaction. The push handler drives the returned
1003
1104
  * `StorageTransaction` imperatively (getRow/upsert/…/commit), but the
@@ -1058,7 +1159,7 @@ export class PostgresServerStorage implements ServerStorage {
1058
1159
  }
1059
1160
  const prepared = bindAuthoritativePartition(
1060
1161
  prepareAuthoritativeQuery(
1061
- query.sql,
1162
+ query.plan,
1062
1163
  query.params,
1063
1164
  query.tables,
1064
1165
  this.#tables,
@@ -1087,6 +1188,14 @@ export class PostgresServerStorage implements ServerStorage {
1087
1188
  });
1088
1189
  }
1089
1190
 
1191
+ async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
1192
+ const { rows } = await this.#exec.query<{ log_epoch: string }>(
1193
+ 'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
1194
+ [partition],
1195
+ );
1196
+ return rows[0]?.log_epoch;
1197
+ }
1198
+
1090
1199
  async getHorizonSeq(partition: string): Promise<number> {
1091
1200
  const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
1092
1201
  'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
@@ -1098,25 +1207,52 @@ export class PostgresServerStorage implements ServerStorage {
1098
1207
  async setHorizonSeq(partition: string, seq: number): Promise<void> {
1099
1208
  await this.#exec.query(
1100
1209
  `INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
1101
- ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`,
1210
+ ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`,
1102
1211
  [partition, seq],
1103
1212
  );
1104
1213
  }
1105
1214
 
1106
- async pruneCommitsThrough(partition: string, seq: number): Promise<number> {
1107
- const removed = await this.#exec.query(
1108
- 'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
1109
- [partition, seq],
1110
- );
1111
- await this.#exec.query(
1112
- 'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
1113
- [partition, seq],
1114
- );
1115
- await this.#exec.query(
1116
- 'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
1117
- [partition, seq],
1118
- );
1119
- return removed.rowCount;
1215
+ async pruneCommitsThrough(
1216
+ partition: string,
1217
+ query: CommitPruneQuery,
1218
+ ): Promise<CommitPruneResult> {
1219
+ validateCommitPruneQuery(query);
1220
+ return this.#exec.transaction(async (client) => {
1221
+ await lockPartitionOn(client, partition);
1222
+ const epoch = await client.query<{ log_epoch: string }>(
1223
+ 'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE',
1224
+ [partition],
1225
+ );
1226
+ if (epoch.rows[0]?.log_epoch !== query.logEpoch)
1227
+ throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
1228
+ const previous = await client.query<{ horizon_seq: unknown }>(
1229
+ 'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
1230
+ [partition],
1231
+ );
1232
+ const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
1233
+ const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
1234
+ await client.query(
1235
+ 'UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1',
1236
+ [partition, horizonSeq],
1237
+ );
1238
+ const removed = await client.query(
1239
+ 'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
1240
+ [partition, horizonSeq],
1241
+ );
1242
+ await client.query(
1243
+ 'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
1244
+ [partition, horizonSeq],
1245
+ );
1246
+ await client.query(
1247
+ 'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
1248
+ [partition, horizonSeq],
1249
+ );
1250
+ return {
1251
+ previousHorizonSeq,
1252
+ horizonSeq,
1253
+ removedCommits: removed.rowCount,
1254
+ };
1255
+ });
1120
1256
  }
1121
1257
 
1122
1258
  async getCommitSeqBefore(
@@ -1496,11 +1632,12 @@ export class PostgresServerStorage implements ServerStorage {
1496
1632
  const { rows } = await this.#exec.query<{
1497
1633
  client_id: string;
1498
1634
  actor_id: string;
1635
+ wire_version: unknown;
1499
1636
  cursor: unknown;
1500
1637
  subscriptions: unknown;
1501
1638
  updated_at_ms: unknown;
1502
1639
  }>(
1503
- 'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
1640
+ 'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
1504
1641
  [partition, clientId],
1505
1642
  );
1506
1643
  const record = rows[0];
@@ -1508,6 +1645,7 @@ export class PostgresServerStorage implements ServerStorage {
1508
1645
  return {
1509
1646
  clientId: record.client_id,
1510
1647
  actorId: record.actor_id,
1648
+ wireVersion: asNumber(record.wire_version),
1511
1649
  cursor: asNumber(record.cursor),
1512
1650
  updatedAtMs: asNumber(record.updated_at_ms),
1513
1651
  subscriptions: asJson<ClientSubscription[]>(record.subscriptions),
@@ -1519,16 +1657,18 @@ export class PostgresServerStorage implements ServerStorage {
1519
1657
  record: ClientRecord,
1520
1658
  ): Promise<void> {
1521
1659
  await this.#exec.query(
1522
- `INSERT INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms)
1523
- VALUES ($1,$2,$3,$4,$5,$6)
1660
+ `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
1661
+ VALUES ($1,$2,$3,$4,$5,$6,$7)
1524
1662
  ON CONFLICT (partition, client_id) DO UPDATE
1525
- SET actor_id=EXCLUDED.actor_id, cursor=EXCLUDED.cursor,
1663
+ SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
1664
+ cursor=EXCLUDED.cursor,
1526
1665
  subscriptions=EXCLUDED.subscriptions,
1527
1666
  updated_at_ms=EXCLUDED.updated_at_ms`,
1528
1667
  [
1529
1668
  partition,
1530
1669
  record.clientId,
1531
1670
  record.actorId,
1671
+ record.wireVersion,
1532
1672
  record.cursor,
1533
1673
  JSON.stringify(record.subscriptions),
1534
1674
  record.updatedAtMs,
@@ -1536,6 +1676,17 @@ export class PostgresServerStorage implements ServerStorage {
1536
1676
  );
1537
1677
  }
1538
1678
 
1679
+ async getActiveClientCursorFloor(
1680
+ partition: string,
1681
+ cutoffMs: number,
1682
+ ): Promise<number | null> {
1683
+ const { rows } = await this.#exec.query<{ cursor: unknown }>(
1684
+ 'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
1685
+ [partition, cutoffMs],
1686
+ );
1687
+ return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
1688
+ }
1689
+
1539
1690
  async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
1540
1691
  const { rows } = await this.#exec.query<{
1541
1692
  client_id: string;
@@ -1608,16 +1759,18 @@ export class PostgresServerStorage implements ServerStorage {
1608
1759
  const { rows } = await this.#exec.query<{
1609
1760
  client_id: string;
1610
1761
  actor_id: string;
1762
+ wire_version: unknown;
1611
1763
  cursor: unknown;
1612
1764
  subscriptions: unknown;
1613
1765
  updated_at_ms: unknown;
1614
1766
  }>(
1615
- 'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
1767
+ 'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
1616
1768
  [partition],
1617
1769
  );
1618
1770
  return rows.map((r) => ({
1619
1771
  clientId: r.client_id,
1620
1772
  actorId: r.actor_id,
1773
+ wireVersion: asNumber(r.wire_version),
1621
1774
  cursor: asNumber(r.cursor),
1622
1775
  updatedAtMs: asNumber(r.updated_at_ms),
1623
1776
  subscriptions: (typeof r.subscriptions === 'string'
@@ -1742,13 +1895,6 @@ export class PostgresServerStorage implements ServerStorage {
1742
1895
  }
1743
1896
 
1744
1897
  async listPartitions(): Promise<string[]> {
1745
- // Union: the registry row appears on first commit, the client row on
1746
- // first pull — a partition with only one of the two still shows up.
1747
- const { rows } = await this.#exec.query<{ partition: string }>(
1748
- `SELECT partition FROM sync_partitions
1749
- UNION SELECT partition FROM sync_clients ORDER BY partition`,
1750
- [],
1751
- );
1752
- return rows.map((r) => r.partition);
1898
+ return (await this.listPartitionRegistry()).map((entry) => entry.partition);
1753
1899
  }
1754
1900
  }
package/src/prune.ts CHANGED
@@ -7,7 +7,20 @@
7
7
  * least the newest `minRetainedCommits` commits are always retained.
8
8
  */
9
9
  import { emitEvent, type SyncularServerEvents } from './events';
10
- import type { ServerStorage } from './storage';
10
+ import type { CommitPruneQuery, ServerStorage } from './storage';
11
+ import { StorageQueryError } from './storage-errors';
12
+
13
+ /** Shared validation for the built-in atomic pruning adapters. */
14
+ export function validateCommitPruneQuery(query: CommitPruneQuery): void {
15
+ if (
16
+ !Number.isSafeInteger(query.throughSeq) ||
17
+ query.throughSeq < 0 ||
18
+ typeof query.logEpoch !== 'string' ||
19
+ query.logEpoch.length === 0
20
+ ) {
21
+ throw new StorageQueryError('sync.storage.invalid_prune_cursor');
22
+ }
23
+ }
11
24
 
12
25
  export interface RetentionPolicy {
13
26
  /** Active window for laggard cursors (default 14 days). */
@@ -37,28 +50,29 @@ export interface PruneOptions {
37
50
  export async function pruneCommitLog(options: PruneOptions): Promise<number> {
38
51
  const { storage, partition, nowMs } = options;
39
52
  const policy = { ...DEFAULT_RETENTION, ...options.retention };
53
+ const logEpoch = await storage.getPartitionLogEpoch(partition);
54
+ if (logEpoch === undefined)
55
+ throw new StorageQueryError('sync.storage.partition_unregistered');
40
56
  const maxSeq = await storage.getMaxCommitSeq(partition);
41
- const cursors = await storage.listClientCursors(partition);
42
- const activeCursors = cursors
43
- .filter((c) => c.updatedAtMs >= nowMs - policy.activeWindowMs)
44
- .map((c) => c.cursor);
45
57
  const cursorFloor =
46
- activeCursors.length > 0
47
- ? Math.min(...activeCursors)
48
- : Number.MAX_SAFE_INTEGER;
58
+ (await storage.getActiveClientCursorFloor(
59
+ partition,
60
+ nowMs - policy.activeWindowMs,
61
+ )) ?? Number.MAX_SAFE_INTEGER;
49
62
  const forcedSeq = await storage.getCommitSeqBefore(
50
63
  partition,
51
64
  nowMs - policy.ageForceMs,
52
65
  );
53
66
  const retainFloor = maxSeq - policy.minRetainedCommits;
54
67
  const target = Math.min(Math.max(cursorFloor, forcedSeq), retainFloor);
55
- const current = await storage.getHorizonSeq(partition);
56
- const horizon = Math.max(current, Math.max(0, target));
57
- let removedCommits = 0;
58
- if (horizon > current) {
59
- await storage.setHorizonSeq(partition, horizon);
60
- removedCommits = await storage.pruneCommitsThrough(partition, horizon);
61
- }
68
+ const {
69
+ previousHorizonSeq: current,
70
+ horizonSeq: horizon,
71
+ removedCommits,
72
+ } = await storage.pruneCommitsThrough(partition, {
73
+ logEpoch,
74
+ throughSeq: Math.max(0, target),
75
+ });
62
76
  const events = options.events;
63
77
  if (events !== undefined) {
64
78
  emitEvent(events, {
package/src/pull.ts CHANGED
@@ -17,10 +17,16 @@ import { clockOf, limitsOf } from './context';
17
17
  import type { PullSegmentSummary } from './events';
18
18
  import type { CompiledSchema, CompiledTable } from './schema';
19
19
  import { scopeDigest } from './scopes';
20
- import type { SegmentRecord } from './segment-store';
20
+ import type { SegmentRecord, SegmentStore } from './segment-store';
21
21
  import { issueSegmentUrl } from './signed-url';
22
22
  import type { SqliteImageBuilder } from './sqlite-image';
23
- import type { StoredCommit, StoredRow } from './storage';
23
+ import type { ServerStorage, StoredCommit, StoredRow } from './storage';
24
+
25
+ // One artifact build per owning storage pair and complete immutable identity.
26
+ const imageBuilds = new WeakMap<
27
+ ServerStorage,
28
+ WeakMap<SegmentStore, Map<string, Promise<SegmentRecord>>>
29
+ >();
24
30
 
25
31
  /**
26
32
  * Resolve the §5.3 image builder: the host-injected one if present, else the
@@ -212,12 +218,14 @@ async function* sqliteImageSegment(
212
218
  asOf: number,
213
219
  digest: string,
214
220
  trace: PullSectionTrace | undefined,
221
+ logEpoch: string,
215
222
  ): AsyncGenerator<ResponseFrame, boolean> {
216
223
  const { storage, segments, partition } = ctx;
217
224
  const now = clockOf(ctx)();
218
225
  const existing = await segments.find(
219
226
  {
220
227
  partition,
228
+ logEpoch,
221
229
  table: plan.table.name,
222
230
  schemaVersion: schema.version,
223
231
  mediaType: 'sqlite',
@@ -258,56 +266,89 @@ async function* sqliteImageSegment(
258
266
  // support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
259
267
  const buildImage = await resolveImageBuilder(ctx);
260
268
  if (buildImage === undefined) return false;
261
- // The probe rows are the snapshot's first page — keep them and scan on
262
- // from the probe's cursor instead of re-reading the whole prefix (the
263
- // scan is keyset-ordered by rowId, so the concatenation is exactly the
264
- // rows a single full scan would return).
265
- const rows: StoredRow[] = [...probe];
266
- let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
267
- for (;;) {
268
- const scanned: StoredRow[] = await storage.scanRows(partition, {
269
- table: plan.table.name,
270
- scopeFilter: plan.effective,
271
- afterRowId,
272
- limit: 50_000,
273
- });
274
- rows.push(...scanned);
275
- const last = scanned[scanned.length - 1];
276
- if (scanned.length < 50_000 || last === undefined) break;
277
- afterRowId = last.rowId;
269
+ let stores = imageBuilds.get(storage);
270
+ if (stores === undefined) {
271
+ stores = new WeakMap();
272
+ imageBuilds.set(storage, stores);
278
273
  }
279
- const bytes = buildImage({
280
- table: plan.table,
274
+ let builds = stores.get(segments);
275
+ if (builds === undefined) {
276
+ builds = new Map();
277
+ stores.set(segments, builds);
278
+ }
279
+ const identity = {
280
+ partition,
281
+ logEpoch,
282
+ table: plan.table.name,
281
283
  schemaVersion: schema.version,
282
- asOfCommitSeq: asOf,
284
+ mediaType: 'sqlite' as const,
283
285
  scopeDigest: digest,
284
- rows,
285
- });
286
- const record = await segments.put(
287
- {
288
- partition,
289
- table: plan.table.name,
290
- schemaVersion: schema.version,
291
- mediaType: 'sqlite',
292
- scopeDigest: digest,
293
- asOfCommitSeq: asOf,
294
- rowCount: rows.length,
295
- rowCursor: null,
296
- nextRowCursor: null,
297
- },
298
- bytes,
299
- now,
300
- );
286
+ asOfCommitSeq: asOf,
287
+ };
288
+ const key = JSON.stringify(identity);
289
+ let building = builds.get(key);
290
+ let builtHere = false;
291
+ if (building === undefined) {
292
+ building = (async () => {
293
+ // Another request can finish while this one's eligibility probe awaits.
294
+ const cached = await segments.find(identity, clockOf(ctx)());
295
+ if (cached !== undefined) return cached;
296
+ builtHere = true;
297
+ let rowCount = 0;
298
+ const bytes = await buildImage({
299
+ table: plan.table,
300
+ schemaVersion: schema.version,
301
+ asOfCommitSeq: asOf,
302
+ scopeDigest: digest,
303
+ rowBatches: (async function* () {
304
+ rowCount += probe.length;
305
+ yield probe;
306
+ let afterRowId = probe[probe.length - 1]!.rowId;
307
+ for (;;) {
308
+ const rows = await storage.scanRows(partition, {
309
+ table: plan.table.name,
310
+ scopeFilter: plan.effective,
311
+ afterRowId,
312
+ limit: 5_000,
313
+ });
314
+ rowCount += rows.length;
315
+ yield rows;
316
+ const last = rows[rows.length - 1];
317
+ if (rows.length < 5_000 || last === undefined) break;
318
+ afterRowId = last.rowId;
319
+ }
320
+ })(),
321
+ });
322
+ return segments.put(
323
+ { ...identity, rowCount, rowCursor: null, nextRowCursor: null },
324
+ bytes,
325
+ clockOf(ctx)(),
326
+ );
327
+ })();
328
+ builds.set(key, building);
329
+ }
330
+ let record: SegmentRecord;
331
+ try {
332
+ record = await building;
333
+ } finally {
334
+ if (builds.get(key) === building) builds.delete(key);
335
+ }
301
336
  trace?.segments.push({
302
337
  mediaType: 'sqlite',
303
338
  delivery: 'ref',
304
- origin: 'built',
339
+ origin: builtHere ? 'built' : 'reused',
305
340
  bytes: record.byteLength,
306
341
  rows: record.rowCount,
307
342
  });
308
343
  yield segmentRefFrame(
309
344
  record,
310
- await signedUrlFields(ctx, limits, record.segmentId, digest, now),
345
+ await signedUrlFields(
346
+ ctx,
347
+ limits,
348
+ record.segmentId,
349
+ digest,
350
+ clockOf(ctx)(),
351
+ ),
311
352
  );
312
353
  return true;
313
354
  }
@@ -320,6 +361,7 @@ async function* bootstrapSegments(
320
361
  asOf: number,
321
362
  startRowCursor: string | null,
322
363
  trace: PullSectionTrace | undefined,
364
+ logEpoch: string,
323
365
  ): AsyncGenerator<
324
366
  ResponseFrame,
325
367
  { complete: boolean; rowCursor: string | null }
@@ -347,6 +389,7 @@ async function* bootstrapSegments(
347
389
  asOf,
348
390
  digest,
349
391
  trace,
392
+ logEpoch,
350
393
  );
351
394
  if (imaged) return { complete: true, rowCursor: null };
352
395
  }
@@ -393,6 +436,7 @@ async function* bootstrapSegments(
393
436
  const record = await segments.put(
394
437
  {
395
438
  partition,
439
+ logEpoch,
396
440
  table: plan.table.name,
397
441
  schemaVersion: schema.version,
398
442
  mediaType: 'rows',
@@ -435,8 +479,12 @@ export async function* subscriptionSection(
435
479
  maxSeq: number,
436
480
  horizonSeq: number,
437
481
  trace?: PullSectionTrace,
482
+ logEpoch?: string,
438
483
  ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
439
484
  const sub = plan.frame;
485
+ if (logEpoch === undefined || logEpoch.length === 0) {
486
+ throw new Error('subscriptionSection requires a non-empty log epoch');
487
+ }
440
488
 
441
489
  if (plan.status === 'revoked') {
442
490
  yield {
@@ -496,6 +544,7 @@ export async function* subscriptionSection(
496
544
  asOf,
497
545
  startCursor,
498
546
  trace,
547
+ logEpoch,
499
548
  );
500
549
  if (outcome.complete) {
501
550
  yield { type: 'SUB_END', nextCursor: asOf };
package/src/push.ts CHANGED
@@ -185,7 +185,7 @@ async function runValidator(
185
185
  return errorRecord(
186
186
  opIndex,
187
187
  'sync.constraint_violation',
188
- `write validator for table ${JSON.stringify(table.name)} threw: ${error instanceof Error ? error.message : String(error)}`,
188
+ 'write validator failed',
189
189
  );
190
190
  }
191
191
  return undefined;
@@ -220,7 +220,7 @@ async function mergeCrdtColumns(
220
220
  return errorRecord(
221
221
  opIndex,
222
222
  'sync.crdt_merge_failed',
223
- `no CRDT merger registered for crdtType ${JSON.stringify(crdtType)} (§5.10.2)`,
223
+ 'no CRDT merger registered',
224
224
  );
225
225
  }
226
226
  const storedRaw = storedValues?.[index];
@@ -228,11 +228,11 @@ async function mergeCrdtColumns(
228
228
  let merged: Uint8Array;
229
229
  try {
230
230
  merged = await merger(stored, incoming);
231
- } catch (error) {
231
+ } catch {
232
232
  return errorRecord(
233
233
  opIndex,
234
234
  'sync.crdt_merge_failed',
235
- `CRDT merger for ${JSON.stringify(crdtType)} threw: ${error instanceof Error ? error.message : String(error)}`,
235
+ 'CRDT merger failed',
236
236
  );
237
237
  }
238
238
  values[index] = merged;
@@ -732,7 +732,7 @@ async function runCommitValidator(
732
732
  return errorRecord(
733
733
  operations[0]?.opIndex ?? 0,
734
734
  'sync.constraint_violation',
735
- `whole-commit validator threw: ${error instanceof Error ? error.message : String(error)}`,
735
+ 'whole-commit validator failed',
736
736
  );
737
737
  }
738
738
  return undefined;