@tanstack/db-sqlite-persistence-core 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +32 -3
  2. package/dist/cjs/broadcast-coordinator.cjs +282 -43
  3. package/dist/cjs/broadcast-coordinator.cjs.map +1 -1
  4. package/dist/cjs/broadcast-coordinator.d.cts +25 -8
  5. package/dist/cjs/committed-tx-anchor.cjs +9 -0
  6. package/dist/cjs/committed-tx-anchor.cjs.map +1 -0
  7. package/dist/cjs/committed-tx-anchor.d.cts +1 -0
  8. package/dist/cjs/index.cjs +4 -0
  9. package/dist/cjs/index.cjs.map +1 -1
  10. package/dist/cjs/persisted.cjs +1599 -313
  11. package/dist/cjs/persisted.cjs.map +1 -1
  12. package/dist/cjs/persisted.d.cts +106 -13
  13. package/dist/cjs/remote-subset-wire.cjs +11 -0
  14. package/dist/cjs/remote-subset-wire.cjs.map +1 -1
  15. package/dist/cjs/remote-subset-wire.d.cts +1 -0
  16. package/dist/cjs/sqlite-core-adapter.cjs +705 -64
  17. package/dist/cjs/sqlite-core-adapter.cjs.map +1 -1
  18. package/dist/cjs/sqlite-core-adapter.d.cts +53 -10
  19. package/dist/esm/broadcast-coordinator.d.ts +25 -8
  20. package/dist/esm/broadcast-coordinator.js +283 -44
  21. package/dist/esm/broadcast-coordinator.js.map +1 -1
  22. package/dist/esm/committed-tx-anchor.d.ts +1 -0
  23. package/dist/esm/committed-tx-anchor.js +9 -0
  24. package/dist/esm/committed-tx-anchor.js.map +1 -0
  25. package/dist/esm/index.js +6 -2
  26. package/dist/esm/persisted.d.ts +106 -13
  27. package/dist/esm/persisted.js +1601 -315
  28. package/dist/esm/persisted.js.map +1 -1
  29. package/dist/esm/remote-subset-wire.d.ts +1 -0
  30. package/dist/esm/remote-subset-wire.js +11 -0
  31. package/dist/esm/remote-subset-wire.js.map +1 -1
  32. package/dist/esm/sqlite-core-adapter.d.ts +53 -10
  33. package/dist/esm/sqlite-core-adapter.js +708 -67
  34. package/dist/esm/sqlite-core-adapter.js.map +1 -1
  35. package/package.json +2 -2
  36. package/src/broadcast-coordinator.ts +415 -62
  37. package/src/committed-tx-anchor.ts +13 -0
  38. package/src/persisted.ts +2283 -370
  39. package/src/remote-subset-wire.ts +12 -0
  40. package/src/sqlite-core-adapter.ts +891 -68
package/README.md CHANGED
@@ -172,10 +172,23 @@ semantics.
172
172
  After a mutating RPC transport failure, replay is allowed only while the
173
173
  requester can prove the same non-null leader id and term still own the route.
174
174
  If either value was unknown for the first attempt, or either value changes,
175
- the request rejects with `IndeterminateCommitError` and requires application
176
- reconciliation. Coordinators never retry an indeterminate mutation against an
175
+ the direct request rejects with `IndeterminateCommitError` and requires
176
+ application reconciliation. Coordinators never retry that mutation against an
177
177
  unknown or replacement leader.
178
178
 
179
+ A source sync transaction has a separate exact-ID reconciliation path. The
180
+ persisted wrapper holds its applied receipt while an elected coordinator and
181
+ SQLite adapter check the transaction under the writer lock. An existing ID
182
+ acknowledges the original durable position. An absent ID permits one application
183
+ only if the durable row version and reset epoch equal the wrapper's pre-send
184
+ snapshot; another durable write may carry newer row values or an opaque source
185
+ cursor and cannot be overwritten safely. On a certified result the wrapper
186
+ fulfills the receipt in the same sync run and the coordinator invalidates peers,
187
+ including a peer that missed the original notification. An uncertified result
188
+ retains `IndeterminateCommitError` and terminal source failure. Custom adapters
189
+ and coordinators without this optional operation keep that terminal behavior.
190
+ One generated `txId` identifies one immutable source transaction payload.
191
+
179
192
  `SingleProcessCoordinator` also implements the complete route. It is always
180
193
  leader and has no cross-process transport, but it applies the transaction to
181
194
  the resolved adapter registered for the specific collection. The Browser and
@@ -278,13 +291,22 @@ separate from the accepted-transaction precedence rule.
278
291
  ### Atomic resume snapshots
279
292
 
280
293
  Persistence adapters implement
281
- `loadResumeSnapshot(collectionId, options)` to let a sync source certify a
294
+ `loadResumeSnapshot(target, options)` to let a sync source certify a
282
295
  persisted resume baseline. One call must read rows, collection metadata, stream
283
296
  position, reset epoch, and key-set evidence from the same atomic database
284
297
  snapshot. `includeRows: false` requests the same certification data without
285
298
  materializing rows; `requiredIndexSignatures` carries the indexes needed by a
286
299
  row-bearing snapshot.
287
300
 
301
+ The adapter's data methods require an explicit `PersistedStorageTarget`. Use
302
+ `{ kind: 'eager', collectionId }` for an eager Collection, or
303
+ `{ kind: 'managed', storageCollectionId, claimId }` for a claimed on-demand
304
+ persisted cache generation. A bare string ID is rejected. The managed target
305
+ checks the claim even after its generation has been collected from SQLite;
306
+ choosing an eager target explicitly may reuse the same ID. Claim and
307
+ rotation methods take the logical Collection ID; renewal takes the physical
308
+ storage ID and release takes the claim ID.
309
+
288
310
  `PersistedKeySetEvidence.status` has three states:
289
311
 
290
312
  - `consistent`: the persisted rows match the adapter's durable expected-key
@@ -294,6 +316,13 @@ row-bearing snapshot.
294
316
  - `unknown`: the adapter has no authoritative pre-migration key set and does
295
317
  not claim completeness.
296
318
 
319
+ A no-write election can reserve a greater durable term and reset its sequence
320
+ to zero without changing persisted rows or the source resume cursor. The
321
+ persisted wrapper keeps an already certified resume baseline across that
322
+ transition only when the atomic snapshot has the same row version and reset
323
+ epoch. A committed write or schema reset requires fresh certification and
324
+ cannot inherit the earlier baseline merely because the key set still matches.
325
+
297
326
  The method is required because the versioned `metadata.persistence` capability
298
327
  always carries hydration, durable row scanning, certification, evidence, and
299
328
  generation ownership as one complete bundle. Sync wrappers must forward the
@@ -1,9 +1,11 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
3
  const db = require("@tanstack/db");
4
+ const committedTxAnchor = require("./committed-tx-anchor.cjs");
4
5
  const errors = require("./errors.cjs");
5
6
  const remoteSubsetWire = require("./remote-subset-wire.cjs");
6
7
  const remoteSubsetOwner = require("./remote-subset-owner.cjs");
8
+ const persisted = require("./persisted.cjs");
7
9
  const HEARTBEAT_INTERVAL_MS = 3e3;
8
10
  const LEADERSHIP_RETRY_DELAY_MS = 200;
9
11
  const RPC_TIMEOUT_MS = 1e4;
@@ -13,10 +15,16 @@ const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2;
13
15
  const RPC_DEDUPE_RETENTION_MS = 6e4;
14
16
  const WRITER_LOCK_BUSY_RETRY_MS = 50;
15
17
  const WRITER_LOCK_MAX_RETRIES = 20;
18
+ function requireDurableElection(adapter) {
19
+ if (typeof adapter.reserveLeadershipTerm !== `function`) {
20
+ throw new errors.InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
21
+ }
22
+ }
16
23
  class BroadcastCollectionCoordinator {
17
24
  constructor(options) {
18
25
  this.nodeId = db.safeRandomUUID();
19
26
  this.collectionAdapters = /* @__PURE__ */ new Map();
27
+ this.collectionClaimIds = /* @__PURE__ */ new Map();
20
28
  this.remoteSubsetOwners = /* @__PURE__ */ new Map();
21
29
  this.remoteSubsetIds = /* @__PURE__ */ new Map();
22
30
  this.outboundRemoteSubsetAcquisitions = /* @__PURE__ */ new Map();
@@ -29,6 +37,7 @@ class BroadcastCollectionCoordinator {
29
37
  this.appliedEnvelopePruneTimer = null;
30
38
  this.rejectDisposed = null;
31
39
  this.disposed = false;
40
+ if (options.adapter) requireDurableElection(options.adapter);
32
41
  this.dbName = options.dbName;
33
42
  this.coordinatorName = options.coordinatorName;
34
43
  this.defaultAdapter = options.adapter ?? null;
@@ -67,11 +76,18 @@ class BroadcastCollectionCoordinator {
67
76
  * Called by a host persistence factory to wire the internally-created adapter.
68
77
  */
69
78
  setAdapter(adapter) {
79
+ requireDurableElection(adapter);
70
80
  this.defaultAdapter = adapter;
71
81
  }
72
82
  /** Register the persistence adapter that owns one collection. */
73
- setAdapterForCollection(collectionId, adapter) {
83
+ setAdapterForCollection(collectionId, adapter, cacheGenerationClaimId) {
84
+ requireDurableElection(adapter);
74
85
  this.collectionAdapters.set(collectionId, adapter);
86
+ if (cacheGenerationClaimId) {
87
+ this.collectionClaimIds.set(collectionId, cacheGenerationClaimId);
88
+ } else {
89
+ this.collectionClaimIds.delete(collectionId);
90
+ }
75
91
  }
76
92
  registerRemoteSubsetOwner(collectionId, owner) {
77
93
  if (this.remoteSubsetOwners.has(collectionId)) {
@@ -215,10 +231,29 @@ class BroadcastCollectionCoordinator {
215
231
  }
216
232
  try {
217
233
  await acquisition.release;
234
+ } catch (error) {
235
+ this.scheduleRemoteSubsetReleaseRetry(acquisition, options);
236
+ throw error;
218
237
  } finally {
219
238
  acquisition.release = null;
220
239
  }
221
240
  }
241
+ scheduleRemoteSubsetReleaseRetry(acquisition, options) {
242
+ const key = remoteSubsetAcquisitionKey(
243
+ acquisition.collectionId,
244
+ acquisition.acquisitionId
245
+ );
246
+ if (this.isDisposed() || this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || acquisition.retryTimer !== null) {
247
+ return;
248
+ }
249
+ acquisition.retryTimer = setTimeout(() => {
250
+ acquisition.retryTimer = null;
251
+ void this.requestReleaseRemoteSubset(
252
+ acquisition.collectionId,
253
+ options
254
+ ).catch(() => void 0);
255
+ }, RPC_RETRY_DELAY_MS);
256
+ }
222
257
  async acquireRemoteSubset(acquisition) {
223
258
  if (acquisition.inFlight) return acquisition.inFlight;
224
259
  const route = { localOwner: false };
@@ -278,7 +313,9 @@ class BroadcastCollectionCoordinator {
278
313
  try {
279
314
  await work;
280
315
  acquired = true;
281
- this.cancelRemoteSubsetReplayRetry(acquisition);
316
+ if (!acquisition.releaseRequested) {
317
+ this.cancelRemoteSubsetReplayRetry(acquisition);
318
+ }
282
319
  } finally {
283
320
  if (acquisition.inFlight === work) acquisition.inFlight = null;
284
321
  const current = this.collections.get(acquisition.collectionId);
@@ -295,21 +332,34 @@ class BroadcastCollectionCoordinator {
295
332
  }
296
333
  }
297
334
  }
298
- async requestEnsurePersistedIndex(collectionId, signature, spec, scopedAdapter, localEnsureCompleted = false) {
335
+ async requestEnsurePersistedIndex(collectionId, signature, spec, scopedAdapter, localEnsureCompleted = false, cacheGenerationClaimId) {
299
336
  if (this.isLeader(collectionId)) {
300
337
  if (localEnsureCompleted) return;
301
- await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex(
302
- collectionId,
303
- signature,
304
- spec
305
- );
338
+ const adapter = scopedAdapter ?? this.requireAdapter(collectionId);
339
+ if (cacheGenerationClaimId) {
340
+ await adapter.ensureIndex(
341
+ persisted.managedStorageTarget(collectionId, cacheGenerationClaimId),
342
+ signature,
343
+ spec,
344
+ {
345
+ cacheGenerationClaimId
346
+ }
347
+ );
348
+ } else {
349
+ await adapter.ensureIndex(
350
+ persisted.eagerStorageTarget(collectionId),
351
+ signature,
352
+ spec
353
+ );
354
+ }
306
355
  return;
307
356
  }
308
357
  const response = await this.sendRPC(collectionId, {
309
358
  type: `rpc:ensurePersistedIndex:req`,
310
359
  rpcId: db.safeRandomUUID(),
311
360
  signature,
312
- spec
361
+ spec,
362
+ cacheGenerationClaimId
313
363
  });
314
364
  if (!response.ok) {
315
365
  throw new Error(
@@ -317,7 +367,7 @@ class BroadcastCollectionCoordinator {
317
367
  );
318
368
  }
319
369
  }
320
- async requestApplyLocalMutations(collectionId, mutations) {
370
+ async requestApplyLocalMutations(collectionId, mutations, cacheGenerationClaimId) {
321
371
  const initialRoute = this.waitForInitialRoute(collectionId);
322
372
  if (initialRoute) await initialRoute;
323
373
  if (this.isLeader(collectionId)) {
@@ -325,14 +375,16 @@ class BroadcastCollectionCoordinator {
325
375
  type: `rpc:applyLocalMutations:req`,
326
376
  rpcId: db.safeRandomUUID(),
327
377
  envelopeId: db.safeRandomUUID(),
328
- mutations
378
+ mutations,
379
+ cacheGenerationClaimId
329
380
  });
330
381
  }
331
382
  return this.sendRPC(collectionId, {
332
383
  type: `rpc:applyLocalMutations:req`,
333
384
  rpcId: db.safeRandomUUID(),
334
385
  envelopeId: db.safeRandomUUID(),
335
- mutations
386
+ mutations,
387
+ cacheGenerationClaimId
336
388
  });
337
389
  }
338
390
  async requestApplyCommittedTx(collectionId, tx, scopedAdapter) {
@@ -349,14 +401,39 @@ class BroadcastCollectionCoordinator {
349
401
  }
350
402
  return this.sendRPC(collectionId, request);
351
403
  }
352
- async pullSince(collectionId, fromRowVersion, scopedAdapter) {
404
+ async reconcileCommittedTx(collectionId, tx, anchor, scopedAdapter) {
405
+ const initialRoute = this.waitForInitialRoute(collectionId);
406
+ if (initialRoute) await initialRoute;
407
+ let response;
408
+ for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) {
409
+ if (attempt > 0) await sleep(RPC_RETRY_DELAY_MS * attempt);
410
+ const request = {
411
+ type: `rpc:reconcileCommittedTx:req`,
412
+ rpcId: db.safeRandomUUID(),
413
+ tx,
414
+ anchor
415
+ };
416
+ response = this.isLeader(collectionId) ? await this.handleReconcileCommittedTx(
417
+ collectionId,
418
+ request,
419
+ scopedAdapter
420
+ ) : await this.sendRPC(
421
+ collectionId,
422
+ request
423
+ );
424
+ if (response.ok || response.code !== `NOT_LEADER`) return response;
425
+ }
426
+ return response;
427
+ }
428
+ async pullSince(collectionId, fromRowVersion, scopedAdapter, cacheGenerationClaimId) {
353
429
  if (this.isLeader(collectionId)) {
354
430
  return this.handlePullSince(
355
431
  collectionId,
356
432
  {
357
433
  type: `rpc:pullSince:req`,
358
434
  rpcId: db.safeRandomUUID(),
359
- fromRowVersion
435
+ fromRowVersion,
436
+ cacheGenerationClaimId
360
437
  },
361
438
  scopedAdapter
362
439
  );
@@ -364,7 +441,8 @@ class BroadcastCollectionCoordinator {
364
441
  return this.sendRPC(collectionId, {
365
442
  type: `rpc:pullSince:req`,
366
443
  rpcId: db.safeRandomUUID(),
367
- fromRowVersion
444
+ fromRowVersion,
445
+ cacheGenerationClaimId
368
446
  });
369
447
  }
370
448
  // -----------------------------------------------------------------------
@@ -397,6 +475,7 @@ class BroadcastCollectionCoordinator {
397
475
  this.channel.close();
398
476
  this.collections.clear();
399
477
  this.collectionAdapters.clear();
478
+ this.collectionClaimIds.clear();
400
479
  for (const collectionId of this.remoteSubsetOwners.keys()) {
401
480
  this.releaseInboundRemoteSubsetAcquisitions(collectionId);
402
481
  }
@@ -454,16 +533,24 @@ class BroadcastCollectionCoordinator {
454
533
  }
455
534
  try {
456
535
  const adapter = this.requireAdapter(collectionId);
457
- if (adapter.getStreamPosition) {
458
- const pos = await adapter.getStreamPosition(collectionId);
459
- state.latestTerm = pos.latestTerm;
460
- state.latestSeq = pos.latestSeq;
461
- state.latestRowVersion = pos.latestRowVersion;
462
- }
536
+ const reserveTerm = adapter.reserveLeadershipTerm;
537
+ if (!reserveTerm)
538
+ throw new errors.InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
539
+ const pos = await this.withWriterLock(() => {
540
+ const claimId = this.collectionClaimIds.get(collectionId);
541
+ return reserveTerm.call(
542
+ adapter,
543
+ claimId ? persisted.managedStorageTarget(collectionId, claimId) : persisted.eagerStorageTarget(collectionId),
544
+ state.latestTerm,
545
+ { cacheGenerationClaimId: claimId }
546
+ );
547
+ });
548
+ state.latestTerm = pos.latestTerm;
549
+ state.latestSeq = pos.latestSeq;
550
+ state.latestRowVersion = pos.latestRowVersion;
463
551
  if (this.isDisposed() || !this.participatesInLeadership(state) || this.collections.get(collectionId) !== state || abortController.signal.aborted || state.lockAbortController !== abortController) {
464
552
  return;
465
553
  }
466
- state.latestTerm++;
467
554
  state.isLeader = true;
468
555
  state.leaderId = this.nodeId;
469
556
  this.notifyRouteWaiters(state);
@@ -604,6 +691,7 @@ class BroadcastCollectionCoordinator {
604
691
  this.releaseLeadership(collectionId, state);
605
692
  this.collections.delete(collectionId);
606
693
  this.collectionAdapters.delete(collectionId);
694
+ this.collectionClaimIds.delete(collectionId);
607
695
  const prefix = `${JSON.stringify([collectionId]).slice(0, -1)},`;
608
696
  for (const key of this.appliedEnvelopes.keys()) {
609
697
  if (key.startsWith(prefix)) this.appliedEnvelopes.delete(key);
@@ -720,6 +808,7 @@ class BroadcastCollectionCoordinator {
720
808
  const heartbeat = payload;
721
809
  if (typeof heartbeat.leaderId === `string` && typeof heartbeat.term === `number` && typeof heartbeat.latestSeq === `number` && typeof heartbeat.latestRowVersion === `number`) {
722
810
  const state2 = this.getOrCreateCollectionState(envelope.collectionId);
811
+ if (state2.isLeader) return;
723
812
  if (heartbeat.term < state2.latestTerm) return;
724
813
  const changedLeader = state2.leaderId !== heartbeat.leaderId;
725
814
  state2.leaderId = heartbeat.leaderId;
@@ -900,6 +989,8 @@ class BroadcastCollectionCoordinator {
900
989
  return this.handleApplyLocalMutations(collectionId, request);
901
990
  case `rpc:applyCommittedTx:req`:
902
991
  return this.handleApplyCommittedTx(collectionId, request);
992
+ case `rpc:reconcileCommittedTx:req`:
993
+ return this.handleReconcileCommittedTx(collectionId, request);
903
994
  case `rpc:pullSince:req`:
904
995
  return this.handlePullSince(collectionId, request);
905
996
  }
@@ -1014,10 +1105,7 @@ class BroadcastCollectionCoordinator {
1014
1105
  acquisition.released = true;
1015
1106
  acquisition.release = (async () => {
1016
1107
  try {
1017
- await acquisition.load;
1018
- } catch {
1019
- }
1020
- try {
1108
+ await acquisition.admitted;
1021
1109
  if (acquisition.transferred) {
1022
1110
  await remoteSubsetOwner.unloadRemoteSubsetOwner(acquisition.owner, acquisition.options);
1023
1111
  }
@@ -1133,6 +1221,7 @@ class BroadcastCollectionCoordinator {
1133
1221
  owner: options.owner,
1134
1222
  options: options.options,
1135
1223
  load: Promise.resolve(),
1224
+ admitted: Promise.resolve(),
1136
1225
  transferred: false,
1137
1226
  released: false,
1138
1227
  terminalRelease: false,
@@ -1140,6 +1229,10 @@ class BroadcastCollectionCoordinator {
1140
1229
  };
1141
1230
  this.releasedRemoteSubsetAcquisitionTimes.delete(key);
1142
1231
  this.inboundRemoteSubsetAcquisitions.set(key, acquisition);
1232
+ let resolveAdmission;
1233
+ acquisition.admitted = new Promise((resolve) => {
1234
+ resolveAdmission = resolve;
1235
+ });
1143
1236
  let resolveLoad;
1144
1237
  let rejectLoad;
1145
1238
  acquisition.load = new Promise((resolve, reject) => {
@@ -1152,6 +1245,8 @@ class BroadcastCollectionCoordinator {
1152
1245
  void Promise.resolve(load).then(resolveLoad, rejectLoad);
1153
1246
  } catch (error) {
1154
1247
  rejectLoad(error);
1248
+ } finally {
1249
+ resolveAdmission();
1155
1250
  }
1156
1251
  try {
1157
1252
  await acquisition.load;
@@ -1176,10 +1271,23 @@ class BroadcastCollectionCoordinator {
1176
1271
  }
1177
1272
  }
1178
1273
  async handleEnsurePersistedIndex(collectionId, request) {
1179
- await this.withScheduledWriterLock(
1180
- collectionId,
1181
- (adapter) => adapter.ensureIndex(collectionId, request.signature, request.spec)
1182
- );
1274
+ await this.withScheduledWriterLock(collectionId, (adapter) => {
1275
+ if (request.cacheGenerationClaimId) {
1276
+ return adapter.ensureIndex(
1277
+ persisted.managedStorageTarget(collectionId, request.cacheGenerationClaimId),
1278
+ request.signature,
1279
+ request.spec,
1280
+ {
1281
+ cacheGenerationClaimId: request.cacheGenerationClaimId
1282
+ }
1283
+ );
1284
+ }
1285
+ return adapter.ensureIndex(
1286
+ persisted.eagerStorageTarget(collectionId),
1287
+ request.signature,
1288
+ request.spec
1289
+ );
1290
+ });
1183
1291
  return {
1184
1292
  type: `rpc:ensurePersistedIndex:res`,
1185
1293
  rpcId: request.rpcId,
@@ -1208,6 +1316,11 @@ class BroadcastCollectionCoordinator {
1208
1316
  });
1209
1317
  }
1210
1318
  async applyLocalMutationsOnce(collectionId, request) {
1319
+ if (this.collectionClaimIds.has(collectionId) && !request.cacheGenerationClaimId) {
1320
+ throw new errors.InvalidPersistedCollectionConfigError(
1321
+ `A managed local mutation requires a persisted cache claim`
1322
+ );
1323
+ }
1211
1324
  const state = this.collections.get(collectionId);
1212
1325
  if (!state || !state.isLeader) {
1213
1326
  return {
@@ -1233,6 +1346,7 @@ class BroadcastCollectionCoordinator {
1233
1346
  }
1234
1347
  const pendingTx = {
1235
1348
  txId: db.safeRandomUUID(),
1349
+ cacheGenerationClaimId: request.cacheGenerationClaimId,
1236
1350
  mutations: request.mutations.map((m) => ({
1237
1351
  type: m.type,
1238
1352
  key: m.key,
@@ -1393,6 +1507,10 @@ class BroadcastCollectionCoordinator {
1393
1507
  }
1394
1508
  );
1395
1509
  this.pruneAppliedEnvelopes();
1510
+ this.publishCommittedTx(collectionId, state, tx);
1511
+ return response;
1512
+ }
1513
+ publishCommittedTx(collectionId, state, tx, forceReload = false) {
1396
1514
  const committedBase = {
1397
1515
  type: `tx:committed`,
1398
1516
  term: tx.term,
@@ -1400,9 +1518,10 @@ class BroadcastCollectionCoordinator {
1400
1518
  txId: tx.txId,
1401
1519
  latestRowVersion: tx.rowVersion
1402
1520
  };
1403
- const committedPayload = tx.truncate ? {
1521
+ const committedPayload = tx.truncate || forceReload ? {
1404
1522
  ...committedBase,
1405
- requiresFullReload: true
1523
+ requiresFullReload: true,
1524
+ ...forceReload ? { reconciliationReload: true } : {}
1406
1525
  } : {
1407
1526
  ...committedBase,
1408
1527
  requiresFullReload: false,
@@ -1426,24 +1545,123 @@ class BroadcastCollectionCoordinator {
1426
1545
  for (const subscriber of state.subscribers) {
1427
1546
  subscriber(committed);
1428
1547
  }
1548
+ }
1549
+ async handleReconcileCommittedTx(collectionId, request, scopedAdapter) {
1550
+ const state = this.collections.get(collectionId);
1551
+ const notLeader = () => ({
1552
+ type: `rpc:reconcileCommittedTx:res`,
1553
+ rpcId: request.rpcId,
1554
+ ok: false,
1555
+ code: `NOT_LEADER`,
1556
+ error: `not the leader for ${collectionId}`
1557
+ });
1558
+ if (!state?.isLeader) return notLeader();
1559
+ if (!committedTxAnchor.isValidCommittedTxAnchor(request.anchor) || !hasReconciliationTxPayload(request.tx)) {
1560
+ return {
1561
+ type: `rpc:reconcileCommittedTx:res`,
1562
+ rpcId: request.rpcId,
1563
+ ok: false,
1564
+ code: `UNDETERMINED`,
1565
+ error: `the source transaction has no valid durable proof`
1566
+ };
1567
+ }
1568
+ const expectedTerm = state.latestTerm;
1569
+ const result = await this.withScheduledWriterLock(
1570
+ collectionId,
1571
+ async (adapter) => {
1572
+ const candidate = this.nextPersistedTxIfLeader(
1573
+ collectionId,
1574
+ state,
1575
+ expectedTerm,
1576
+ request.tx
1577
+ );
1578
+ if (candidate === null) return null;
1579
+ if (!adapter.reconcileCommittedTx) return { kind: `unknown` };
1580
+ try {
1581
+ const reconciled = await adapter.reconcileCommittedTx(
1582
+ candidate.cacheGenerationClaimId ? persisted.managedStorageTarget(
1583
+ collectionId,
1584
+ candidate.cacheGenerationClaimId
1585
+ ) : persisted.eagerStorageTarget(collectionId),
1586
+ candidate,
1587
+ request.anchor
1588
+ );
1589
+ if (reconciled.kind !== `unknown`) {
1590
+ state.latestRowVersion = Math.max(
1591
+ state.latestRowVersion,
1592
+ reconciled.latestRowVersion
1593
+ );
1594
+ if (reconciled.kind === `applied-now`) {
1595
+ state.latestSeq = reconciled.committed.seq;
1596
+ }
1597
+ }
1598
+ return reconciled;
1599
+ } catch (error) {
1600
+ throw errors.toPersistedCollectionDurabilityError(collectionId, error);
1601
+ }
1602
+ },
1603
+ scopedAdapter
1604
+ );
1605
+ if (result === null) return notLeader();
1606
+ if (result.kind === `unknown`) {
1607
+ return {
1608
+ type: `rpc:reconcileCommittedTx:res`,
1609
+ rpcId: request.rpcId,
1610
+ ok: false,
1611
+ code: `UNDETERMINED`,
1612
+ error: `the durable outcome cannot be certified`
1613
+ };
1614
+ }
1615
+ const current = {
1616
+ term: state.latestTerm,
1617
+ seq: state.latestSeq,
1618
+ rowVersion: state.latestRowVersion
1619
+ };
1620
+ const response = {
1621
+ type: `rpc:reconcileCommittedTx:res`,
1622
+ rpcId: request.rpcId,
1623
+ ok: true,
1624
+ alreadyApplied: result.kind === `already-applied`,
1625
+ committed: result.committed
1626
+ };
1627
+ if (!this.isDisposed() && this.collections.get(collectionId) === state) {
1628
+ this.publishCommittedTx(
1629
+ collectionId,
1630
+ state,
1631
+ result.kind === `already-applied` ? { ...request.tx, ...current } : { ...request.tx, ...result.committed },
1632
+ result.kind === `already-applied`
1633
+ );
1634
+ }
1429
1635
  return response;
1430
1636
  }
1637
+ nextPersistedTxIfLeader(collectionId, state, expectedTerm, pendingTx) {
1638
+ if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
1639
+ return null;
1640
+ }
1641
+ return {
1642
+ ...pendingTx,
1643
+ term: state.latestTerm,
1644
+ seq: state.latestSeq + 1,
1645
+ rowVersion: state.latestRowVersion + 1
1646
+ };
1647
+ }
1431
1648
  async applyDurablyAtNextStreamPosition(collectionId, state, pendingTx, scopedAdapter) {
1432
1649
  const expectedTerm = state.latestTerm;
1433
1650
  return this.withScheduledWriterLock(
1434
1651
  collectionId,
1435
1652
  async (adapter) => {
1436
- if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
1437
- return null;
1438
- }
1439
- const tx = {
1440
- ...pendingTx,
1441
- term: state.latestTerm,
1442
- seq: state.latestSeq + 1,
1443
- rowVersion: state.latestRowVersion + 1
1444
- };
1653
+ const tx = this.nextPersistedTxIfLeader(
1654
+ collectionId,
1655
+ state,
1656
+ expectedTerm,
1657
+ pendingTx
1658
+ );
1659
+ if (tx === null) return null;
1445
1660
  try {
1446
- await adapter.applyCommittedTx(collectionId, tx);
1661
+ await adapter.applyCommittedTx(
1662
+ tx.cacheGenerationClaimId ? persisted.managedStorageTarget(collectionId, tx.cacheGenerationClaimId) : persisted.eagerStorageTarget(collectionId),
1663
+ tx
1664
+ );
1447
1665
  } catch (error) {
1448
1666
  throw errors.toPersistedCollectionDurabilityError(collectionId, error);
1449
1667
  }
@@ -1468,7 +1686,13 @@ class BroadcastCollectionCoordinator {
1468
1686
  requiresFullReload: true
1469
1687
  };
1470
1688
  }
1471
- const result = await adapter.pullSince(collectionId, request.fromRowVersion);
1689
+ const result = await adapter.pullSince(
1690
+ request.cacheGenerationClaimId ? persisted.managedStorageTarget(collectionId, request.cacheGenerationClaimId) : persisted.eagerStorageTarget(collectionId),
1691
+ request.fromRowVersion,
1692
+ {
1693
+ cacheGenerationClaimId: request.cacheGenerationClaimId
1694
+ }
1695
+ );
1472
1696
  if (result.requiresFullReload) {
1473
1697
  return {
1474
1698
  type: `rpc:pullSince:res`,
@@ -1584,12 +1808,18 @@ function isRPCRequest(payload) {
1584
1808
  case `rpc:ensurePersistedIndex:req`:
1585
1809
  case `rpc:applyLocalMutations:req`:
1586
1810
  case `rpc:applyCommittedTx:req`:
1811
+ case `rpc:reconcileCommittedTx:req`:
1587
1812
  case `rpc:pullSince:req`:
1588
1813
  return true;
1589
1814
  default:
1590
1815
  return false;
1591
1816
  }
1592
1817
  }
1818
+ function hasReconciliationTxPayload(value) {
1819
+ if (!value || typeof value !== `object`) return false;
1820
+ const tx = value;
1821
+ return typeof tx.txId === `string` && Array.isArray(tx.mutations);
1822
+ }
1593
1823
  function sleep(ms) {
1594
1824
  return new Promise((resolve) => setTimeout(resolve, ms));
1595
1825
  }
@@ -1663,6 +1893,15 @@ function createRPCErrorResponse(request, cause) {
1663
1893
  code: `CONFLICT`,
1664
1894
  error
1665
1895
  };
1896
+ case `rpc:reconcileCommittedTx:req`:
1897
+ return {
1898
+ type: `rpc:reconcileCommittedTx:res`,
1899
+ rpcId: request.rpcId,
1900
+ ok: false,
1901
+ code: cause instanceof errors.PersistedCollectionDurabilityError ? `PERSISTENCE_ERROR` : `UNDETERMINED`,
1902
+ error,
1903
+ ...cause instanceof errors.PersistedCollectionDurabilityError ? toSafeDurabilityDetails(cause) : {}
1904
+ };
1666
1905
  case `rpc:pullSince:req`:
1667
1906
  return {
1668
1907
  type: `rpc:pullSince:res`,