@powerhousedao/reactor 6.2.3-dev.34 → 6.2.3-dev.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,7 +1,7 @@
1
- import { $ as OptimisticLockError, A as DEFAULT_DEFERRED_JOB_TTL_MS, B as summarizeSubmittedActions, C as SettledWatermark, D as snapshotFunctionsFor, E as parseSnapshot, F as ModuleNotFoundError, G as authDecisionModel, H as decideAtHead, I as GATED_DOCUMENT_ACTIONS, J as APPEND_CONDITION_FAILED_PREFIX, K as buildDecisionModel, L as createConsistencyToken, M as DuplicateManifestError, N as DuplicateModuleError, O as DocumentMetaCache, P as InvalidModuleError, Q as DuplicateOperationError, R as createEmptyConsistencyToken, S as ProbeSettler, T as describeWaitingSessions, U as selectDecisionModel, V as targetDocumentId, W as documentDecisionModel, X as DocumentAlreadyExistsError, Y as AppendConditionFailedError, Z as DocumentExistence, _ as KyselyExecutionScope, _t as throwIfAborted, a as DEFAULT_DRIVE_CONTAINER_TYPES, at as DocumentNotFoundError, b as KyselyWriteCache, c as runMigrations, ct as InvalidSignatureError, d as KyselyOperationStore, dt as UnsupportedStoredProtocolError, et as RevisionMismatchError, f as KyselyKeyframeStore, ft as UpgradePreconditionFailedError, g as PassthroughSigner, gt as parsePagingOptions, h as verifyActionSignature, ht as matchesScope, it as DocumentDeletedError, j as JobExecutorEventTypes, k as CollectionMembershipCache, l as createForwardingPoolInstrumentation, lt as RelationshipNotFoundError, m as SimpleJobExecutor, mt as ReactorEventTypes, n as errorToInfo, nt as AuthTimestampNotMonotonicError, o as REACTOR_SCHEMA, ot as ExcessiveReshuffleError, p as DocumentModelRegistry, pt as EventBusAggregateError, q as DriveCollectionId, r as sanitizeArg, rt as AuthorizationDeniedError, s as getMigrationStatus, st as InvalidOperationTimestampError, t as createForwardingLogger, tt as AuthEnforcementDisabledError, u as instrumentPgPool, ut as UnsupportedProtocolVersionError, v as resolveFeatureFlags, w as createKyselyWatermarkProbe, x as KyselyOperationIndex, y as EventBus, yt as __exportAll, z as submittedActionIds } from "./forwarding-logger-BkwlvD9W.js";
1
+ import { $ as selectDecisionModel, A as ProbeSettler, At as matchesScope, B as JobExecutorEventTypes, C as KyselyWriteCache, Ct as RelationshipNotFoundError, D as acquirePurgeLocks, Dt as UpgradePreconditionFailedError, E as PURGE_NS, Et as UnsupportedStoredProtocolError, F as snapshotFunctionsFor, G as GATED_DOCUMENT_ACTIONS, H as DuplicateModuleError, I as DocumentMetaCache, J as createEmptyConsistencyToken, K as createConsistencyToken, L as CollectionMembershipCache, M as createKyselyWatermarkProbe, Mt as throwIfAborted, N as describeWaitingSessions, O as findPurged, Ot as EventBusAggregateError, P as parseSnapshot, Pt as __exportAll, Q as decideAtHead, R as DEFAULT_DEFERRED_JOB_TTL_MS, S as EventBus, St as PurgeTooLargeError, T as PURGE_LOCK_BUCKETS, Tt as UnsupportedProtocolVersionError, U as InvalidModuleError, V as DuplicateManifestError, W as ModuleNotFoundError, X as summarizeSubmittedActions, Y as submittedActionIds, Z as targetDocumentId, _ as KyselyExecutionScope, _t as DocumentPurgedError, a as DEFAULT_DRIVE_CONTAINER_TYPES, at as AppendConditionFailedError, b as appliedDelete, bt as InvalidOperationTimestampError, c as runMigrations, ct as DuplicateOperationError, d as KyselyOperationStore, dt as AuthEnforcementDisabledError, et as documentDecisionModel, f as KyselyKeyframeStore, ft as AuthTimestampNotMonotonicError, g as PassthroughSigner, gt as DocumentNotFoundError, h as verifyActionSignature, ht as DocumentNotDeletedError, it as APPEND_CONDITION_FAILED_PREFIX, j as SettledWatermark, jt as parsePagingOptions, k as listPurged, kt as ReactorEventTypes, l as createForwardingPoolInstrumentation, lt as OptimisticLockError, m as SimpleJobExecutor, mt as DocumentDeletedError, n as errorToInfo, nt as buildDecisionModel, o as REACTOR_SCHEMA, ot as DocumentAlreadyExistsError, p as DocumentModelRegistry, pt as AuthorizationDeniedError, q as createDocumentFromAction, r as sanitizeArg, rt as DriveCollectionId, s as getMigrationStatus, st as DocumentExistence, t as createForwardingLogger, tt as authDecisionModel, u as instrumentPgPool, ut as RevisionMismatchError, v as DEFAULT_PURGE_DELETE_BATCH, vt as ExcessiveReshuffleError, w as KyselyOperationIndex, wt as ReservedActionError, x as resolveFeatureFlags, xt as InvalidSignatureError, y as KyselyDocumentPurger, yt as GroupInUseError, z as DEFAULT_MAX_PURGE_OPERATIONS } from "./forwarding-logger-DMqDs9tE.js";
2
2
  import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-BM_ULVWd.js";
3
- import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-I6i_YzsR.js";
4
- import { DEFAULT_SIGNATURE_POLICY, DowngradeNotSupportedError, MAX_AUTH_GRANTS, PEER_CAPABILITIES, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
3
+ import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-CZqIpdz2.js";
4
+ import { DEFAULT_SIGNATURE_POLICY, DOCUMENT_PURGE_PROTOCOL, DowngradeNotSupportedError, MARKER_REFUSAL_FEATURE, MAX_AUTH_GRANTS, PEER_CAPABILITIES, PURGE_DOCUMENT, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, isPurgeMarker, isPurgeMarker as isPurgeMarker$1, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, purgedProtocolVersions, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
5
5
  import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
6
6
  import { v4 } from "uuid";
7
7
  import { ConsoleLogger, childLogger } from "document-model";
@@ -326,11 +326,13 @@ var DriveClient = class {
326
326
  logger;
327
327
  reactor;
328
328
  signer;
329
- constructor(client, logger, reactor, signer) {
329
+ resolveReference;
330
+ constructor(client, logger, reactor, signer, resolveReference) {
330
331
  this.client = client;
331
332
  this.logger = logger;
332
333
  this.reactor = reactor;
333
334
  this.signer = signer;
335
+ this.resolveReference = resolveReference;
334
336
  }
335
337
  async create(input, signal) {
336
338
  this.logger.verbose("drives.create(@input)", input);
@@ -351,6 +353,7 @@ var DriveClient = class {
351
353
  async addFile(driveIdentifier, document, parentFolder, signal) {
352
354
  this.logger.verbose("drives.addFile(@driveIdentifier, @document, @parentFolder)", driveIdentifier, document.header.id, parentFolder);
353
355
  const documentId = document.header.id;
356
+ const driveId = await this.resolveReference(driveIdentifier, signal);
354
357
  const createInput = {
355
358
  model: document.header.documentType,
356
359
  version: 0,
@@ -377,7 +380,7 @@ var DriveClient = class {
377
380
  toVersion: normalizeDocumentModelVersion(document.state.document?.version),
378
381
  initialState: document.state
379
382
  }),
380
- addRelationshipAction(driveIdentifier, documentId, "child")
383
+ addRelationshipAction(driveId, documentId, "child")
381
384
  ], this.signer, {
382
385
  documentId,
383
386
  branch: "main"
@@ -388,7 +391,7 @@ var DriveClient = class {
388
391
  documentType: document.header.documentType,
389
392
  parentFolder
390
393
  })], this.signer, {
391
- documentId: driveIdentifier,
394
+ documentId: driveId,
392
395
  branch: "main"
393
396
  }, signal);
394
397
  await this.runJobs([{
@@ -401,7 +404,7 @@ var DriveClient = class {
401
404
  }], signal);
402
405
  await this.runJobs([{
403
406
  key: "drive",
404
- documentId: driveIdentifier,
407
+ documentId: driveId,
405
408
  scope: getSharedActionScope(driveActions),
406
409
  branch: "main",
407
410
  actions: driveActions,
@@ -420,22 +423,24 @@ var DriveClient = class {
420
423
  if (!node || !isFolderNode(node)) throw new Error("Folder creation failed");
421
424
  return node;
422
425
  }
423
- async removeNode(driveIdentifier, nodeId, signal) {
424
- this.logger.verbose("drives.removeNode(@driveIdentifier, @nodeId)", driveIdentifier, nodeId);
425
- const drive = await this.client.get(driveIdentifier, void 0, signal);
426
+ async removeNode(driveIdentifier, nodeIdentifier, signal) {
427
+ this.logger.verbose("drives.removeNode(@driveIdentifier, @nodeIdentifier)", driveIdentifier, nodeIdentifier);
428
+ const driveId = await this.resolveReference(driveIdentifier, signal);
429
+ const nodeId = await this.resolveReference(nodeIdentifier, signal);
430
+ const drive = await this.client.get(driveId, void 0, signal);
426
431
  const node = drive.state.global.nodes.find((n) => n.id === nodeId);
427
432
  if (!node) {
428
433
  if (!await this.documentExists(nodeId, signal)) throw new Error(`Node ${nodeId} not found in drive ${driveIdentifier}`);
429
- await this.removeFileNode(driveIdentifier, nodeId, signal);
434
+ await this.removeFileNode(driveId, nodeId, signal);
430
435
  return;
431
436
  }
432
437
  if (isFolderNode(node)) {
433
438
  const fileDescendants = getDescendants(node, drive.state.global.nodes).filter(isFileNode);
434
- for (const file of fileDescendants) await this.removeFileNode(driveIdentifier, file.id, signal);
435
- await this.client.execute(driveIdentifier, "main", [deleteNode({ id: nodeId })], signal);
439
+ for (const file of fileDescendants) await this.removeFileNode(driveId, file.id, signal);
440
+ await this.client.execute(driveId, "main", [deleteNode({ id: nodeId })], signal);
436
441
  return;
437
442
  }
438
- await this.removeFileNode(driveIdentifier, nodeId, signal);
443
+ await this.removeFileNode(driveId, nodeId, signal);
439
444
  }
440
445
  async renameNode(driveIdentifier, nodeId, name, signal) {
441
446
  this.logger.verbose("drives.renameNode(@driveIdentifier, @nodeId, @name)", driveIdentifier, nodeId, name);
@@ -546,34 +551,41 @@ var DriveClient = class {
546
551
  }
547
552
  }
548
553
  async removeFileNode(driveId, fileId, signal) {
549
- const relationshipActions = await signActions([removeRelationshipAction(driveId, fileId, "child")], this.signer, {
554
+ const driveActions = await signActions([deleteNode({ id: fileId })], this.signer, {
550
555
  documentId: driveId,
551
556
  branch: "main"
552
557
  }, signal);
553
- const driveActions = await signActions([deleteNode({ id: fileId })], this.signer, {
558
+ await this.runJobs([{
559
+ key: "drive",
554
560
  documentId: driveId,
561
+ scope: getSharedActionScope(driveActions),
562
+ branch: "main",
563
+ actions: driveActions,
564
+ dependsOn: []
565
+ }], signal);
566
+ const deleteActions = await signActions([deleteDocumentAction(fileId)], this.signer, {
567
+ documentId: fileId,
555
568
  branch: "main"
556
569
  }, signal);
557
- const batchResult = await this.reactor.executeBatch({ jobs: [{
558
- key: "relationship",
570
+ const relationshipActions = await signActions([removeRelationshipAction(driveId, fileId, "child")], this.signer, {
559
571
  documentId: driveId,
560
- scope: getSharedActionScope(relationshipActions),
572
+ branch: "main"
573
+ }, signal);
574
+ await this.runJobs([{
575
+ key: "delete",
576
+ documentId: fileId,
577
+ scope: getSharedActionScope(deleteActions),
561
578
  branch: "main",
562
- actions: relationshipActions,
579
+ actions: deleteActions,
563
580
  dependsOn: []
564
581
  }, {
565
- key: "drive",
582
+ key: "relationship",
566
583
  documentId: driveId,
567
- scope: getSharedActionScope(driveActions),
584
+ scope: getSharedActionScope(relationshipActions),
568
585
  branch: "main",
569
- actions: driveActions,
570
- dependsOn: ["relationship"]
571
- }] }, signal);
572
- const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.client.waitForJob(job, signal)));
573
- for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
574
- const deleteJob = await this.reactor.deleteDocument(fileId, this.signer, signal);
575
- const deleteCompleted = await this.client.waitForJob(deleteJob, signal);
576
- if (deleteCompleted.status === "FAILED") throw new Error(deleteCompleted.error?.message);
586
+ actions: relationshipActions,
587
+ dependsOn: ["delete"]
588
+ }], signal);
577
589
  }
578
590
  };
579
591
  //#endregion
@@ -849,8 +861,8 @@ var BareReadGate = class {
849
861
  }
850
862
  };
851
863
  /**
852
- * Rethrows a failed read unless the document is confirmed not live. The read
853
- * side reports absence as a plain Error, so absence is checked rather than
864
+ * Rethrows a failed read unless the document is confirmed not live. Any error
865
+ * other than DocumentNotFoundError is checked against `exists` rather than
854
866
  * inferred from the message, and a failed check surfaces the read's own error.
855
867
  */
856
868
  async function assertAbsent(documentView, documentId, error, signal) {
@@ -885,7 +897,7 @@ var SeededStateReader = class {
885
897
  * A stream this replica does not hold has to reach buildDecisionModel as the
886
898
  * absence it recognises, or the whole read fails instead of leaving the group
887
899
  * out of the model, where its principal does not match and the policy fails
888
- * closed. The read side reports absence as a plain Error, so the absence is
900
+ * closed. An error other than DocumentNotFoundError has its absence
889
901
  * confirmed rather than inferred from the message: a transient failure must
890
902
  * surface, not silently deny.
891
903
  */
@@ -1203,6 +1215,9 @@ function keepReadableScopes(scopes, readable) {
1203
1215
  }
1204
1216
  //#endregion
1205
1217
  //#region src/client/reactor-client.ts
1218
+ function deletePlanKey(documentId) {
1219
+ return `delete:${documentId}`;
1220
+ }
1206
1221
  /**
1207
1222
  * The document a candidate is decided against. Routed on the action type alone,
1208
1223
  * which is what the executor routes on: every action the reactor reduces onto
@@ -1275,7 +1290,7 @@ var ReactorClient = class {
1275
1290
  this.eventReads = new EventReadsSource(reactor, documentView, readGate);
1276
1291
  this.actionEvaluation = actionEvaluation;
1277
1292
  this.createSignaturePolicy = createSignaturePolicy;
1278
- this.drives = new DriveClient(this, logger, reactor, signer);
1293
+ this.drives = new DriveClient(this, logger, reactor, signer, (id, signal) => this.resolveReference(id, "main", signal));
1279
1294
  this.logger.verbose("ReactorClient initialized");
1280
1295
  }
1281
1296
  /**
@@ -1523,8 +1538,23 @@ var ReactorClient = class {
1523
1538
  */
1524
1539
  async create(document, parentIdentifier, signal) {
1525
1540
  this.logger.verbose("create(@id, @parentIdentifier)", document.header.id, parentIdentifier);
1541
+ const batchResult = await this.submitCreate(document, parentIdentifier, signal);
1542
+ const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
1543
+ for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
1544
+ const created = await this.reactor.get(document.header.id);
1545
+ return this.gateDocument(created, void 0, signal);
1546
+ }
1547
+ /**
1548
+ * Submits a document's create batch without waiting for it
1549
+ */
1550
+ async createAsync(document, parentIdentifier, signal) {
1551
+ this.logger.verbose("createAsync(@id, @parentIdentifier)", document.header.id, parentIdentifier);
1552
+ return this.submitCreate(document, parentIdentifier, signal);
1553
+ }
1554
+ async submitCreate(document, parentIdentifier, signal) {
1526
1555
  const documentId = document.header.id;
1527
1556
  const branch = document.header.branch || "main";
1557
+ const parentId = parentIdentifier ? await this.resolveReference(parentIdentifier, "main", signal) : void 0;
1528
1558
  const createInput = {
1529
1559
  model: document.header.documentType,
1530
1560
  version: 0,
@@ -1560,31 +1590,39 @@ var ReactorClient = class {
1560
1590
  actions: createActions,
1561
1591
  dependsOn: []
1562
1592
  }];
1563
- if (parentIdentifier) {
1564
- const parentActions = await signActions([addRelationshipAction(parentIdentifier, documentId, "child")], this.signer, {
1565
- documentId: parentIdentifier,
1593
+ if (parentId) {
1594
+ const parentActions = await signActions([addRelationshipAction(parentId, documentId, "child")], this.signer, {
1595
+ documentId: parentId,
1566
1596
  branch: "main"
1567
1597
  }, signal);
1568
1598
  jobs.push({
1569
1599
  key: "parent",
1570
- documentId: parentIdentifier,
1600
+ documentId: parentId,
1571
1601
  scope: getSharedActionScope(parentActions),
1572
1602
  branch: "main",
1573
1603
  actions: parentActions,
1574
1604
  dependsOn: ["create"]
1575
1605
  });
1576
1606
  }
1577
- const batchResult = await this.reactor.executeBatch({ jobs }, signal);
1578
- const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
1579
- for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
1580
- const created = await this.reactor.get(documentId);
1581
- return this.gateDocument(created, void 0, signal);
1607
+ return this.reactor.executeBatch({ jobs }, signal);
1582
1608
  }
1583
1609
  /**
1584
1610
  * Creates an empty document and waits for completion
1585
1611
  */
1586
1612
  async createEmpty(documentModelType, options, signal) {
1587
1613
  this.logger.verbose("createEmpty(@documentModelType, @options)", documentModelType, options);
1614
+ const document = await this.emptyDocument(documentModelType, options, signal);
1615
+ return this.create(document, options?.parentIdentifier, signal);
1616
+ }
1617
+ /**
1618
+ * Submits an empty document's create batch without waiting for it
1619
+ */
1620
+ async createEmptyAsync(documentModelType, options, signal) {
1621
+ this.logger.verbose("createEmptyAsync(@documentModelType, @options)", documentModelType, options);
1622
+ const document = await this.emptyDocument(documentModelType, options, signal);
1623
+ return this.submitCreate(document, options?.parentIdentifier, signal);
1624
+ }
1625
+ async emptyDocument(documentModelType, options, signal) {
1588
1626
  const matchingModules = (await this.reactor.getDocumentModels(void 0, void 0, signal)).results.filter((m) => m.documentModel.global.id === documentModelType);
1589
1627
  let module;
1590
1628
  if (options?.documentModelVersion !== void 0) {
@@ -1604,7 +1642,7 @@ var ReactorClient = class {
1604
1642
  ...options?.protocolVersions
1605
1643
  } });
1606
1644
  document.state.document.version = normalizeDocumentModelVersion(module.version);
1607
- return this.create(document, options?.parentIdentifier, signal);
1645
+ return document;
1608
1646
  }
1609
1647
  /**
1610
1648
  * Upgrades a document to a newer document model version by dispatching an
@@ -1739,10 +1777,12 @@ var ReactorClient = class {
1739
1777
  */
1740
1778
  async addRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch = "main", signal) {
1741
1779
  this.logger.verbose("addRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @metadata, @branch)", sourceIdentifier, targetIdentifier, relationshipType, metadata, branch);
1742
- const jobInfo = await this.reactor.addRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch, this.signer, signal);
1780
+ const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
1781
+ const targetId = await this.resolveReference(targetIdentifier, branch, signal);
1782
+ const jobInfo = await this.reactor.addRelationship(sourceId, targetId, relationshipType, metadata, branch, this.signer, signal);
1743
1783
  const completedJob = await this.waitForJob(jobInfo, signal);
1744
1784
  if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
1745
- const result = await this.reactor.getByIdOrSlug(sourceIdentifier, { branch }, completedJob.consistencyToken, signal);
1785
+ const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
1746
1786
  return this.gateDocument(result, { branch }, signal);
1747
1787
  }
1748
1788
  /**
@@ -1750,11 +1790,13 @@ var ReactorClient = class {
1750
1790
  */
1751
1791
  async updateRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch = "main", signal) {
1752
1792
  this.logger.verbose("updateRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @metadata, @branch)", sourceIdentifier, targetIdentifier, relationshipType, metadata, branch);
1753
- if (!await this.readRelationshipEdge(sourceIdentifier, targetIdentifier, relationshipType, { branch }, signal)) throw new RelationshipNotFoundError(sourceIdentifier, targetIdentifier, relationshipType);
1754
- const jobInfo = await this.reactor.updateRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch, this.signer, signal);
1793
+ const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
1794
+ const targetId = await this.resolveReference(targetIdentifier, branch, signal);
1795
+ if (!await this.readRelationshipEdge(sourceId, targetId, relationshipType, { branch }, signal)) throw new RelationshipNotFoundError(sourceIdentifier, targetIdentifier, relationshipType);
1796
+ const jobInfo = await this.reactor.updateRelationship(sourceId, targetId, relationshipType, metadata, branch, this.signer, signal);
1755
1797
  const completedJob = await this.waitForJob(jobInfo, signal);
1756
1798
  if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
1757
- const result = await this.reactor.getByIdOrSlug(sourceIdentifier, { branch }, completedJob.consistencyToken, signal);
1799
+ const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
1758
1800
  return this.gateDocument(result, { branch }, signal);
1759
1801
  }
1760
1802
  /**
@@ -1762,10 +1804,12 @@ var ReactorClient = class {
1762
1804
  */
1763
1805
  async removeRelationship(sourceIdentifier, targetIdentifier, relationshipType, branch = "main", signal) {
1764
1806
  this.logger.verbose("removeRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @branch)", sourceIdentifier, targetIdentifier, relationshipType, branch);
1765
- const jobInfo = await this.reactor.removeRelationship(sourceIdentifier, targetIdentifier, relationshipType, branch, this.signer, signal);
1807
+ const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
1808
+ const targetId = await this.resolveReference(targetIdentifier, branch, signal);
1809
+ const jobInfo = await this.reactor.removeRelationship(sourceId, targetId, relationshipType, branch, this.signer, signal);
1766
1810
  const completedJob = await this.waitForJob(jobInfo, signal);
1767
1811
  if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
1768
- const result = await this.reactor.getByIdOrSlug(sourceIdentifier, { branch }, completedJob.consistencyToken, signal);
1812
+ const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
1769
1813
  return this.gateDocument(result, { branch }, signal);
1770
1814
  }
1771
1815
  /**
@@ -1773,15 +1817,18 @@ var ReactorClient = class {
1773
1817
  */
1774
1818
  async moveRelationship(sourceParentIdentifier, targetParentIdentifier, targetIdentifier, relationshipType, branch = "main", signal) {
1775
1819
  this.logger.verbose("moveRelationship(@sourceParentIdentifier, @targetParentIdentifier, @targetIdentifier, @relationshipType, @branch)", sourceParentIdentifier, targetParentIdentifier, targetIdentifier, relationshipType, branch);
1776
- const metadata = (await this.readRelationshipEdge(sourceParentIdentifier, targetIdentifier, relationshipType, { branch }, signal))?.metadata;
1777
- const removeJobInfo = await this.reactor.removeRelationship(sourceParentIdentifier, targetIdentifier, relationshipType, branch, this.signer, signal);
1820
+ const sourceParentId = await this.resolveReference(sourceParentIdentifier, branch, signal);
1821
+ const targetParentId = await this.resolveReference(targetParentIdentifier, branch, signal);
1822
+ const targetId = await this.resolveReference(targetIdentifier, branch, signal);
1823
+ const metadata = (await this.readRelationshipEdge(sourceParentId, targetId, relationshipType, { branch }, signal))?.metadata;
1824
+ const removeJobInfo = await this.reactor.removeRelationship(sourceParentId, targetId, relationshipType, branch, this.signer, signal);
1778
1825
  const removeCompletedJob = await this.waitForJob(removeJobInfo, signal);
1779
1826
  if (removeCompletedJob.status === "FAILED") throw new Error(removeCompletedJob.error?.message);
1780
- const addJobInfo = await this.reactor.addRelationship(targetParentIdentifier, targetIdentifier, relationshipType, metadata, branch, this.signer, signal);
1827
+ const addJobInfo = await this.reactor.addRelationship(targetParentId, targetId, relationshipType, metadata, branch, this.signer, signal);
1781
1828
  const addCompletedJob = await this.waitForJob(addJobInfo, signal);
1782
1829
  if (addCompletedJob.status === "FAILED") throw new Error(addCompletedJob.error?.message);
1783
- const sourceResult = await this.reactor.getByIdOrSlug(sourceParentIdentifier, { branch }, removeCompletedJob.consistencyToken, signal);
1784
- const targetResult = await this.reactor.getByIdOrSlug(targetParentIdentifier, { branch }, addCompletedJob.consistencyToken, signal);
1830
+ const sourceResult = await this.reactor.getByIdOrSlug(sourceParentId, { branch }, removeCompletedJob.consistencyToken, signal);
1831
+ const targetResult = await this.reactor.getByIdOrSlug(targetParentId, { branch }, addCompletedJob.consistencyToken, signal);
1785
1832
  return {
1786
1833
  source: await this.gateDocument(sourceResult, { branch }, signal),
1787
1834
  target: await this.gateDocument(targetResult, { branch }, signal)
@@ -1801,9 +1848,9 @@ var ReactorClient = class {
1801
1848
  */
1802
1849
  async deleteDocument(identifier, propagate, signal) {
1803
1850
  this.logger.verbose("deleteDocument(@identifier, @propagate)", identifier, propagate);
1804
- const jobs = [];
1851
+ const rootId = await this.resolveReference(identifier, "main", signal);
1852
+ const toDelete = /* @__PURE__ */ new Set([rootId]);
1805
1853
  if (propagate === "cascade") {
1806
- const toDelete = /* @__PURE__ */ new Set([identifier]);
1807
1854
  let changed = true;
1808
1855
  while (changed) {
1809
1856
  if (signal?.aborted) throw new Error("Operation aborted");
@@ -1814,19 +1861,33 @@ var ReactorClient = class {
1814
1861
  changed = true;
1815
1862
  }
1816
1863
  }
1817
- for (const descendantId of toDelete) {
1818
- if (descendantId === identifier) continue;
1819
- const removalJobs = await this.removeAllIncomingRelationships(descendantId, signal);
1820
- jobs.push(...removalJobs);
1821
- const jobInfo = await this.reactor.deleteDocument(descendantId, this.signer, signal);
1822
- jobs.push(jobInfo);
1823
- }
1824
1864
  }
1825
- const removalJobs = await this.removeAllIncomingRelationships(identifier, signal);
1826
- jobs.push(...removalJobs);
1827
- const jobInfo = await this.reactor.deleteDocument(identifier, this.signer, signal);
1828
- jobs.push(jobInfo);
1829
- const completedJobs = await Promise.all(jobs.map((job) => this.waitForJob(job, signal)));
1865
+ const order = [...toDelete].reverse();
1866
+ const position = new Map(order.map((id, index) => [id, index]));
1867
+ const incoming = /* @__PURE__ */ new Map();
1868
+ for (const documentId of order) {
1869
+ const page = await this.documentIndexer.getIncoming(documentId, void 0, void 0, void 0, signal);
1870
+ incoming.set(documentId, page.results);
1871
+ }
1872
+ const waitsForTarget = (rel) => {
1873
+ const source = position.get(rel.sourceId);
1874
+ return source === void 0 || source > position.get(rel.targetId);
1875
+ };
1876
+ const leadingBySource = /* @__PURE__ */ new Map();
1877
+ for (const rels of incoming.values()) for (const rel of rels) {
1878
+ if (waitsForTarget(rel)) continue;
1879
+ const leading = leadingBySource.get(rel.sourceId) ?? [];
1880
+ leading.push(rel);
1881
+ leadingBySource.set(rel.sourceId, leading);
1882
+ }
1883
+ const plans = [];
1884
+ for (const documentId of order) {
1885
+ for (const rel of leadingBySource.get(documentId) ?? []) plans.push(await this.planRemoval(rel, [], plans.length, signal));
1886
+ plans.push(await this.planDelete(documentId, signal));
1887
+ for (const rel of incoming.get(documentId)) if (waitsForTarget(rel)) plans.push(await this.planRemoval(rel, [deletePlanKey(documentId)], plans.length, signal));
1888
+ }
1889
+ const batchResult = await this.reactor.executeBatch({ jobs: plans }, signal);
1890
+ const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
1830
1891
  for (const completedJob of completedJobs) if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
1831
1892
  }
1832
1893
  /**
@@ -1896,12 +1957,16 @@ var ReactorClient = class {
1896
1957
  return served("created", documents, reads);
1897
1958
  })());
1898
1959
  }, search);
1899
- const unsubscribeDeleted = this.subscriptionManager.onDocumentDeleted((documentIds) => {
1960
+ const unsubscribeDeleted = this.subscriptionManager.onDocumentDeleted((documentIds, info) => {
1900
1961
  const reads = this.eventReads.forEvent();
1962
+ const purged = info?.purged ? { purged: true } : {};
1901
1963
  for (const childId of documentIds) deliver((async () => await this.servesEvery([childId], view, reads) ? {
1902
1964
  type: "deleted",
1903
1965
  documents: [],
1904
- context: { childId }
1966
+ context: {
1967
+ childId,
1968
+ ...purged
1969
+ }
1905
1970
  } : void 0)());
1906
1971
  }, search);
1907
1972
  const unsubscribeUpdated = this.subscriptionManager.onDocumentStateUpdated((result) => {
@@ -2102,11 +2167,18 @@ var ReactorClient = class {
2102
2167
  }
2103
2168
  /**
2104
2169
  * The id a write on `identifier` is stored under, which its signatures bind.
2105
- * A create names its own id, and an id no slug maps to is taken as given, so
2106
- * a document still in flight resolves to itself.
2170
+ * A create names its own id.
2107
2171
  */
2108
2172
  async resolveWriteTarget(identifier, branch, actions, signal) {
2109
2173
  if (actions.some((action) => action.type === "CREATE_DOCUMENT")) return identifier;
2174
+ return this.resolveReference(identifier, branch, signal);
2175
+ }
2176
+ /**
2177
+ * The id a document reference names. An id no slug maps to is taken as
2178
+ * given, so a document still in flight resolves to itself; an id that is
2179
+ * another document's slug is refused as ambiguous.
2180
+ */
2181
+ async resolveReference(identifier, branch, signal) {
2110
2182
  const view = { branch };
2111
2183
  const bySlug = await this.documentView.resolveSlug(identifier, view, void 0, signal);
2112
2184
  if (bySlug === void 0 || bySlug === identifier) return identifier;
@@ -2145,14 +2217,33 @@ var ReactorClient = class {
2145
2217
  scopeStates: document.state ?? {}
2146
2218
  };
2147
2219
  }
2148
- async removeAllIncomingRelationships(documentId, signal) {
2149
- const incoming = await this.documentIndexer.getIncoming(documentId, void 0, void 0, void 0, signal);
2150
- const jobs = [];
2151
- for (const rel of incoming.results) {
2152
- const jobInfo = await this.reactor.removeRelationship(rel.sourceId, documentId, rel.relationshipType, "main", this.signer, signal);
2153
- jobs.push(jobInfo);
2154
- }
2155
- return jobs;
2220
+ async planDelete(documentId, signal) {
2221
+ const signed = await signActions([deleteDocumentAction(documentId)], this.signer, {
2222
+ documentId,
2223
+ branch: "main"
2224
+ }, signal);
2225
+ return {
2226
+ key: deletePlanKey(documentId),
2227
+ documentId,
2228
+ scope: "document",
2229
+ branch: "main",
2230
+ actions: signed,
2231
+ dependsOn: []
2232
+ };
2233
+ }
2234
+ async planRemoval(rel, dependsOn, index, signal) {
2235
+ const signed = await signActions([removeRelationshipAction(rel.sourceId, rel.targetId, rel.relationshipType)], this.signer, {
2236
+ documentId: rel.sourceId,
2237
+ branch: "main"
2238
+ }, signal);
2239
+ return {
2240
+ key: `remove:${index}`,
2241
+ documentId: rel.sourceId,
2242
+ scope: "document",
2243
+ branch: "main",
2244
+ actions: signed,
2245
+ dependsOn
2246
+ };
2156
2247
  }
2157
2248
  };
2158
2249
  //#endregion
@@ -2178,11 +2269,15 @@ const QueueEventTypes = { JOB_AVAILABLE: 1e4 };
2178
2269
  /** Conflict retries a job may take without charging its retry limit. */
2179
2270
  const MAX_EXEMPT_CONFLICT_RETRIES = 20;
2180
2271
  function toErrorInfo(error) {
2181
- if (error instanceof Error) return {
2182
- name: error.name,
2183
- message: error.message,
2184
- stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
2185
- };
2272
+ if (error instanceof Error) {
2273
+ const documentId = error.documentId;
2274
+ return {
2275
+ name: error.name,
2276
+ message: error.message,
2277
+ stack: error.stack || (/* @__PURE__ */ new Error()).stack || "",
2278
+ ...typeof documentId === "string" ? { documentId } : {}
2279
+ };
2280
+ }
2186
2281
  return {
2187
2282
  name: "Error",
2188
2283
  message: error,
@@ -2229,12 +2324,12 @@ var JobResultHandler = class {
2229
2324
  return;
2230
2325
  } catch {}
2231
2326
  }
2232
- if (result.error && DocumentNotFoundError.isError(result.error) && handle.job.kind === "load") {
2327
+ if (result.error && DocumentNotFoundError.isError(result.error) && !DocumentPurgedError.isError(result.error) && handle.job.kind === "load") {
2233
2328
  handle.defer();
2234
2329
  callbacks.deferJob(result.error.documentId, handle.job);
2235
2330
  return;
2236
2331
  }
2237
- if (result.error && (DocumentDeletedError.isError(result.error) || DocumentAlreadyExistsError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || InvalidSignatureError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error) || UnsupportedProtocolVersionError.isError(result.error))) {
2332
+ if (result.error && (DocumentDeletedError.isError(result.error) || DocumentAlreadyExistsError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || InvalidSignatureError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error) || UnsupportedProtocolVersionError.isError(result.error) || DocumentPurgedError.isError(result.error) || DocumentNotDeletedError.isError(result.error) || GroupInUseError.isError(result.error) || PurgeTooLargeError.isError(result.error) || ReservedActionError.isError(result.error))) {
2238
2333
  const errorInfo = toErrorInfo(result.error);
2239
2334
  this.jobTracker.markFailed(handle.job.id, errorInfo, handle.job);
2240
2335
  this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
@@ -2654,9 +2749,12 @@ var WorkerPoolJobExecutorManager = class {
2654
2749
  };
2655
2750
  this.eventBus.emit(JobExecutorEventTypes.JOB_FAILED, failedEvent).catch(() => {});
2656
2751
  }
2657
- if (outcome.result.success && outcome.writeReady) this.emitWriteReady(handle.job, outcome.writeReady).catch((error) => {
2658
- this.logger.error("emitWriteReady failed for job @jobId: @Error", { jobId: handle.job.id }, error);
2659
- });
2752
+ if (outcome.result.success && outcome.writeReady) {
2753
+ this.evictPurged(outcome.writeReady);
2754
+ this.emitWriteReady(handle.job, outcome.writeReady).catch((error) => {
2755
+ this.logger.error("emitWriteReady failed for job @jobId: @Error", { jobId: handle.job.id }, error);
2756
+ });
2757
+ }
2660
2758
  await this.resultHandler.handleResult(handle, outcome.result, {
2661
2759
  deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
2662
2760
  flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
@@ -2664,6 +2762,12 @@ var WorkerPoolJobExecutorManager = class {
2664
2762
  this.activeJobs--;
2665
2763
  await this.tryDispatchFor(worker);
2666
2764
  }
2765
+ /** Every worker drops a committed purge's id; routing covers only its own. */
2766
+ evictPurged(payload) {
2767
+ const documentIds = [...new Set(payload.operations.filter((operation) => isPurgeMarker$1(operation)).map((operation) => operation.context.documentId))];
2768
+ if (documentIds.length === 0) return;
2769
+ for (const worker of this.workers) worker.evictPurged(documentIds);
2770
+ }
2667
2771
  async emitWriteReady(job, payload) {
2668
2772
  const documentIds = [...new Set(payload.operations.map((op) => op.context.documentId))];
2669
2773
  let collectionMemberships = {};
@@ -2755,8 +2859,78 @@ function fillMissingMemberships(documentIds, found) {
2755
2859
  return result;
2756
2860
  }
2757
2861
  //#endregion
2862
+ //#region src/admin/document-purge-service.ts
2863
+ /** The only local origin of a purge: one purge job per erased document. */
2864
+ var DocumentPurgeService = class DocumentPurgeService {
2865
+ queue;
2866
+ jobTracker;
2867
+ eventBus;
2868
+ constructor(queue, jobTracker, eventBus) {
2869
+ this.queue = queue;
2870
+ this.jobTracker = jobTracker;
2871
+ this.eventBus = eventBus;
2872
+ }
2873
+ static fromModule(module) {
2874
+ return new DocumentPurgeService(module.queue, module.jobTracker, module.eventBus);
2875
+ }
2876
+ /** One purge job per id; resolves once enqueued, not once complete. */
2877
+ async enqueuePurge(ids, requestId, opts = {}) {
2878
+ const requestDocumentIds = [...new Set(ids)];
2879
+ const infos = [];
2880
+ for (const documentId of requestDocumentIds) {
2881
+ const jobId = v4();
2882
+ const createdAtUtcIso = (/* @__PURE__ */ new Date()).toISOString();
2883
+ const meta = buildSingleJobMeta(jobId, {
2884
+ purgeRequestId: requestId,
2885
+ purgeRequestDocumentIds: requestDocumentIds
2886
+ });
2887
+ const job = {
2888
+ id: jobId,
2889
+ kind: "purge",
2890
+ documentId,
2891
+ scope: "document",
2892
+ branch: "main",
2893
+ actions: [],
2894
+ operations: [],
2895
+ createdAt: createdAtUtcIso,
2896
+ queueHint: [],
2897
+ maxRetries: 3,
2898
+ errorHistory: [],
2899
+ meta,
2900
+ purge: {
2901
+ requestId,
2902
+ allowLarge: opts.allowLarge ?? false
2903
+ }
2904
+ };
2905
+ const info = {
2906
+ id: jobId,
2907
+ documentId,
2908
+ status: "PENDING",
2909
+ createdAtUtcIso,
2910
+ consistencyToken: {
2911
+ version: 1,
2912
+ createdAtUtcIso,
2913
+ coordinates: []
2914
+ },
2915
+ meta
2916
+ };
2917
+ this.jobTracker.registerJob(info);
2918
+ const pending = {
2919
+ jobId,
2920
+ jobMeta: meta
2921
+ };
2922
+ this.eventBus.emit(ReactorEventTypes.JOB_PENDING, pending).catch(() => {});
2923
+ const dispatched = this.queue.enqueue(job);
2924
+ dispatched.catch(() => {});
2925
+ await Promise.race([dispatched, Promise.resolve()]);
2926
+ infos.push(info);
2927
+ }
2928
+ return infos;
2929
+ }
2930
+ };
2931
+ //#endregion
2758
2932
  //#region src/storage/kysely/stored-protocol-versions.ts
2759
- const created = sql.raw(`scope = 'document' and "index" = 0 and (action->'input'->'protocolVersions') is not null`);
2933
+ const created = sql.raw(`scope = 'document' and "index" = 0 and (action->'input'->'protocolVersions') is not null and action->>'type' <> 'PURGE_DOCUMENT'`);
2760
2934
  const key = sql.raw(`md5((action->'input'->'protocolVersions')::text)`);
2761
2935
  /** Distinct creation protocolVersions, one index probe per value. */
2762
2936
  async function storedProtocolVersions(db, schema) {
@@ -3108,14 +3282,23 @@ var InMemoryJobTracker = class {
3108
3282
  }
3109
3283
  };
3110
3284
  //#endregion
3285
+ //#region src/read-models/types.ts
3286
+ /** A cursor row whose factory is unregistered; it only records a drive's data. */
3287
+ const RELEASED_CURSOR_STATUS = "released";
3288
+ //#endregion
3111
3289
  //#region src/processors/utils.ts
3112
3290
  function isDriveDeletion(op) {
3113
- return op.operation.action.type === "DELETE_DOCUMENT";
3291
+ const type = op.operation.action.type;
3292
+ return type === "DELETE_DOCUMENT" || type === "PURGE_DOCUMENT";
3114
3293
  }
3115
3294
  function extractDriveHeader(op) {
3116
3295
  if (!op.context.resultingState) return void 0;
3117
3296
  return JSON.parse(op.context.resultingState).header;
3118
3297
  }
3298
+ function extractCreationHeader(op) {
3299
+ if (op.operation.action.type !== "CREATE_DOCUMENT") return void 0;
3300
+ return extractDriveHeader(op) ?? createDocumentFromAction(op.operation.action).header;
3301
+ }
3119
3302
  function extractDeletedDocumentId(op) {
3120
3303
  return op.operation.action.input.documentId ?? op.context.documentId;
3121
3304
  }
@@ -3183,11 +3366,25 @@ function lowestOf(ops) {
3183
3366
  for (const op of ops) lowest = Math.min(lowest, op.context.ordinal);
3184
3367
  return lowest;
3185
3368
  }
3369
+ /** Drops operations of tombstoned ids; the marker always passes. */
3370
+ function withoutPurged(ops, purged, keep) {
3371
+ if (purged.size === 0) return ops;
3372
+ return ops.filter((op) => isPurgeMarker$1(op.operation) || !purged.has(op.context.documentId) || keep?.has(op.context.ordinal) === true);
3373
+ }
3374
+ /** The ids whose tombstone would drop an operation of `ops`. */
3375
+ function purgeCandidates(ops) {
3376
+ const ids = /* @__PURE__ */ new Set();
3377
+ for (const op of ops) if (!isPurgeMarker$1(op.operation)) ids.add(op.context.documentId);
3378
+ return [...ids];
3379
+ }
3186
3380
  function highestOf(ops) {
3187
3381
  let highest = 0;
3188
3382
  for (const op of ops) highest = Math.max(highest, op.context.ordinal);
3189
3383
  return highest;
3190
3384
  }
3385
+ const LOOKUP_RETRY_MS = 100;
3386
+ const MAX_LOOKUP_RETRY_MS = 3e4;
3387
+ const LOOKUP_RETRIES = 10;
3191
3388
  var ProcessorQueue = class {
3192
3389
  options;
3193
3390
  tasks = [];
@@ -3195,17 +3392,24 @@ var ProcessorQueue = class {
3195
3392
  closed = false;
3196
3393
  replaysAhead = 0;
3197
3394
  unrouted = /* @__PURE__ */ new Set();
3395
+ wake;
3396
+ parked = false;
3198
3397
  constructor(options) {
3199
3398
  this.options = options;
3200
3399
  }
3201
3400
  get isClosed() {
3202
3401
  return this.closed;
3203
3402
  }
3403
+ /** True while a backfill or retry is queued or running. */
3404
+ get replaying() {
3405
+ return this.replaysAhead > 0;
3406
+ }
3204
3407
  /** Resolves at once behind a replay, so a pass never waits out a backfill. */
3205
- live(ops) {
3408
+ live(ops, check) {
3206
3409
  const delivered = this.push((done) => ({
3207
3410
  kind: "live",
3208
3411
  ops,
3412
+ check,
3209
3413
  done
3210
3414
  }));
3211
3415
  return this.replaysAhead > 0 ? Promise.resolve() : delivered;
@@ -3223,6 +3427,21 @@ var ProcessorQueue = class {
3223
3427
  done
3224
3428
  }));
3225
3429
  }
3430
+ /** `share` if active, then `deletion` whatever the status; true once delivered. */
3431
+ erase(share, deletion, check) {
3432
+ if (this.closed) return Promise.resolve(false);
3433
+ let delivered = false;
3434
+ const task = {
3435
+ kind: "erase",
3436
+ ops: share,
3437
+ deletion,
3438
+ check,
3439
+ run: async () => {
3440
+ delivered = await this.runErase(task);
3441
+ }
3442
+ };
3443
+ return this.push((done) => Object.assign(task, { done })).then(() => delivered);
3444
+ }
3226
3445
  /** Replays from the cursor as it stands when the task runs. */
3227
3446
  backfill() {
3228
3447
  return this.replay("backfill", () => this.runBackfill());
@@ -3250,6 +3469,7 @@ var ProcessorQueue = class {
3250
3469
  }
3251
3470
  });
3252
3471
  this.closed = true;
3472
+ this.wake?.();
3253
3473
  return disconnect;
3254
3474
  }
3255
3475
  replay(kind, run) {
@@ -3320,19 +3540,121 @@ var ProcessorQueue = class {
3320
3540
  await this.raiseCursor(through);
3321
3541
  return;
3322
3542
  }
3323
- if (cursor.status !== "active") {
3543
+ if (cursor.status !== "active" || this.parked) {
3324
3544
  await this.parkBelow(fresh);
3325
3545
  return;
3326
3546
  }
3327
- if (!await this.deliver(fresh)) {
3547
+ const live = await this.dropPurgedLive(batch, fresh);
3548
+ if (live === void 0) {
3549
+ await this.parkBelow(fresh);
3550
+ return;
3551
+ }
3552
+ if (live.length > 0 && !await this.deliver(live)) {
3328
3553
  await this.parkBelow(fresh);
3329
3554
  return;
3330
3555
  }
3331
3556
  await this.raiseCursor(Math.max(highestOf(fresh), through));
3332
3557
  }
3558
+ /** Undefined once closed, or parked after the lookup's last retry. */
3559
+ async dropPurgedLive(batch, fresh) {
3560
+ const keep = /* @__PURE__ */ new Set();
3561
+ const checks = [];
3562
+ for (const task of batch) {
3563
+ if (task.kind !== "live" || !task.check) continue;
3564
+ checks.push(task.check.purged);
3565
+ for (const ordinal of task.check.keep ?? []) keep.add(ordinal);
3566
+ }
3567
+ let purged;
3568
+ try {
3569
+ const sets = await Promise.all(checks);
3570
+ purged = new Set(sets.flatMap((set) => [...set]));
3571
+ } catch (error) {
3572
+ purged = await this.retryLookup(fresh, error);
3573
+ }
3574
+ return purged && withoutPurged(fresh, purged, keep);
3575
+ }
3576
+ async retryLookup(ops, error) {
3577
+ const ids = purgeCandidates(ops);
3578
+ let wait = this.options.lookupRetryMs ?? LOOKUP_RETRY_MS;
3579
+ const retries = this.options.lookupRetries ?? LOOKUP_RETRIES;
3580
+ for (let attempt = 0; attempt < retries; attempt++) {
3581
+ this.options.logger.error("Processor '@ProcessorId' failed checking tombstones, retrying in @Wait ms: @Error", this.options.processorId, wait, error);
3582
+ await this.sleep(wait);
3583
+ if (this.closed) return void 0;
3584
+ try {
3585
+ return await this.options.purged(ids);
3586
+ } catch (retryError) {
3587
+ error = retryError;
3588
+ wait = Math.min(wait * 2, MAX_LOOKUP_RETRY_MS);
3589
+ }
3590
+ }
3591
+ await this.park(ops, ids, wait, error);
3592
+ }
3593
+ async park(ops, ids, wait, error) {
3594
+ this.parked = true;
3595
+ this.noteLookupError(error);
3596
+ this.options.logger.error("Processor '@ProcessorId' parked: tombstones unread after @Retries retries, retrying every @Wait ms: @Error", this.options.processorId, this.options.lookupRetries ?? LOOKUP_RETRIES, wait, error);
3597
+ this.replay("backfill", () => this.resume(ids, wait));
3598
+ await this.parkBelow(ops);
3599
+ }
3600
+ async resume(ids, wait) {
3601
+ for (;;) {
3602
+ if (this.closed) return;
3603
+ await this.sleep(wait);
3604
+ if (this.closed) return;
3605
+ try {
3606
+ await this.options.purged(ids);
3607
+ break;
3608
+ } catch (error) {
3609
+ this.noteLookupError(error);
3610
+ this.options.logger.error("Processor '@ProcessorId' still parked, tombstones unread; retrying in @Wait ms: @Error", this.options.processorId, wait, error);
3611
+ }
3612
+ }
3613
+ this.parked = false;
3614
+ const { cursor } = this.options;
3615
+ cursor.lastError = void 0;
3616
+ cursor.lastErrorTimestamp = void 0;
3617
+ await this.persist();
3618
+ this.options.logger.info("Processor '@ProcessorId' resumed from ordinal @Ordinal", this.options.processorId, cursor.lastOrdinal);
3619
+ await this.runBackfill();
3620
+ }
3621
+ noteLookupError(error) {
3622
+ const { cursor } = this.options;
3623
+ cursor.lastError = errorMessage(error);
3624
+ cursor.lastErrorTimestamp = /* @__PURE__ */ new Date();
3625
+ }
3626
+ sleep(ms) {
3627
+ return new Promise((resolve) => {
3628
+ const done = () => {
3629
+ clearTimeout(timer);
3630
+ this.wake = void 0;
3631
+ resolve();
3632
+ };
3633
+ const timer = setTimeout(done, ms);
3634
+ timer.unref?.();
3635
+ this.wake = done;
3636
+ });
3637
+ }
3638
+ async runErase(task) {
3639
+ const { cursor, floor } = this.options;
3640
+ const fresh = task.ops.filter((op) => {
3641
+ const ordinal = op.context.ordinal;
3642
+ if (ordinal <= floor) return false;
3643
+ return !this.unrouted.delete(ordinal);
3644
+ });
3645
+ const deletion = task.deletion && !this.unrouted.delete(task.deletion.context.ordinal) ? task.deletion : void 0;
3646
+ let ops = [];
3647
+ if (cursor.status === "active" && !this.parked && fresh.length > 0) try {
3648
+ ops = withoutPurged(fresh, await task.check.purged);
3649
+ } catch (error) {
3650
+ this.options.logger.error("Processor '@ProcessorId' skipped its drive's last batch, tombstones unread: @Error", this.options.processorId, error);
3651
+ }
3652
+ if (deletion) ops.push(deletion);
3653
+ return ops.length === 0 || await this.deliver(ops);
3654
+ }
3333
3655
  async runBackfill() {
3334
3656
  const { cursor, filter, floor } = this.options;
3335
- if (cursor.status !== "active") return;
3657
+ if (cursor.status !== "active" || this.parked) return;
3336
3658
  let page;
3337
3659
  try {
3338
3660
  page = await this.options.readSince(cursor.lastOrdinal);
@@ -3343,13 +3665,28 @@ var ProcessorQueue = class {
3343
3665
  while (page.results.length > 0) {
3344
3666
  if (this.closed) return;
3345
3667
  const matching = page.results.filter((op) => op.context.ordinal > floor && matchesFilter(op, filter));
3668
+ let live = matching;
3346
3669
  if (matching.length > 0) {
3347
- if (!await this.deliver(matching)) {
3670
+ const ids = purgeCandidates(matching);
3671
+ let purged;
3672
+ try {
3673
+ purged = await this.options.purged(ids);
3674
+ } catch (error) {
3675
+ purged = await this.retryLookup(matching, error);
3676
+ }
3677
+ if (purged === void 0) {
3348
3678
  await this.persist();
3349
3679
  return;
3350
3680
  }
3351
- this.dedupeQueued(matching);
3681
+ live = withoutPurged(matching, purged);
3352
3682
  }
3683
+ if (live.length > 0) {
3684
+ if (!await this.deliver(live)) {
3685
+ await this.persist();
3686
+ return;
3687
+ }
3688
+ }
3689
+ if (matching.length > 0) this.dedupeQueued(matching);
3353
3690
  await this.raiseCursor(highestOf(page.results));
3354
3691
  if (!page.next) break;
3355
3692
  try {
@@ -3370,7 +3707,10 @@ var ProcessorQueue = class {
3370
3707
  }
3371
3708
  if (queued.size === 0) return;
3372
3709
  for (const task of this.tasks) {
3373
- if (task.kind !== "live") continue;
3710
+ if (task.kind === "erase") {
3711
+ const { deletion } = task;
3712
+ if (deletion && queued.has(deletion.context.ordinal)) task.deletion = void 0;
3713
+ } else if (task.kind !== "live") continue;
3374
3714
  task.ops = task.ops.filter((op) => !queued.has(op.context.ordinal));
3375
3715
  }
3376
3716
  }
@@ -3398,7 +3738,8 @@ var ProcessorQueue = class {
3398
3738
  async raiseCursor(through) {
3399
3739
  const { cursor } = this.options;
3400
3740
  const capped = Math.min(through, this.options.confirmedThrough());
3401
- if (cursor.status !== "active" || capped <= cursor.lastOrdinal) return;
3741
+ if (cursor.status !== "active" || this.parked) return;
3742
+ if (capped <= cursor.lastOrdinal) return;
3402
3743
  cursor.lastOrdinal = capped;
3403
3744
  await this.persist();
3404
3745
  }
@@ -3424,6 +3765,8 @@ var ProcessorQueue = class {
3424
3765
  };
3425
3766
  //#endregion
3426
3767
  //#region src/processors/processor-manager.ts
3768
+ const DRIVE_STREAM_PAGE = 500;
3769
+ const SHUTDOWN_GRACE_MS = 5e3;
3427
3770
  var ProcessorManager = class extends BaseReadModel {
3428
3771
  factoryRegistry = /* @__PURE__ */ new Map();
3429
3772
  processorsByDrive = /* @__PURE__ */ new Map();
@@ -3435,6 +3778,14 @@ var ProcessorManager = class extends BaseReadModel {
3435
3778
  advancing = Promise.resolve();
3436
3779
  cursorWrites = /* @__PURE__ */ new Map();
3437
3780
  draining = /* @__PURE__ */ new Map();
3781
+ deletedDrives = /* @__PURE__ */ new Map();
3782
+ erasing = /* @__PURE__ */ new Map();
3783
+ driveHeaders = /* @__PURE__ */ new Map();
3784
+ headerReads = /* @__PURE__ */ new Map();
3785
+ headerSlots = 8;
3786
+ headerWaiters = [];
3787
+ stopped = false;
3788
+ owedFailures = /* @__PURE__ */ new Map();
3438
3789
  logger;
3439
3790
  driveContainerTypes;
3440
3791
  legacyProcessorIds;
@@ -3442,7 +3793,8 @@ var ProcessorManager = class extends BaseReadModel {
3442
3793
  super(db, operationIndex, writeCache, consistencyTracker, {
3443
3794
  readModelId: "processor-manager",
3444
3795
  rebuildStateOnInit: true,
3445
- indexing: unchunkedReadModelIndexingConfig
3796
+ indexing: unchunkedReadModelIndexingConfig,
3797
+ purgeFence: "none"
3446
3798
  });
3447
3799
  this.logger = logger;
3448
3800
  this.driveContainerTypes = driveContainerTypes;
@@ -3452,29 +3804,48 @@ var ProcessorManager = class extends BaseReadModel {
3452
3804
  await this.loadAllCursors();
3453
3805
  await super.init();
3454
3806
  await this.discoverExistingDrives();
3807
+ await this.discoverDeletedDrives();
3808
+ for (const [identifier, factory] of this.factoryRegistry) this.eraseOwed(identifier, factory);
3455
3809
  }
3456
3810
  async commitOperations(items) {
3457
3811
  const { runs, reserved } = this.detectNewDrives(items);
3458
- const disconnects = this.detectDeletedDrives(items);
3459
- const deliveries = this.enqueueRouted(items, reserved);
3460
- await Promise.all([
3461
- ...runs.map((run) => run()),
3462
- ...disconnects,
3463
- ...deliveries
3464
- ]);
3812
+ const deletions = this.findDriveDeletions(items);
3813
+ const deliveries = this.enqueueRouted(items, reserved, deletions);
3814
+ this.detectDeletedDrives(items);
3815
+ this.forgetPurgedHeaders(items);
3816
+ await Promise.all([...runs.map((run) => run()), ...deliveries]);
3465
3817
  }
3466
3818
  async registerFactory(identifier, factory) {
3819
+ if (this.stopped) return;
3467
3820
  const removals = this.removeFactory(identifier);
3468
3821
  this.factoryRegistry.set(identifier, factory);
3469
3822
  const previous = this.draining.get(identifier);
3470
3823
  const creationOrdinal = this.highWater() + 1;
3471
3824
  const runs = [];
3472
- for (const [driveId, documentType] of this.knownDrives) runs.push(this.reserveSlot(identifier, factory, driveId, createMinimalDriveHeader(driveId, documentType), creationOrdinal, void 0, false, previous).run);
3825
+ for (const [driveId, documentType] of this.knownDrives) runs.push(this.reserveSlot(identifier, factory, driveId, this.headerOf(driveId, documentType), creationOrdinal, void 0, false, previous).run);
3826
+ this.eraseOwed(identifier, factory, previous);
3473
3827
  await Promise.all([...removals, ...runs.map((run) => run())]);
3474
3828
  }
3475
3829
  async unregisterFactory(identifier) {
3476
3830
  await Promise.all(this.removeFactory(identifier));
3477
3831
  }
3832
+ /** Closes every processor queue; the reactor's kill() calls it. */
3833
+ async shutdown() {
3834
+ if (this.stopped) return;
3835
+ this.stopped = true;
3836
+ this.pendingSlots.clear();
3837
+ this.owedFailures.clear();
3838
+ const closing = [...this.allBound()].map(({ queue }) => queue.close());
3839
+ this.processorsByDrive.clear();
3840
+ let timer;
3841
+ const grace = new Promise((resolve) => {
3842
+ timer = setTimeout(() => resolve(false), SHUTDOWN_GRACE_MS);
3843
+ timer.unref?.();
3844
+ });
3845
+ const closed = await Promise.race([Promise.all(closing).then(() => true), grace]);
3846
+ clearTimeout(timer);
3847
+ if (!closed) this.logger.warn("Processors still busy @Ms ms into shutdown; not waiting for them", SHUTDOWN_GRACE_MS);
3848
+ }
3478
3849
  get(processorId) {
3479
3850
  for (const tracked of this.allTrackedProcessors()) if (tracked.processorId === processorId) return tracked;
3480
3851
  }
@@ -3492,6 +3863,7 @@ var ProcessorManager = class extends BaseReadModel {
3492
3863
  this.knownDrives.set(driveId, op.context.documentType);
3493
3864
  const driveHeader = extractDriveHeader(op);
3494
3865
  if (!driveHeader) continue;
3866
+ this.driveHeaders.set(driveId, driveHeader);
3495
3867
  for (const [identifier, factory] of this.factoryRegistry) {
3496
3868
  const { slot, run } = this.reserveSlot(identifier, factory, driveId, driveHeader, op.context.ordinal, items, true);
3497
3869
  reserved.add(slot);
@@ -3503,23 +3875,85 @@ var ProcessorManager = class extends BaseReadModel {
3503
3875
  reserved
3504
3876
  };
3505
3877
  }
3506
- /** Synchronous: drops a deleted drive's processors from the tables. */
3878
+ /** Synchronous: drops a deleted drive's processors; their rows stay owed. */
3507
3879
  detectDeletedDrives(items) {
3508
- const pending = [];
3509
3880
  for (const op of items) {
3510
3881
  if (!isDriveDeletion(op)) continue;
3511
3882
  const driveId = extractDeletedDocumentId(op);
3512
3883
  if (!driveId || !this.knownDrives.has(driveId)) continue;
3513
3884
  this.knownDrives.delete(driveId);
3885
+ this.deletedDrives.set(driveId, op);
3514
3886
  for (const slot of this.pendingSlots) if (slot.driveId === driveId) this.pendingSlots.delete(slot);
3515
3887
  for (const { queue } of this.processorsByDrive.get(driveId) ?? []) queue.close();
3516
3888
  this.processorsByDrive.delete(driveId);
3517
- pending.push(...this.deleteCursors((row) => row.driveId === driveId));
3889
+ this.pruneDeletedDrive(driveId);
3890
+ }
3891
+ }
3892
+ /** Synchronous: a purged drive's header is erased data. */
3893
+ forgetPurgedHeaders(items) {
3894
+ for (const op of items) if (isPurgeMarker$1(op.operation)) {
3895
+ this.driveHeaders.delete(op.context.documentId);
3896
+ const read = this.headerReads.get(op.context.documentId);
3897
+ if (read) read.purged = true;
3898
+ }
3899
+ }
3900
+ /** A drive's creation header; minimal once the drive's stream is purged. */
3901
+ headerOf(driveId, documentType) {
3902
+ const cached = this.driveHeaders.get(driveId);
3903
+ if (cached) return Promise.resolve(cached);
3904
+ const inFlight = this.headerReads.get(driveId)?.header;
3905
+ if (inFlight) return inFlight;
3906
+ const read = { purged: false };
3907
+ this.headerReads.set(driveId, read);
3908
+ read.header = this.readHeader(driveId, documentType, read);
3909
+ return read.header;
3910
+ }
3911
+ async readHeader(driveId, documentType, read) {
3912
+ let first;
3913
+ await this.takeHeaderSlot();
3914
+ try {
3915
+ [first] = await this.operationIndex.getStreamAfter({
3916
+ documentId: driveId,
3917
+ scope: "document",
3918
+ branch: "main"
3919
+ }, 0, void 0, 1);
3920
+ } catch (error) {
3921
+ this.logger.error("Failed reading drive '@DriveId' header: @Error", driveId, error);
3922
+ } finally {
3923
+ this.releaseHeaderSlot();
3924
+ if (this.headerReads.get(driveId) === read) this.headerReads.delete(driveId);
3518
3925
  }
3519
- return pending;
3926
+ const header = first && extractCreationHeader(first);
3927
+ if (!header || read.purged) return createMinimalDriveHeader(driveId, documentType);
3928
+ if (this.knownDrives.has(driveId) || this.deletedDrives.has(driveId)) this.driveHeaders.set(driveId, header);
3929
+ return header;
3930
+ }
3931
+ async takeHeaderSlot() {
3932
+ if (this.headerSlots > 0) {
3933
+ this.headerSlots--;
3934
+ return;
3935
+ }
3936
+ await new Promise((resolve) => this.headerWaiters.push(resolve));
3937
+ }
3938
+ releaseHeaderSlot() {
3939
+ const next = this.headerWaiters.shift();
3940
+ if (next) next();
3941
+ else this.headerSlots++;
3942
+ }
3943
+ /** Synchronous: the known drives this batch deletes. */
3944
+ findDriveDeletions(items) {
3945
+ const deletions = /* @__PURE__ */ new Map();
3946
+ for (const op of items) {
3947
+ if (!isDriveDeletion(op)) continue;
3948
+ const driveId = extractDeletedDocumentId(op);
3949
+ if (!driveId || !this.knownDrives.has(driveId)) continue;
3950
+ const first = deletions.get(driveId);
3951
+ if (!first || op.context.ordinal < first.context.ordinal) deletions.set(driveId, op);
3952
+ }
3953
+ return deletions;
3520
3954
  }
3521
3955
  /** Synchronous: puts each processor's share of the batch on its queue. */
3522
- enqueueRouted(items, reserved) {
3956
+ enqueueRouted(items, reserved, deletions = /* @__PURE__ */ new Map()) {
3523
3957
  if (items.length === 0) return [];
3524
3958
  let lowest = items[0].context.ordinal;
3525
3959
  let highest = 0;
@@ -3532,14 +3966,179 @@ var ProcessorManager = class extends BaseReadModel {
3532
3966
  if (reserved.has(slot)) continue;
3533
3967
  slot.lowestRoutedOrdinal = Math.min(slot.lowestRoutedOrdinal ?? lowest, lowest);
3534
3968
  }
3969
+ let check;
3970
+ const checkOf = () => check ??= this.checkPurged(items);
3535
3971
  const deliveries = [];
3536
3972
  for (const { tracked, queue } of this.allBound()) {
3973
+ const deletion = deletions.get(tracked.driveId);
3974
+ if (deletion) {
3975
+ deliveries.push(this.deliverDeletion(tracked, queue, items, deletion, checkOf()));
3976
+ continue;
3977
+ }
3537
3978
  const matching = items.filter((op) => matchesFilter(op, tracked.record.filter));
3538
- if (matching.length > 0) deliveries.push(queue.live(matching));
3979
+ if (matching.length > 0) deliveries.push(queue.live(matching, checkOf()));
3539
3980
  else queue.advance(highest);
3540
3981
  }
3541
3982
  return deliveries;
3542
3983
  }
3984
+ deliverDeletion(tracked, queue, items, deletion, check) {
3985
+ const through = deletion.context.ordinal;
3986
+ const share = items.filter((op) => op.context.ordinal < through && matchesFilter(op, tracked.record.filter));
3987
+ const behindReplay = queue.replaying;
3988
+ const settled = queue.erase(share, deletion, check).then((delivered) => this.settleErased(tracked, delivered, deletion));
3989
+ this.holdErasure(tracked.factoryId, tracked.driveId, settled);
3990
+ return behindReplay ? Promise.resolve() : settled;
3991
+ }
3992
+ /** A delivered deletion deletes the row; a failed one keeps it, errored. */
3993
+ settleErased(tracked, delivered, deletion) {
3994
+ if (!delivered) {
3995
+ this.deletedDrives.set(tracked.driveId, deletion);
3996
+ this.trackOwedFailure(tracked, tracked.record, tracked);
3997
+ return this.writeCursor(tracked, tracked);
3998
+ }
3999
+ const deleted = this.deleteCursors((row) => row.processorId === tracked.processorId);
4000
+ this.pruneDeletedDrive(tracked.driveId);
4001
+ return Promise.all(deleted).then(() => void 0);
4002
+ }
4003
+ /** Keeps a failed deletion in getAll(); retry() delivers it again. */
4004
+ trackOwedFailure(owner, record, state) {
4005
+ const { processorId, factoryId, driveId, processorIndex } = owner;
4006
+ this.owedFailures.set(processorId, {
4007
+ processorId,
4008
+ factoryId,
4009
+ driveId,
4010
+ processorIndex,
4011
+ record,
4012
+ lastOrdinal: state.lastOrdinal,
4013
+ status: "errored",
4014
+ lastError: state.lastError,
4015
+ lastErrorTimestamp: state.lastErrorTimestamp,
4016
+ retry: () => this.retryOwed(factoryId, driveId)
4017
+ });
4018
+ }
4019
+ async retryOwed(factoryId, driveId) {
4020
+ const factory = this.factoryRegistry.get(factoryId);
4021
+ const deletion = this.deletedDrives.get(driveId);
4022
+ if (!factory || !deletion || this.stopped) return;
4023
+ await this.eraseDrive(factoryId, factory, driveId, deletion);
4024
+ }
4025
+ /** Forgets a deleted drive once no cursor row owes its deletion. */
4026
+ pruneDeletedDrive(driveId) {
4027
+ for (const row of this.cursorCache.values()) if (row.driveId === driveId) return;
4028
+ this.deletedDrives.delete(driveId);
4029
+ if (!this.knownDrives.has(driveId)) this.driveHeaders.delete(driveId);
4030
+ }
4031
+ /** Delivers each deleted drive's deletion its rows of `factoryId` still owe. */
4032
+ eraseOwed(factoryId, factory, previous) {
4033
+ if (![...this.cursorCache.values()].some((row) => row.factoryId === factoryId && this.deletedDrives.has(row.driveId)) || this.stopped) return;
4034
+ (async () => {
4035
+ await previous;
4036
+ if (this.factoryRegistry.get(factoryId) !== factory) return;
4037
+ const owed = /* @__PURE__ */ new Set();
4038
+ for (const row of this.cursorCache.values()) if (row.factoryId === factoryId && this.deletedDrives.has(row.driveId)) owed.add(row.driveId);
4039
+ await Promise.all([...owed].map((driveId) => this.eraseOwedDrive(factoryId, factory, driveId)));
4040
+ })().catch((error) => {
4041
+ this.logger.error("Owed deletions of '@FactoryId' failed: @Error", factoryId, error);
4042
+ });
4043
+ }
4044
+ /** After any delivery of the pair's deletion in flight, if still owed. */
4045
+ async eraseOwedDrive(factoryId, factory, driveId) {
4046
+ const key = `${factoryId}:${driveId}`;
4047
+ for (let held = this.erasing.get(key); held; held = this.erasing.get(key)) await held;
4048
+ if (this.stopped || this.factoryRegistry.get(factoryId) !== factory) return;
4049
+ const owes = [...this.cursorCache.values()].some((row) => row.factoryId === factoryId && row.driveId === driveId);
4050
+ const deletion = this.deletedDrives.get(driveId);
4051
+ if (!owes || !deletion) return;
4052
+ await this.eraseDrive(factoryId, factory, driveId, deletion);
4053
+ }
4054
+ /** Marks the pair's deletion as being delivered until `erasure` settles. */
4055
+ holdErasure(factoryId, driveId, erasure) {
4056
+ const key = `${factoryId}:${driveId}`;
4057
+ const held = Promise.allSettled([this.erasing.get(key), erasure]).then(() => void 0);
4058
+ this.erasing.set(key, held);
4059
+ held.then(() => {
4060
+ if (this.erasing.get(key) === held) this.erasing.delete(key);
4061
+ });
4062
+ }
4063
+ /** Runs `erase` unless the pair's deletion is already being delivered. */
4064
+ async claimErasure(factoryId, driveId, erase) {
4065
+ if (this.erasing.has(`${factoryId}:${driveId}`)) return false;
4066
+ let release;
4067
+ this.holdErasure(factoryId, driveId, new Promise((resolve) => {
4068
+ release = resolve;
4069
+ }));
4070
+ try {
4071
+ await erase();
4072
+ } finally {
4073
+ release();
4074
+ }
4075
+ return true;
4076
+ }
4077
+ async eraseDrive(factoryId, factory, driveId, deletion) {
4078
+ await this.claimErasure(factoryId, driveId, async () => {
4079
+ const header = await this.headerOf(driveId, deletion.context.documentType);
4080
+ const slot = {
4081
+ factoryId,
4082
+ driveId
4083
+ };
4084
+ const records = await this.runFactory(slot, factory, header);
4085
+ if (!records) return;
4086
+ await this.eraseRecords(factoryId, driveId, deletion, records);
4087
+ });
4088
+ }
4089
+ /** The deletion to each record a row owes it to, then every disconnect. */
4090
+ async eraseRecords(factoryId, driveId, deletion, records) {
4091
+ const ids = resolveProcessorSlots(records, this.legacyProcessorIds);
4092
+ const delivered = /* @__PURE__ */ new Set();
4093
+ const failed = /* @__PURE__ */ new Map();
4094
+ const recordOf = /* @__PURE__ */ new Map();
4095
+ await Promise.all(records.map(async (record, i) => {
4096
+ const processorId = `${factoryId}:${driveId}:${ids[i]}`;
4097
+ if (this.cursorCache.has(processorId)) {
4098
+ recordOf.set(processorId, record);
4099
+ try {
4100
+ await record.processor.onOperations([deletion]);
4101
+ delivered.add(processorId);
4102
+ } catch (error) {
4103
+ failed.set(processorId, error);
4104
+ this.logger.error("Processor '@ProcessorId' failed on its drive's deletion: @Error", processorId, error);
4105
+ }
4106
+ }
4107
+ await this.discard({
4108
+ factoryId,
4109
+ driveId
4110
+ }, record.processor.onDisconnect.bind(record.processor));
4111
+ }));
4112
+ const writes = [];
4113
+ for (const row of this.cursorCache.values()) {
4114
+ if (row.factoryId !== factoryId || row.driveId !== driveId) continue;
4115
+ if (delivered.has(row.processorId)) continue;
4116
+ if (!failed.has(row.processorId)) {
4117
+ this.logger.warn("Factory '@FactoryId' made no processor '@ProcessorId' for deleted drive '@DriveId'; its deletion stays owed", factoryId, row.processorId, driveId);
4118
+ continue;
4119
+ }
4120
+ const error = failed.get(row.processorId);
4121
+ const state = {
4122
+ lastOrdinal: row.lastOrdinal,
4123
+ status: "errored",
4124
+ lastError: error instanceof Error ? error.message : String(error),
4125
+ lastErrorTimestamp: /* @__PURE__ */ new Date()
4126
+ };
4127
+ this.trackOwedFailure(row, recordOf.get(row.processorId), state);
4128
+ writes.push(this.writeCursor(row, state));
4129
+ }
4130
+ for (const processorId of delivered) this.owedFailures.delete(processorId);
4131
+ writes.push(...this.deleteCursors((row) => delivered.has(row.processorId)));
4132
+ this.pruneDeletedDrive(driveId);
4133
+ await Promise.all(writes);
4134
+ }
4135
+ /** Started at routing; takes no purge lock, since processors hold none. */
4136
+ checkPurged(items) {
4137
+ const ids = purgeCandidates(items);
4138
+ const purged = ids.length === 0 ? Promise.resolve(/* @__PURE__ */ new Set()) : findPurged(this.db, ids);
4139
+ purged.catch(() => void 0);
4140
+ return { purged };
4141
+ }
3543
4142
  /** Synchronous: binds records, or discards them if the slot was cancelled. */
3544
4143
  bind(slot, records, creationOrdinal, creationItems) {
3545
4144
  const released = this.pendingSlots.delete(slot);
@@ -3550,9 +4149,8 @@ var ProcessorManager = class extends BaseReadModel {
3550
4149
  delivered: settled
3551
4150
  };
3552
4151
  }
3553
- if (!released) {
3554
- const discarded = records.map((record) => this.discard(slot, record.processor.onDisconnect.bind(record.processor)));
3555
- const settled = Promise.all(discarded).then(() => void 0);
4152
+ if (!released || this.stopped) {
4153
+ const settled = this.discardRecords(slot, records);
3556
4154
  return {
3557
4155
  persisted: settled,
3558
4156
  delivered: settled
@@ -3571,16 +4169,24 @@ var ProcessorManager = class extends BaseReadModel {
3571
4169
  const missed = tracked.lastOrdinal < slot.reservedAt || lowestRoutedOrdinal !== void 0;
3572
4170
  if (tracked.status === "active" && missed) deliveries.push(queue.backfill());
3573
4171
  const matching = creationItems?.filter((op) => matchesFilter(op, tracked.record.filter));
3574
- if (matching && matching.length > 0) deliveries.push(queue.live(matching));
4172
+ if (matching && matching.length > 0) deliveries.push(queue.live(matching, this.checkPurged(matching)));
3575
4173
  }
3576
4174
  return {
3577
4175
  persisted: Promise.all(writes).then(() => void 0),
3578
4176
  delivered: Promise.all(deliveries).then(() => void 0)
3579
4177
  };
3580
4178
  }
3581
- /** Synchronous: removes a factory's slots, processors and cursor rows. */
4179
+ /** A cancelled slot's records; on a deleted drive they pay what rows owe. */
4180
+ async discardRecords(slot, records) {
4181
+ const { factoryId, driveId } = slot;
4182
+ const deletion = this.stopped ? void 0 : this.deletedDrives.get(driveId);
4183
+ if (deletion !== void 0 && await this.claimErasure(factoryId, driveId, () => this.eraseRecords(factoryId, driveId, deletion, records))) return;
4184
+ await Promise.all(records.map((record) => this.discard(slot, record.processor.onDisconnect.bind(record.processor))));
4185
+ }
4186
+ /** Synchronous: removes a factory's slots and processors; releases its rows. */
3582
4187
  removeFactory(identifier) {
3583
4188
  if (!this.factoryRegistry.delete(identifier)) return [];
4189
+ for (const [processorId, failure] of this.owedFailures) if (failure.factoryId === identifier) this.owedFailures.delete(processorId);
3584
4190
  const closing = [];
3585
4191
  for (const slot of this.pendingSlots) {
3586
4192
  if (slot.factoryId !== identifier) continue;
@@ -3601,7 +4207,25 @@ var ProcessorManager = class extends BaseReadModel {
3601
4207
  if (this.draining.get(identifier) === drained) this.draining.delete(identifier);
3602
4208
  });
3603
4209
  }
3604
- return this.deleteCursors((row) => row.factoryId === identifier);
4210
+ return this.releaseCursors(identifier);
4211
+ }
4212
+ releaseCursors(factoryId) {
4213
+ const writes = [];
4214
+ for (const row of this.cursorCache.values()) {
4215
+ if (row.factoryId !== factoryId) continue;
4216
+ if (row.status === "released") continue;
4217
+ if (this.deletedDrives.has(row.driveId)) continue;
4218
+ const released = {
4219
+ ...row,
4220
+ status: RELEASED_CURSOR_STATUS
4221
+ };
4222
+ this.cursorCache.set(row.processorId, released);
4223
+ writes.push(this.lane(row.processorId, () => this.db.updateTable("ProcessorCursor").set({
4224
+ status: RELEASED_CURSOR_STATUS,
4225
+ updatedAt: /* @__PURE__ */ new Date()
4226
+ }).where("processorId", "=", row.processorId).execute()));
4227
+ }
4228
+ return writes;
3605
4229
  }
3606
4230
  reserveSlot(factoryId, factory, driveId, driveHeader, creationOrdinal, creationItems, awaitDelivery, previous) {
3607
4231
  const slot = {
@@ -3615,7 +4239,8 @@ var ProcessorManager = class extends BaseReadModel {
3615
4239
  const run = async () => {
3616
4240
  const bound = (async () => {
3617
4241
  await previous;
3618
- const records = await this.runFactory(slot, factory, driveHeader);
4242
+ const header = await driveHeader;
4243
+ const records = await this.runFactory(slot, factory, header);
3619
4244
  return this.bind(slot, records, creationOrdinal, creationItems);
3620
4245
  })();
3621
4246
  slot.settled = bound.then(({ persisted }) => persisted);
@@ -3639,13 +4264,15 @@ var ProcessorManager = class extends BaseReadModel {
3639
4264
  const cached = this.cursorCache.get(processorId);
3640
4265
  let floor = 0;
3641
4266
  let cursor;
3642
- if (cached) cursor = {
3643
- lastOrdinal: cached.lastOrdinal,
3644
- status: cached.status,
3645
- lastError: cached.lastError ?? void 0,
3646
- lastErrorTimestamp: cached.lastErrorTimestamp ?? void 0
3647
- };
3648
- else {
4267
+ if (cached && cached.status !== "released") {
4268
+ const errored = cached.status !== "active";
4269
+ cursor = {
4270
+ lastOrdinal: cached.lastOrdinal,
4271
+ status: cached.status,
4272
+ lastError: errored ? cached.lastError ?? void 0 : void 0,
4273
+ lastErrorTimestamp: errored ? cached.lastErrorTimestamp ?? void 0 : void 0
4274
+ };
4275
+ } else {
3649
4276
  if (record.startFrom === "current") floor = creationOrdinal - 1;
3650
4277
  cursor = {
3651
4278
  lastOrdinal: floor,
@@ -3674,6 +4301,7 @@ var ProcessorManager = class extends BaseReadModel {
3674
4301
  routedThrough: () => this.highWater(),
3675
4302
  confirmedThrough: () => this.lastOrdinal,
3676
4303
  persist: (state) => this.writeCursor(tracked, state),
4304
+ purged: (ids) => findPurged(this.db, ids),
3677
4305
  logger: this.logger
3678
4306
  });
3679
4307
  return {
@@ -3706,6 +4334,7 @@ var ProcessorManager = class extends BaseReadModel {
3706
4334
  }
3707
4335
  *allTrackedProcessors() {
3708
4336
  for (const { tracked } of this.allBound()) yield tracked;
4337
+ yield* this.owedFailures.values();
3709
4338
  }
3710
4339
  isDriveCreation(op) {
3711
4340
  return op.operation.action.type === "CREATE_DOCUMENT" && this.driveContainerTypes.has(op.context.documentType);
@@ -3714,6 +4343,38 @@ var ProcessorManager = class extends BaseReadModel {
3714
4343
  const drives = await this.db.selectFrom("DocumentSnapshot").select(["documentId", "documentType"]).where("documentType", "in", [...this.driveContainerTypes]).where("isDeleted", "=", false).execute();
3715
4344
  for (const drive of drives) this.knownDrives.set(drive.documentId, drive.documentType);
3716
4345
  }
4346
+ /** Cursor rows of a drive that is gone: find the deletion they still owe. */
4347
+ async discoverDeletedDrives() {
4348
+ const driveIds = /* @__PURE__ */ new Set();
4349
+ for (const row of this.cursorCache.values()) if (!this.knownDrives.has(row.driveId)) driveIds.add(row.driveId);
4350
+ for (const driveId of driveIds) try {
4351
+ await this.discoverDeletedDrive(driveId);
4352
+ } catch (error) {
4353
+ this.logger.error("Failed reading deleted drive '@DriveId': @Error", driveId, error);
4354
+ }
4355
+ }
4356
+ async discoverDeletedDrive(driveId) {
4357
+ const key = {
4358
+ documentId: driveId,
4359
+ scope: "document",
4360
+ branch: "main"
4361
+ };
4362
+ let header;
4363
+ let deletion;
4364
+ let after = 0;
4365
+ for (;;) {
4366
+ const page = await this.operationIndex.getStreamAfter(key, after, void 0, DRIVE_STREAM_PAGE);
4367
+ for (const op of page) {
4368
+ header ??= extractCreationHeader(op);
4369
+ if (!deletion && isDriveDeletion(op) && extractDeletedDocumentId(op) === driveId) deletion = op;
4370
+ }
4371
+ if (page.length < DRIVE_STREAM_PAGE) break;
4372
+ after = page.at(-1).context.ordinal;
4373
+ }
4374
+ if (!deletion) return;
4375
+ this.deletedDrives.set(driveId, deletion);
4376
+ if (header) this.driveHeaders.set(driveId, header);
4377
+ }
3717
4378
  async loadAllCursors() {
3718
4379
  const rows = await this.db.selectFrom("ProcessorCursor").selectAll().execute();
3719
4380
  for (const row of rows) {
@@ -4576,11 +5237,12 @@ var ReactorSubscriptionManager = class {
4576
5237
  /**
4577
5238
  * Notify subscribers about deleted documents
4578
5239
  */
4579
- notifyDocumentsDeleted(documentIds, documentTypes, parentIds) {
5240
+ notifyDocumentsDeleted(documentIds, documentTypes, parentIds, info) {
4580
5241
  for (const subscription of this.deletedSubscriptions.values()) {
4581
5242
  const filteredIds = this.filterDocumentIds(documentIds, subscription.search, documentTypes, parentIds);
4582
5243
  if (filteredIds.length > 0) try {
4583
- subscription.callback(filteredIds);
5244
+ if (info) subscription.callback(filteredIds, info);
5245
+ else subscription.callback(filteredIds);
4584
5246
  } catch (error) {
4585
5247
  this.errorHandler.handleError(error, {
4586
5248
  eventType: "deleted",
@@ -4695,14 +5357,19 @@ var SubscriptionNotificationReadModel = class {
4695
5357
  if (operations.length === 0) return;
4696
5358
  const created = [];
4697
5359
  const deleted = [];
5360
+ const purgeDeleted = [];
4698
5361
  const updatedIds = /* @__PURE__ */ new Set();
5362
+ const purgedIds = /* @__PURE__ */ new Set();
4699
5363
  const documentTypes = /* @__PURE__ */ new Map();
4700
5364
  const parentIds = /* @__PURE__ */ new Map();
4701
5365
  for (const item of operations) {
4702
5366
  const { operation, context } = item;
4703
5367
  const actionType = operation.action.type;
4704
5368
  documentTypes.set(context.documentId, context.documentType);
4705
- if (actionType === "CREATE_DOCUMENT") created.push(context.documentId);
5369
+ if (isPurgeMarker$1(operation)) {
5370
+ purgedIds.add(context.documentId);
5371
+ if (context.appliedDeletion) purgeDeleted.push(context.documentId);
5372
+ } else if (actionType === "CREATE_DOCUMENT") created.push(context.documentId);
4706
5373
  else if (actionType === "DELETE_DOCUMENT") {
4707
5374
  const deletedId = operation.action.input.documentId ?? context.documentId;
4708
5375
  deleted.push(deletedId);
@@ -4716,9 +5383,21 @@ var SubscriptionNotificationReadModel = class {
4716
5383
  }
4717
5384
  if (created.length > 0) this.subscriptionManager.notifyDocumentsCreated(created, documentTypes, parentIds);
4718
5385
  if (deleted.length > 0) this.subscriptionManager.notifyDocumentsDeleted(deleted, documentTypes, parentIds);
5386
+ if (purgeDeleted.length > 0) this.subscriptionManager.notifyDocumentsDeleted(purgeDeleted, documentTypes, parentIds, { purged: true });
5387
+ for (const id of purgedIds) updatedIds.delete(id);
4719
5388
  if (updatedIds.size > 0 && this.documentView) {
4720
- const documents = await Promise.all(Array.from(updatedIds).map((id) => this.documentView.get(id)));
4721
- this.subscriptionManager.notifyDocumentsUpdated(documents);
5389
+ const documentView = this.documentView;
5390
+ const present = (await Promise.all(Array.from(updatedIds).map((id) => this.readUpdated(documentView, id)))).filter((document) => document !== void 0);
5391
+ if (present.length > 0) this.subscriptionManager.notifyDocumentsUpdated(present);
5392
+ }
5393
+ }
5394
+ /** Undefined for a document gone since its update; others still notify. */
5395
+ async readUpdated(documentView, id) {
5396
+ try {
5397
+ return await documentView.get(id);
5398
+ } catch (error) {
5399
+ if (DocumentNotFoundError.isError(error)) return void 0;
5400
+ throw error;
4722
5401
  }
4723
5402
  }
4724
5403
  };
@@ -4750,7 +5429,8 @@ var GroupReevaluationTrigger = class extends BaseReadModel {
4750
5429
  rebuildStateOnInit: false,
4751
5430
  indexing: unchunkedReadModelIndexingConfig,
4752
5431
  startFrom: "head",
4753
- replayStreamSuffix: false
5432
+ replayStreamSuffix: false,
5433
+ purgeFence: "none"
4754
5434
  });
4755
5435
  this.logger = logger;
4756
5436
  this.eventBus = eventBus;
@@ -4878,7 +5558,8 @@ const SyncEventTypes = {
4878
5558
  DEAD_LETTER_ADDED: 20004,
4879
5559
  CONNECTION_STATE_CHANGED: 20005,
4880
5560
  SYNC_HELD: 20006,
4881
- SYNC_RELEASED: 20007
5561
+ SYNC_RELEASED: 20007,
5562
+ PURGE_REFUSED: 20008
4882
5563
  };
4883
5564
  //#endregion
4884
5565
  //#region src/sync/mailbox.ts
@@ -4892,7 +5573,10 @@ var MailboxAggregateError = class extends Error {
4892
5573
  }
4893
5574
  };
4894
5575
  var Mailbox = class {
5576
+ holdAckBelowMarkers;
4895
5577
  itemsMap = /* @__PURE__ */ new Map();
5578
+ /** Unapplied items carrying a marker, so the held ack reads only these. */
5579
+ heldMarkers = /* @__PURE__ */ new Set();
4896
5580
  addedCallbacks = [];
4897
5581
  removedCallbacks = [];
4898
5582
  paused = false;
@@ -4900,6 +5584,9 @@ var Mailbox = class {
4900
5584
  removedBuffer = [];
4901
5585
  _ack = 0;
4902
5586
  _latestOrdinal = 0;
5587
+ constructor(options = {}) {
5588
+ this.holdAckBelowMarkers = options.holdAckBelowMarkers ?? false;
5589
+ }
4903
5590
  init(ackOrdinal) {
4904
5591
  this._ack = this._latestOrdinal = ackOrdinal;
4905
5592
  }
@@ -4910,7 +5597,13 @@ var Mailbox = class {
4910
5597
  return Array.from(this.itemsMap.values());
4911
5598
  }
4912
5599
  get ackOrdinal() {
4913
- return this._ack;
5600
+ if (!this.holdAckBelowMarkers) return this._ack;
5601
+ let floor = Number.POSITIVE_INFINITY;
5602
+ for (const item of this.heldMarkers) for (const op of item.operations) {
5603
+ const ordinal = op.context.ordinal;
5604
+ if (ordinal > 0 && ordinal < floor) floor = ordinal;
5605
+ }
5606
+ return Math.min(this._ack, floor - 1);
4914
5607
  }
4915
5608
  get latestOrdinal() {
4916
5609
  return this._latestOrdinal;
@@ -4920,10 +5613,20 @@ var Mailbox = class {
4920
5613
  }
4921
5614
  add(...items) {
4922
5615
  for (const item of items) {
5616
+ const replaced = this.itemsMap.get(item.id);
5617
+ if (replaced !== void 0) this.heldMarkers.delete(replaced);
4923
5618
  this.itemsMap.set(item.id, item);
4924
- for (const op of item.operations) this._latestOrdinal = Math.max(this._latestOrdinal, op.context.ordinal);
5619
+ let marker = false;
5620
+ for (const op of item.operations) {
5621
+ this._latestOrdinal = Math.max(this._latestOrdinal, op.context.ordinal);
5622
+ if (isPurgeMarker$1(op)) marker = true;
5623
+ }
5624
+ if (this.holdAckBelowMarkers && marker && item.status !== 2) this.heldMarkers.add(item);
4925
5625
  item.on((syncOp, _, next) => {
4926
- if (next === 2) for (const op of syncOp.operations) this._ack = Math.max(this._ack, op.context.ordinal);
5626
+ if (next === 2) {
5627
+ this.heldMarkers.delete(syncOp);
5628
+ for (const op of syncOp.operations) this._ack = Math.max(this._ack, op.context.ordinal);
5629
+ }
4927
5630
  });
4928
5631
  }
4929
5632
  if (this.paused) {
@@ -4940,7 +5643,10 @@ var Mailbox = class {
4940
5643
  if (errors.length > 0) throw new MailboxAggregateError(errors);
4941
5644
  }
4942
5645
  remove(...items) {
4943
- for (const item of items) this.itemsMap.delete(item.id);
5646
+ for (const item of items) {
5647
+ this.itemsMap.delete(item.id);
5648
+ this.heldMarkers.delete(item);
5649
+ }
4944
5650
  if (this.paused) {
4945
5651
  this.removedBuffer.push(...items);
4946
5652
  return;
@@ -5151,14 +5857,20 @@ var GraphQLRequestError = class extends Error {
5151
5857
  * difference between a channel that recovers and one that is dead for the
5152
5858
  * process lifetime.
5153
5859
  */
5154
- const RECOVERABLE_GRAPHQL_ERROR_CODES = {
5155
- /**
5156
- * A stored operation cannot be represented in the schema - an action with no
5157
- * id, say. The document holding it needs repairing, but the channel serves
5158
- * every other document, and a peer that stopped polling would stop receiving
5159
- * those too.
5160
- */
5161
- malformedStoredOperation: "MALFORMED_STORED_OPERATION" };
5860
+ const RECOVERABLE_GRAPHQL_ERROR_CODES = {
5861
+ /**
5862
+ * A stored operation cannot be represented in the schema - an action with no
5863
+ * id, say. The document holding it needs repairing, but the channel serves
5864
+ * every other document, and a peer that stopped polling would stop receiving
5865
+ * those too.
5866
+ */
5867
+ malformedStoredOperation: "MALFORMED_STORED_OPERATION",
5868
+ /**
5869
+ * The poll's marker refusals could not be recorded, so nothing was served.
5870
+ * The client keeps them pending and reports them on its next poll.
5871
+ */
5872
+ refusalNotRecorded: "REFUSAL_NOT_RECORDED"
5873
+ };
5162
5874
  const RECOVERABLE_CODES = new Set(Object.values(RECOVERABLE_GRAPHQL_ERROR_CODES));
5163
5875
  /**
5164
5876
  * True when every error the response carried named a recoverable code.
@@ -5344,6 +6056,11 @@ function filterOperations(operations, filter) {
5344
6056
  return true;
5345
6057
  });
5346
6058
  }
6059
+ /** Markers pass every remote filter, as they pass sinceTimestamp. */
6060
+ function filterForRemote(operations, filter) {
6061
+ const kept = new Set(filterOperations(operations, filter));
6062
+ return operations.filter((op) => kept.has(op) || isPurgeMarker$1(op));
6063
+ }
5347
6064
  /**
5348
6065
  * Creates an idle channel health status.
5349
6066
  *
@@ -5621,6 +6338,11 @@ function classifyJobFailure(errorName) {
5621
6338
  case "InvalidSignatureError": return "SIGNATURE_INVALID";
5622
6339
  case "HashMismatchError": return "HASH_MISMATCH";
5623
6340
  case "UnsupportedProtocolVersionError": return "UNSUPPORTED_PROTOCOL";
6341
+ case "DocumentPurgedError": return "DOCUMENT_PURGED";
6342
+ case "DocumentNotDeletedError":
6343
+ case "GroupInUseError":
6344
+ case "PurgeTooLargeError": return "PURGE_PRECONDITION";
6345
+ case "ReservedActionError": return "RESERVED_ACTION";
5624
6346
  default: return "UNCLASSIFIED";
5625
6347
  }
5626
6348
  }
@@ -5632,7 +6354,10 @@ function syncOperationErrorType(error) {
5632
6354
  const NON_QUARANTINING_ERROR_TYPES = /* @__PURE__ */ new Set([
5633
6355
  "AUTH_TIMESTAMP_NOT_MONOTONIC",
5634
6356
  "UNSUPPORTED_PROTOCOL",
5635
- "PEER_PROTOCOL_UNSUPPORTED"
6357
+ "PEER_PROTOCOL_UNSUPPORTED",
6358
+ "DOCUMENT_PURGED",
6359
+ "PURGE_PRECONDITION",
6360
+ "MARKER_REFUSED"
5636
6361
  ]);
5637
6362
  /**
5638
6363
  * A held auth operation must not quarantine: reconciling the two policies needs
@@ -5752,6 +6477,13 @@ var IntervalPollTimer = class {
5752
6477
  }
5753
6478
  };
5754
6479
  //#endregion
6480
+ //#region src/sync/purge-refusals.ts
6481
+ /** The most refusals a poll reports; the rest wait for the next poll. */
6482
+ const MAX_POLLED_REFUSALS = 100;
6483
+ function supportsPurgeRefusals(x) {
6484
+ return typeof x === "object" && x !== null && typeof x.recordPurgeRefusal === "function" && typeof x.recordPolledMarkerRefusals === "function";
6485
+ }
6486
+ //#endregion
5755
6487
  //#region src/sync/channels/utils.ts
5756
6488
  let syncOpCounter = 0;
5757
6489
  /**
@@ -5879,6 +6611,9 @@ const AGREEMENT_FIELDS = [
5879
6611
  "refusals",
5880
6612
  "SyncRefusalInput"
5881
6613
  ];
6614
+ function refusalKey(refusal) {
6615
+ return `${refusal.documentId}\u0000${refusal.branch}\u0000${refusal.kind ?? ""}`;
6616
+ }
5882
6617
  /** How often a channel whose remote went silent asks again whether it serves agreement. */
5883
6618
  const AGREEMENT_PROBE_INTERVAL_MS = 3e5;
5884
6619
  function pushMutation(withGatedUnder) {
@@ -5932,6 +6667,10 @@ var GqlRequestChannel = class {
5932
6667
  manifestRefresh;
5933
6668
  /** Polled rows this reactor could not run, reported on the next poll. */
5934
6669
  pendingRefusals = /* @__PURE__ */ new Map();
6670
+ /** When each pushed, unacknowledged marker entry was last pushed. */
6671
+ markerPushedAt = /* @__PURE__ */ new Map();
6672
+ /** Documents whose marker the remote refused; its report repeats per poll. */
6673
+ refusedMarkers = /* @__PURE__ */ new Set();
5935
6674
  isRecovering = false;
5936
6675
  connectionState = "connecting";
5937
6676
  /** Latest unrecoverable error was an auth rejection; cleared on connect. */
@@ -5955,21 +6694,30 @@ var GqlRequestChannel = class {
5955
6694
  };
5956
6695
  this.isShutdown = false;
5957
6696
  this.failureCount = 0;
5958
- this.inbox = new Mailbox();
6697
+ this.inbox = new Mailbox({ holdAckBelowMarkers: true });
5959
6698
  this.bufferedOutbox = new BufferedMailbox(500, 25);
5960
6699
  this.outbox = this.bufferedOutbox;
5961
6700
  this.deadLetter = new Mailbox();
5962
6701
  this.deadLetter.onAdded((syncOps) => {
5963
6702
  for (const syncOp of syncOps) {
5964
6703
  this.logger.warn("Dead letter added for document @DocumentId on channel @ChannelId", syncOp.documentId, this.channelId);
5965
- if (syncOp.error?.source === "inbox" && syncOperationErrorType(syncOp.error) === "UNSUPPORTED_PROTOCOL") this.pendingRefusals.set(`${syncOp.documentId}:${syncOp.branch}`, {
6704
+ if (syncOp.error?.source !== "inbox") continue;
6705
+ const errorType = syncOperationErrorType(syncOp.error);
6706
+ const refusal = errorType === "UNSUPPORTED_PROTOCOL" ? {
5966
6707
  documentId: syncOp.documentId,
5967
6708
  branch: syncOp.branch
5968
- });
6709
+ } : errorType === "MARKER_REFUSED" ? {
6710
+ documentId: syncOp.documentId,
6711
+ branch: syncOp.branch,
6712
+ kind: "marker"
6713
+ } : void 0;
6714
+ if (refusal) this.pendingRefusals.set(refusalKey(refusal), refusal);
5969
6715
  }
5970
6716
  });
5971
- this.outbox.onAdded((syncOps) => {
6717
+ this.outbox.onAdded((added) => {
5972
6718
  if (this.isShutdown) return;
6719
+ const syncOps = added.filter((op) => this.outbox.get(op.id) === op);
6720
+ if (syncOps.length === 0) return;
5973
6721
  if (this.isPushing) {
5974
6722
  this.pendingDrain = true;
5975
6723
  return;
@@ -5982,6 +6730,7 @@ var GqlRequestChannel = class {
5982
6730
  this.attemptPush(syncOps);
5983
6731
  });
5984
6732
  this.outbox.onRemoved((syncOps) => {
6733
+ for (const syncOp of syncOps) this.markerPushedAt.delete(syncOp.id);
5985
6734
  const ordinal = Math.min(getLatestAppliedOrdinal(syncOps), this.unappliedFloor() - 1);
5986
6735
  if (ordinal > this.lastPersistedOutboxOrdinal) {
5987
6736
  this.lastPersistedOutboxOrdinal = ordinal;
@@ -5995,8 +6744,8 @@ var GqlRequestChannel = class {
5995
6744
  });
5996
6745
  }
5997
6746
  });
5998
- this.inbox.onRemoved((syncOps) => {
5999
- const maxOrdinal = getLatestAppliedOrdinal(syncOps);
6747
+ this.inbox.onRemoved(() => {
6748
+ const maxOrdinal = this.inbox.ackOrdinal;
6000
6749
  if (maxOrdinal > this.lastPersistedInboxOrdinal) {
6001
6750
  this.lastPersistedInboxOrdinal = maxOrdinal;
6002
6751
  this.cursorStorage.upsert({
@@ -6060,6 +6809,9 @@ var GqlRequestChannel = class {
6060
6809
  this.peerManifestCallbacks.delete(callback);
6061
6810
  };
6062
6811
  }
6812
+ forgetMarkerRefusal(documentId, branch) {
6813
+ this.refusedMarkers.delete(`${documentId}\u0000${branch}`);
6814
+ }
6063
6815
  async hearPeer(manifest, undelivered) {
6064
6816
  if (this.peerManifest !== void 0 && (this.peerManifest?.revision ?? null) === (manifest?.revision ?? null) || isOlderManifest(manifest, this.peerManifest)) return;
6065
6817
  this.peerManifest = manifest;
@@ -6146,6 +6898,8 @@ var GqlRequestChannel = class {
6146
6898
  }
6147
6899
  const { envelopes, ackOrdinal, deadLetters, hasMore, manifestRevision, peerManifestRevision } = response;
6148
6900
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
6901
+ this.retireRefusedMarkers(deadLetters);
6902
+ this.repushUnackedMarkers();
6149
6903
  if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision)) return;
6150
6904
  const allSyncOps = [];
6151
6905
  for (const envelope of envelopes) if (envelope.type.toLowerCase() === "operations" && envelope.operations) {
@@ -6169,7 +6923,15 @@ var GqlRequestChannel = class {
6169
6923
  * Handles dead letters reported by the remote server.
6170
6924
  * Creates local dead letter SyncOperations so the channel quiesces.
6171
6925
  */
6172
- handleRemoteDeadLetters(deadLetters) {
6926
+ handleRemoteDeadLetters(wire) {
6927
+ const deadLetters = wire.filter((dl) => {
6928
+ if (dl.errorType !== "MARKER_REFUSED") return true;
6929
+ const key = `${dl.documentId}\u0000${dl.branch}`;
6930
+ if (this.refusedMarkers.has(key)) return false;
6931
+ this.refusedMarkers.add(key);
6932
+ return true;
6933
+ });
6934
+ if (deadLetters.length === 0) return;
6173
6935
  for (const dl of deadLetters) this.logger.error("Remote dead letter on @ChannelId: document @DocumentId failed with: @Error", this.channelId, dl.documentId, dl.error);
6174
6936
  const syncOps = [];
6175
6937
  for (const dl of deadLetters) {
@@ -6275,7 +7037,7 @@ var GqlRequestChannel = class {
6275
7037
  let refusals = [];
6276
7038
  for (;;) {
6277
7039
  const revision = this.peerServesAgreement ? this.localManifestProvider?.().revision : void 0;
6278
- refusals = this.peerServesAgreement ? [...this.pendingRefusals.values()] : [];
7040
+ refusals = this.peerServesAgreement ? [...this.pendingRefusals.values()].filter((refusal) => refusal.kind === void 0 || this.peerTakesMarkerRefusals()).slice(0, 100) : [];
6279
7041
  try {
6280
7042
  response = await this.executeGraphQL(this.pollQuery(this.peerServesDecisionFields, this.peerServesAgreement), this.peerServesAgreement ? {
6281
7043
  ...variables,
@@ -6293,7 +7055,7 @@ var GqlRequestChannel = class {
6293
7055
  this.peerServesDecisionFields = false;
6294
7056
  }
6295
7057
  }
6296
- for (const refusal of refusals) this.pendingRefusals.delete(`${refusal.documentId}:${refusal.branch}`);
7058
+ for (const refusal of refusals) this.pendingRefusals.delete(refusalKey(refusal));
6297
7059
  return {
6298
7060
  envelopes: response.pollSyncEnvelopes.envelopes,
6299
7061
  ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
@@ -6303,11 +7065,15 @@ var GqlRequestChannel = class {
6303
7065
  peerManifestRevision: response.pollSyncEnvelopes.peerManifestRevision
6304
7066
  };
6305
7067
  }
7068
+ /** A peer before the feature would reject the field; its refusals wait. */
7069
+ peerTakesMarkerRefusals() {
7070
+ return this.peerManifest?.features[MARKER_REFUSAL_FEATURE]?.includes(1) === true;
7071
+ }
6306
7072
  rejectsAgreementFields(error) {
6307
7073
  return this.peerServesAgreement && this.isAgreementRejection(error);
6308
7074
  }
6309
7075
  isAgreementRejection(error) {
6310
- if (!(error instanceof GraphQLRequestError) || error.category !== "graphql") return false;
7076
+ if (!(error instanceof GraphQLRequestError) || error.category !== "graphql" || isRecoverableGraphQLError(error)) return false;
6311
7077
  return AGREEMENT_FIELDS.some((field) => error.message.includes(field));
6312
7078
  }
6313
7079
  /** Resolves once what the silent peer cannot run is held. */
@@ -6491,6 +7257,25 @@ var GqlRequestChannel = class {
6491
7257
  }
6492
7258
  });
6493
7259
  }
7260
+ /** Stops pushing a marker entry the remote refused; its wire has no op id. */
7261
+ retireRefusedMarkers(deadLetters) {
7262
+ const refused = deadLetters.filter((dl) => dl.errorType === "MARKER_REFUSED");
7263
+ if (refused.length === 0) return;
7264
+ const retired = this.outbox.items.filter((syncOp) => this.markerPushedAt.has(syncOp.id) && syncOp.status !== 2 && refused.some((dl) => dl.documentId === syncOp.documentId && dl.branch === syncOp.branch));
7265
+ if (retired.length === 0) return;
7266
+ for (const syncOp of retired) syncOp.failed(new ChannelError("outbox", /* @__PURE__ */ new Error(`Remote refused the purge marker of ${syncOp.documentId}`), "MARKER_REFUSED"));
7267
+ this.outbox.remove(...retired);
7268
+ }
7269
+ /** Re-pushes markers unacked for retryMaxDelayMs: a restarted remote lost them. */
7270
+ repushUnackedMarkers() {
7271
+ if (this.isPushing || this.pushBlocked || this.receivingPages) return;
7272
+ const due = Date.now() - this.config.retryMaxDelayMs;
7273
+ const stale = this.outbox.items.filter((syncOp) => {
7274
+ const pushedAt = this.markerPushedAt.get(syncOp.id);
7275
+ return pushedAt !== void 0 && pushedAt <= due && syncOp.status !== 2;
7276
+ });
7277
+ if (stale.length > 0) this.attemptPush(stale);
7278
+ }
6494
7279
  /**
6495
7280
  * Schedules a retry of all current outbox items using exponential backoff.
6496
7281
  */
@@ -6543,7 +7328,11 @@ var GqlRequestChannel = class {
6543
7328
  * Creates one SyncEnvelope per SyncOperation with key/dependsOn for batch ordering.
6544
7329
  */
6545
7330
  async pushSyncOperations(syncOps) {
6546
- for (const syncOp of syncOps) syncOp.started();
7331
+ const now = Date.now();
7332
+ for (const syncOp of syncOps) {
7333
+ syncOp.started();
7334
+ if (syncOp.operations.some((op) => isPurgeMarker$1(op))) this.markerPushedAt.set(syncOp.id, now);
7335
+ }
6547
7336
  const gatedUnder = this.peerServesAgreement ? this.peerManifest?.revision : void 0;
6548
7337
  try {
6549
7338
  await this.executeGraphQL(pushMutation(gatedUnder !== void 0), {
@@ -6766,7 +7555,7 @@ var GqlResponseChannel = class {
6766
7555
  this.remoteName = remoteName;
6767
7556
  this.cursorStorage = cursorStorage;
6768
7557
  this.isShutdown = false;
6769
- this.inbox = new Mailbox();
7558
+ this.inbox = new Mailbox({ holdAckBelowMarkers: true });
6770
7559
  this.outbox = new Mailbox();
6771
7560
  this.deadLetter = new Mailbox();
6772
7561
  this.outbox.onRemoved((syncOps) => {
@@ -6776,8 +7565,8 @@ var GqlResponseChannel = class {
6776
7565
  this.outbox.onAdded((syncOps) => {
6777
7566
  this.forgetEvictedBelow(syncOps);
6778
7567
  });
6779
- this.inbox.onRemoved((syncOps) => {
6780
- const maxOrdinal = getLatestAppliedOrdinal(syncOps);
7568
+ this.inbox.onRemoved(() => {
7569
+ const maxOrdinal = this.inbox.ackOrdinal;
6781
7570
  if (maxOrdinal > this.lastPersistedInboxOrdinal) {
6782
7571
  this.lastPersistedInboxOrdinal = maxOrdinal;
6783
7572
  this.cursorStorage.upsert({
@@ -6928,6 +7717,40 @@ var GqlResponseChannelFactory = class {
6928
7717
  }
6929
7718
  };
6930
7719
  //#endregion
7720
+ //#region src/storage/kysely/delivery-lookup.ts
7721
+ /** The index row at `ordinal` and every membership of its document. */
7722
+ async function deliveryAt(db, documentId, ordinal) {
7723
+ const row = await db.selectFrom("operation_index_operations").selectAll().where("ordinal", "=", ordinal).where("documentId", "=", documentId).executeTakeFirst();
7724
+ if (!row) return void 0;
7725
+ const memberships = await db.selectFrom("document_collections").select([
7726
+ "collectionId",
7727
+ "joinedOrdinal",
7728
+ "leftOrdinal"
7729
+ ]).where("documentId", "=", documentId).execute();
7730
+ return {
7731
+ entry: {
7732
+ ordinal: row.ordinal,
7733
+ documentId: row.documentId,
7734
+ documentType: row.documentType,
7735
+ branch: row.branch,
7736
+ scope: row.scope,
7737
+ index: row.index,
7738
+ timestampUtcMs: row.timestampUtcMs,
7739
+ hash: row.hash,
7740
+ skip: row.skip,
7741
+ action: row.action,
7742
+ deniedReason: row.deniedReason ?? void 0,
7743
+ id: row.opId,
7744
+ sourceRemote: row.sourceRemote
7745
+ },
7746
+ memberships: memberships.map((membership) => ({
7747
+ collectionId: membership.collectionId,
7748
+ joinedOrdinal: Number(membership.joinedOrdinal),
7749
+ leftOrdinal: membership.leftOrdinal === null ? null : Number(membership.leftOrdinal)
7750
+ }))
7751
+ };
7752
+ }
7753
+ //#endregion
6931
7754
  //#region src/storage/kysely/sync-cursor-storage.ts
6932
7755
  function rowToRemoteCursor(row) {
6933
7756
  return {
@@ -7052,6 +7875,8 @@ var KyselySyncDeadLetterStorage = class {
7052
7875
  async add(deadLetter, signal) {
7053
7876
  if (signal?.aborted) throw new Error("Operation aborted");
7054
7877
  await this.db.transaction().execute(async (trx) => {
7878
+ await acquirePurgeLocks(trx, [deadLetter.documentId], "shared");
7879
+ if ((await findPurged(trx, [deadLetter.documentId])).size > 0) throw new DocumentPurgedError(deadLetter.documentId);
7055
7880
  const insertable = deadLetterRecordToRow(deadLetter);
7056
7881
  await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").doNothing()).execute();
7057
7882
  });
@@ -7109,14 +7934,20 @@ var KyselySyncHoldStorage = class {
7109
7934
  version: hold.version,
7110
7935
  held_at_utc_ms: hold.heldAtUtcMs
7111
7936
  };
7112
- await this.db.insertInto("sync_holds").values(row).onConflict((oc) => oc.columns([
7113
- "remote_name",
7114
- "document_id",
7115
- "branch"
7116
- ]).doUpdateSet({
7117
- protocol: row.protocol,
7118
- version: row.version
7119
- })).execute();
7937
+ await this.db.transaction().execute(async (trx) => {
7938
+ await acquirePurgeLocks(trx, [hold.documentId], "shared");
7939
+ if (hold.protocol !== DOCUMENT_PURGE_PROTOCOL) {
7940
+ if ((await findPurged(trx, [hold.documentId])).size > 0) throw new DocumentPurgedError(hold.documentId);
7941
+ }
7942
+ await trx.insertInto("sync_holds").values(row).onConflict((oc) => oc.columns([
7943
+ "remote_name",
7944
+ "document_id",
7945
+ "branch"
7946
+ ]).doUpdateSet({
7947
+ protocol: row.protocol,
7948
+ version: row.version
7949
+ })).execute();
7950
+ });
7120
7951
  }
7121
7952
  async remove(remoteName, documentId, branch) {
7122
7953
  await this.db.deleteFrom("sync_holds").where("remote_name", "=", remoteName).where("document_id", "=", documentId).where("branch", "=", branch).execute();
@@ -7126,6 +7957,69 @@ var KyselySyncHoldStorage = class {
7126
7957
  }
7127
7958
  };
7128
7959
  //#endregion
7960
+ //#region src/storage/kysely/sync-purge-refusal-storage.ts
7961
+ var KyselySyncPurgeRefusalStorage = class {
7962
+ db;
7963
+ constructor(db) {
7964
+ this.db = db;
7965
+ }
7966
+ async list(documentId) {
7967
+ return (await this.db.selectFrom("sync_purge_refusals").selectAll().where("document_id", "=", documentId).orderBy("refused_at_utc_ms").orderBy("remote_name").execute()).map((row) => ({
7968
+ remoteName: row.remote_name,
7969
+ documentId: row.document_id,
7970
+ branch: row.branch,
7971
+ refusedAtUtcMs: Number(row.refused_at_utc_ms)
7972
+ }));
7973
+ }
7974
+ async record(refusals) {
7975
+ if (refusals.length === 0) return;
7976
+ await this.db.insertInto("sync_purge_refusals").values(refusals.map((refusal) => ({
7977
+ remote_name: refusal.remoteName,
7978
+ document_id: refusal.documentId,
7979
+ branch: refusal.branch,
7980
+ refused_at_utc_ms: refusal.refusedAtUtcMs
7981
+ }))).onConflict((oc) => oc.columns([
7982
+ "remote_name",
7983
+ "document_id",
7984
+ "branch"
7985
+ ]).doNothing()).execute();
7986
+ }
7987
+ };
7988
+ //#endregion
7989
+ //#region src/storage/kysely/sync-received-marker-storage.ts
7990
+ var KyselySyncReceivedMarkerStorage = class {
7991
+ db;
7992
+ constructor(db) {
7993
+ this.db = db;
7994
+ }
7995
+ async list(remoteName) {
7996
+ return (await this.db.selectFrom("sync_received_markers").selectAll().where("remote_name", "=", remoteName).orderBy("received_at_utc_ms").execute()).map((row) => ({
7997
+ remoteName: row.remote_name,
7998
+ markerId: row.marker_id,
7999
+ documentId: row.document_id,
8000
+ branch: row.branch,
8001
+ operation: row.operation,
8002
+ receivedAtUtcMs: Number(row.received_at_utc_ms)
8003
+ }));
8004
+ }
8005
+ async upsert(record) {
8006
+ await this.db.insertInto("sync_received_markers").values({
8007
+ remote_name: record.remoteName,
8008
+ marker_id: record.markerId,
8009
+ document_id: record.documentId,
8010
+ branch: record.branch,
8011
+ operation: JSON.stringify(record.operation),
8012
+ received_at_utc_ms: record.receivedAtUtcMs
8013
+ }).onConflict((oc) => oc.columns(["remote_name", "marker_id"]).doNothing()).execute();
8014
+ }
8015
+ async remove(remoteName, markerId) {
8016
+ await this.db.deleteFrom("sync_received_markers").where("remote_name", "=", remoteName).where("marker_id", "=", markerId).execute();
8017
+ }
8018
+ async removeRemote(remoteName) {
8019
+ await this.db.deleteFrom("sync_received_markers").where("remote_name", "=", remoteName).execute();
8020
+ }
8021
+ };
8022
+ //#endregion
7129
8023
  //#region src/storage/kysely/sync-remote-storage.ts
7130
8024
  function rowToPeer(row) {
7131
8025
  if (row.peer_manifest_at_utc_ms === null) return;
@@ -7244,6 +8138,8 @@ var KyselySyncRemoteStorage = class {
7244
8138
  //#region src/sync/batch-aggregator.ts
7245
8139
  /** Queued between write-ready events when the settled watermark advances. */
7246
8140
  const SETTLED = Symbol("settled");
8141
+ /** Batches remembered as finished, so a repeated JOB_FAILED is ignored. */
8142
+ const FINISHED_BATCH_MEMORY = 1024;
7247
8143
  var BatchAggregator = class {
7248
8144
  logger;
7249
8145
  driveContainerTypes;
@@ -7252,6 +8148,7 @@ var BatchAggregator = class {
7252
8148
  queue = [];
7253
8149
  processing = false;
7254
8150
  pendingBatches = /* @__PURE__ */ new Map();
8151
+ finishedBatches = /* @__PURE__ */ new Set();
7255
8152
  constructor(logger, driveContainerTypes, onBatchReady, onSettled) {
7256
8153
  this.logger = logger;
7257
8154
  this.driveContainerTypes = driveContainerTypes;
@@ -7268,16 +8165,13 @@ var BatchAggregator = class {
7268
8165
  await this.processQueue();
7269
8166
  }
7270
8167
  async handleJobFailed(event) {
7271
- const batchId = event.job?.meta.batchId;
7272
- if (!batchId) return;
7273
- const pending = this.pendingBatches.get(batchId);
7274
- if (!pending) return;
7275
- this.pendingBatches.delete(batchId);
7276
- if (pending.events.length > 0) await this.onBatchReady(this.prepareBatch(pending.events));
8168
+ this.queue.push({ failed: event });
8169
+ await this.processQueue();
7277
8170
  }
7278
8171
  clear() {
7279
8172
  this.queue = [];
7280
8173
  this.pendingBatches.clear();
8174
+ this.finishedBatches.clear();
7281
8175
  }
7282
8176
  async processQueue() {
7283
8177
  if (this.processing) return;
@@ -7293,6 +8187,14 @@ var BatchAggregator = class {
7293
8187
  }
7294
8188
  continue;
7295
8189
  }
8190
+ if ("failed" in item) {
8191
+ try {
8192
+ await this.handleFailed(item.failed);
8193
+ } catch (error) {
8194
+ this.logger.error("Failed to process job-failed event (@jobId, @error)", item.failed.jobId, error instanceof Error ? error.message : String(error));
8195
+ }
8196
+ continue;
8197
+ }
7296
8198
  try {
7297
8199
  await this.handleWriteReady(item);
7298
8200
  } catch (error) {
@@ -7306,25 +8208,49 @@ var BatchAggregator = class {
7306
8208
  }
7307
8209
  async handleWriteReady(event) {
7308
8210
  const { batchId, batchJobIds } = event.jobMeta;
7309
- if (batchJobIds.length <= 1) {
8211
+ const pending = batchJobIds.length > 1 ? this.pendingFor(batchId, batchJobIds) : void 0;
8212
+ if (!pending) {
7310
8213
  await this.onBatchReady(this.prepareBatch([event]));
7311
8214
  return;
7312
8215
  }
8216
+ pending.resolvedJobIds.add(event.jobId);
8217
+ pending.events.push(event);
8218
+ if (this.finishIfResolved(batchId, pending)) await this.onBatchReady(this.prepareBatch(pending.events));
8219
+ }
8220
+ /** A failed job never arrives, so what its batch holds goes now. */
8221
+ async handleFailed(event) {
8222
+ const meta = event.job?.meta;
8223
+ if (!meta?.batchId || meta.batchJobIds.length <= 1) return;
8224
+ const pending = this.pendingFor(meta.batchId, meta.batchJobIds);
8225
+ if (!pending) return;
8226
+ pending.resolvedJobIds.add(event.jobId);
8227
+ this.finishIfResolved(meta.batchId, pending);
8228
+ const events = pending.events;
8229
+ pending.events = [];
8230
+ if (events.length > 0) await this.onBatchReady(this.prepareBatch(events));
8231
+ }
8232
+ pendingFor(batchId, batchJobIds) {
8233
+ if (this.finishedBatches.has(batchId)) return;
7313
8234
  let pending = this.pendingBatches.get(batchId);
7314
8235
  if (!pending) {
7315
8236
  pending = {
7316
8237
  expectedJobIds: new Set(batchJobIds),
7317
- arrivedJobIds: /* @__PURE__ */ new Set(),
8238
+ resolvedJobIds: /* @__PURE__ */ new Set(),
7318
8239
  events: []
7319
8240
  };
7320
8241
  this.pendingBatches.set(batchId, pending);
7321
8242
  }
7322
- pending.arrivedJobIds.add(event.jobId);
7323
- pending.events.push(event);
7324
- if (pending.arrivedJobIds.size >= pending.expectedJobIds.size) {
7325
- this.pendingBatches.delete(batchId);
7326
- await this.onBatchReady(this.prepareBatch(pending.events));
8243
+ return pending;
8244
+ }
8245
+ finishIfResolved(batchId, pending) {
8246
+ for (const id of pending.expectedJobIds) if (!pending.resolvedJobIds.has(id)) return false;
8247
+ this.pendingBatches.delete(batchId);
8248
+ this.finishedBatches.add(batchId);
8249
+ if (this.finishedBatches.size > FINISHED_BATCH_MEMORY) {
8250
+ const oldest = this.finishedBatches.values().next().value;
8251
+ this.finishedBatches.delete(oldest);
7327
8252
  }
8253
+ return true;
7328
8254
  }
7329
8255
  prepareBatch(events) {
7330
8256
  const collectionMemberships = this.mergeCollectionMemberships(events);
@@ -7365,8 +8291,42 @@ var BatchAggregator = class {
7365
8291
  }
7366
8292
  };
7367
8293
  //#endregion
8294
+ //#region src/sync/delivery-tracking.ts
8295
+ function supportsDeliveryTracking(x) {
8296
+ return typeof x === "object" && x !== null && typeof x.pendingDelivery === "function";
8297
+ }
8298
+ /** The outbox's rule: remotes it would serve the row to, not yet acknowledged. */
8299
+ async function pendingDelivery(deps, documentId, ordinal) {
8300
+ const row = await deps.lookup.at(documentId, ordinal);
8301
+ if (!row) throw new Error(`No operation of document ${documentId} at ordinal ${ordinal}`);
8302
+ const { entry, memberships } = row;
8303
+ const operation = toOperationWithContext(entry);
8304
+ const holds = await deps.holds.list({ documentId });
8305
+ const pending = [];
8306
+ for (const remote of deps.remotes) {
8307
+ const name = remote.meta.name;
8308
+ const membership = memberships.find((candidate) => candidate.collectionId === remote.meta.collectionId.key && (candidate.leftOrdinal === null || ordinal < candidate.leftOrdinal));
8309
+ if (!membership) continue;
8310
+ if (entry.sourceRemote === name) continue;
8311
+ if (filterForRemote([operation], remote.meta.filter).length === 0) continue;
8312
+ if (holds.some((hold) => hold.remoteName === name && hold.branch === entry.branch)) {
8313
+ pending.push({
8314
+ remote: name,
8315
+ state: "held"
8316
+ });
8317
+ continue;
8318
+ }
8319
+ const owedThrough = Math.max(ordinal, membership.joinedOrdinal);
8320
+ if ((await deps.cursors.get(name, "outbox")).cursorOrdinal < owedThrough) pending.push({
8321
+ remote: name,
8322
+ state: remote.channel.getConnectionState().state
8323
+ });
8324
+ }
8325
+ return pending;
8326
+ }
8327
+ //#endregion
7368
8328
  //#region src/sync/memory-hold-storage.ts
7369
- const keyOf = (remoteName, documentId, branch) => `${remoteName}\u0000${documentId}\u0000${branch}`;
8329
+ const keyOf$2 = (remoteName, documentId, branch) => `${remoteName}\u0000${documentId}\u0000${branch}`;
7370
8330
  /** For a sync manager built without a database. */
7371
8331
  var InMemorySyncHoldStorage = class {
7372
8332
  holds = /* @__PURE__ */ new Map();
@@ -7374,7 +8334,7 @@ var InMemorySyncHoldStorage = class {
7374
8334
  return Promise.resolve([...this.holds.values()].filter((hold) => (filter.remoteName === void 0 || hold.remoteName === filter.remoteName) && (filter.documentId === void 0 || hold.documentId === filter.documentId)));
7375
8335
  }
7376
8336
  upsert(hold) {
7377
- const key = keyOf(hold.remoteName, hold.documentId, hold.branch);
8337
+ const key = keyOf$2(hold.remoteName, hold.documentId, hold.branch);
7378
8338
  const known = this.holds.get(key);
7379
8339
  this.holds.set(key, {
7380
8340
  ...hold,
@@ -7383,7 +8343,7 @@ var InMemorySyncHoldStorage = class {
7383
8343
  return Promise.resolve();
7384
8344
  }
7385
8345
  remove(remoteName, documentId, branch) {
7386
- this.holds.delete(keyOf(remoteName, documentId, branch));
8346
+ this.holds.delete(keyOf$2(remoteName, documentId, branch));
7387
8347
  return Promise.resolve();
7388
8348
  }
7389
8349
  removeRemote(remoteName) {
@@ -7392,6 +8352,46 @@ var InMemorySyncHoldStorage = class {
7392
8352
  }
7393
8353
  };
7394
8354
  //#endregion
8355
+ //#region src/sync/memory-purge-refusal-storage.ts
8356
+ const keyOf$1 = (refusal) => `${refusal.remoteName}\u0000${refusal.documentId}\u0000${refusal.branch}`;
8357
+ /** For a sync manager built without a database. */
8358
+ var InMemorySyncPurgeRefusalStorage = class {
8359
+ records = /* @__PURE__ */ new Map();
8360
+ list(documentId) {
8361
+ return Promise.resolve([...this.records.values()].filter((record) => record.documentId === documentId));
8362
+ }
8363
+ record(refusals) {
8364
+ for (const refusal of refusals) {
8365
+ const key = keyOf$1(refusal);
8366
+ if (!this.records.has(key)) this.records.set(key, refusal);
8367
+ }
8368
+ return Promise.resolve();
8369
+ }
8370
+ };
8371
+ //#endregion
8372
+ //#region src/sync/memory-received-marker-storage.ts
8373
+ const keyOf = (remoteName, markerId) => `${remoteName}\u0000${markerId}`;
8374
+ /** For a sync manager built without a database. */
8375
+ var InMemorySyncReceivedMarkerStorage = class {
8376
+ records = /* @__PURE__ */ new Map();
8377
+ list(remoteName) {
8378
+ return Promise.resolve([...this.records.values()].filter((record) => record.remoteName === remoteName));
8379
+ }
8380
+ upsert(record) {
8381
+ const key = keyOf(record.remoteName, record.markerId);
8382
+ if (!this.records.has(key)) this.records.set(key, record);
8383
+ return Promise.resolve();
8384
+ }
8385
+ remove(remoteName, markerId) {
8386
+ this.records.delete(keyOf(remoteName, markerId));
8387
+ return Promise.resolve();
8388
+ }
8389
+ removeRemote(remoteName) {
8390
+ for (const [key, record] of this.records) if (record.remoteName === remoteName) this.records.delete(key);
8391
+ return Promise.resolve();
8392
+ }
8393
+ };
8394
+ //#endregion
7395
8395
  //#region src/sync/peer-agreement.ts
7396
8396
  /** Pure over the remote records it is given, live or not. */
7397
8397
  function createPeerAgreement(basis, remotes) {
@@ -7637,7 +8637,9 @@ const defaultSyncManagerConfig = {
7637
8637
  maxDeadLettersPerRemote: 100,
7638
8638
  maxInboxBatchSize: 32,
7639
8639
  maxHeldOperationsPerRemote: 1e4,
7640
- staleRemotePollWindowMs: 3e5
8640
+ staleRemotePollWindowMs: 3e5,
8641
+ markerRetryBaseDelayMs: 1e3,
8642
+ markerRetryMaxDelayMs: 6e4
7641
8643
  };
7642
8644
  const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
7643
8645
  /**
@@ -7655,6 +8657,20 @@ const VERSION_CACHE_CAP = 1e4;
7655
8657
  function createdVersions(operations, documentId) {
7656
8658
  for (const { operation, context } of operations) if (context.documentId === documentId && operation.action.type === "CREATE_DOCUMENT") return operation.action.input.protocolVersions ?? {};
7657
8659
  }
8660
+ /** The load was refused because this very id is purged here: a drop. */
8661
+ function isPurgedFailure(error, documentId) {
8662
+ return error?.name === "DocumentPurgedError" && error.documentId === documentId;
8663
+ }
8664
+ function carriesMarker(syncOp) {
8665
+ return syncOp.operations.some((op) => isPurgeMarker$1(op));
8666
+ }
8667
+ function markerIdsOf(syncOp) {
8668
+ return syncOp.operations.filter((op) => isPurgeMarker$1(op)).map((op) => op.operation.id);
8669
+ }
8670
+ /** Refused, or purged under another id: final, unlike an outage. */
8671
+ function isRefusedMarker(error) {
8672
+ return error?.name === "InvalidSignatureError" || error?.name === "DocumentPurgedError";
8673
+ }
7658
8674
  /** Where a sync operation's run of ordinals begins. */
7659
8675
  function firstOrdinalOf(syncOp) {
7660
8676
  return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
@@ -7680,6 +8696,10 @@ var SyncManager = class {
7680
8696
  config;
7681
8697
  connectionStateUnsubscribes = /* @__PURE__ */ new Map();
7682
8698
  quarantinedDocumentIds = /* @__PURE__ */ new Set();
8699
+ purgedDocumentIds = /* @__PURE__ */ new Set();
8700
+ purges;
8701
+ delivery;
8702
+ forgetDocument;
7683
8703
  backfillAbortControllers = /* @__PURE__ */ new Map();
7684
8704
  planKeyToJobUuid = /* @__PURE__ */ new Map();
7685
8705
  evictedOutboxFloors = /* @__PURE__ */ new Map();
@@ -7691,6 +8711,8 @@ var SyncManager = class {
7691
8711
  lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
7692
8712
  watermark;
7693
8713
  owed = /* @__PURE__ */ new Map();
8714
+ derivedThrough = /* @__PURE__ */ new Map();
8715
+ sweptThrough = 0;
7694
8716
  settledUnsubscribe;
7695
8717
  inboxChunkChain = Promise.resolve();
7696
8718
  capabilities;
@@ -7705,17 +8727,29 @@ var SyncManager = class {
7705
8727
  protocolVersionsOf;
7706
8728
  peerUpdates = /* @__PURE__ */ new Map();
7707
8729
  peerUnsubscribes = /* @__PURE__ */ new Map();
8730
+ receivedMarkers = /* @__PURE__ */ new Map();
8731
+ markerStorage;
8732
+ refusalStorage;
8733
+ markerWrites = /* @__PURE__ */ new Map();
8734
+ restoredMarkers = /* @__PURE__ */ new WeakSet();
8735
+ markerRetries = /* @__PURE__ */ new Map();
7708
8736
  constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, watermark, config = {}, localPeer = {
7709
8737
  capabilities: PEER_CAPABILITIES,
7710
8738
  flags: {}
7711
- }, holds = new InMemorySyncHoldStorage()) {
8739
+ }, holds = new InMemorySyncHoldStorage(), purges, receivedMarkers = new InMemorySyncReceivedMarkerStorage(), delivery, refusals = new InMemorySyncPurgeRefusalStorage()) {
8740
+ this.markerStorage = receivedMarkers;
8741
+ this.refusalStorage = refusals;
7712
8742
  this.watermark = watermark;
8743
+ this.purges = purges;
8744
+ this.delivery = delivery;
8745
+ this.forgetDocument = localPeer.forgetDocument;
7713
8746
  this.capabilities = localPeer.capabilities;
7714
8747
  this.manifest = localPeerManifest(localPeer.capabilities, localPeer.flags, localPeer.appKey, localPeer.sequence ?? Date.now());
7715
8748
  this.localSupport = localSupports(localPeer.capabilities, localPeer.flags);
7716
8749
  this.legacy = legacySupports(localPeer.capabilities);
7717
8750
  for (const capability of localPeer.capabilities) {
7718
8751
  if (capability.kind !== "protocol") continue;
8752
+ if (capability.name === DOCUMENT_PURGE_PROTOCOL) continue;
7719
8753
  const supported = capability.supported(localPeer.flags);
7720
8754
  const wanted = capability.preferred?.(localPeer.flags) ?? (supported.length > 0 ? Math.max(...supported) : void 0);
7721
8755
  if (wanted !== void 0) this.wanted[capability.name] = wanted;
@@ -7749,6 +8783,11 @@ var SyncManager = class {
7749
8783
  } catch (error) {
7750
8784
  this.logger.error("Failed to load quarantined document IDs (@error)", error instanceof Error ? error.message : String(error));
7751
8785
  }
8786
+ try {
8787
+ for (const id of await this.purges?.listPurged() ?? []) this.purgedDocumentIds.add(id);
8788
+ } catch (error) {
8789
+ this.logger.error("Failed to load purged document IDs (@error)", error instanceof Error ? error.message : String(error));
8790
+ }
7752
8791
  try {
7753
8792
  for (const hold of await this.holds.list()) this.heldFor(hold.remoteName).set(holdKey(hold.documentId, hold.branch), hold);
7754
8793
  } catch (error) {
@@ -7756,6 +8795,7 @@ var SyncManager = class {
7756
8795
  }
7757
8796
  const remoteRecords = await this.remoteStorage.list();
7758
8797
  const head = await this.watermarkHead();
8798
+ this.sweptThrough = Math.max(this.sweptThrough, head);
7759
8799
  for (const record of remoteRecords) {
7760
8800
  const channel = this.channelFactory.instance(record.id, record.name, record.channelConfig, this.cursorStorage, record.collectionId, record.filter, this.operationIndex, record.options);
7761
8801
  const remote = {
@@ -7774,6 +8814,8 @@ var SyncManager = class {
7774
8814
  this.owe(record.name, head);
7775
8815
  this.records.set(record.name, remote.meta);
7776
8816
  await this.loadDeadLetters(remote);
8817
+ await this.restoreReceivedMarkers(remote);
8818
+ const unheard = [...remote.channel.inbox.items];
7777
8819
  this.wireChannelCallbacks(remote);
7778
8820
  try {
7779
8821
  await channel.init();
@@ -7782,6 +8824,7 @@ var SyncManager = class {
7782
8824
  await this.dropRemoteAfterFailedInit(remote, false);
7783
8825
  continue;
7784
8826
  }
8827
+ if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
7785
8828
  await this.peerUpdates.get(record.name);
7786
8829
  const outboxAckOrdinal = remote.channel.outbox.ackOrdinal;
7787
8830
  if (outboxAckOrdinal > 0) {
@@ -7811,6 +8854,8 @@ var SyncManager = class {
7811
8854
  this.pruneDrainDeferred = false;
7812
8855
  this.batchAggregator.clear();
7813
8856
  this.owed.clear();
8857
+ for (const retry of this.markerRetries.values()) clearTimeout(retry.timer);
8858
+ this.markerRetries.clear();
7814
8859
  this.settledUnsubscribe?.();
7815
8860
  this.settledUnsubscribe = void 0;
7816
8861
  if (this.eventUnsubscribe) {
@@ -7903,6 +8948,47 @@ var SyncManager = class {
7903
8948
  heldAtUtcMs: record.heldAtUtcMs
7904
8949
  }));
7905
8950
  }
8951
+ async pendingDelivery(documentId, ordinal) {
8952
+ if (!this.delivery) throw new Error("Delivery tracking needs a delivery lookup");
8953
+ return pendingDelivery({
8954
+ remotes: [...this.remotes.values()].filter((remote) => !this.removing.has(remote.meta.name)),
8955
+ cursors: this.cursorStorage,
8956
+ holds: this.holds,
8957
+ lookup: this.delivery
8958
+ }, documentId, ordinal);
8959
+ }
8960
+ recordPurgeRefusal(refusal) {
8961
+ return this.persistRefusals([refusal]);
8962
+ }
8963
+ async recordPolledMarkerRefusals(remoteName, refusals) {
8964
+ const remote = this.getByName(remoteName);
8965
+ const tombstoned = refusals.slice(0, 100).filter((refusal) => this.purgedDocumentIds.has(refusal.documentId));
8966
+ if (tombstoned.length === 0) return;
8967
+ const collections = await this.operationIndex.getCollectionsForDocuments([...new Set(tombstoned.map((refusal) => refusal.documentId))]);
8968
+ const key = remote.meta.collectionId.key;
8969
+ const owed = /* @__PURE__ */ new Map();
8970
+ for (const refusal of tombstoned) {
8971
+ if (!collections[refusal.documentId]?.includes(key)) continue;
8972
+ owed.set(`${refusal.documentId}\u0000${refusal.branch}`, {
8973
+ remoteName,
8974
+ documentId: refusal.documentId,
8975
+ branch: refusal.branch,
8976
+ errorMessage: `Marker refused by ${remoteName}`
8977
+ });
8978
+ }
8979
+ await this.persistRefusals([...owed.values()]);
8980
+ }
8981
+ async persistRefusals(refusals) {
8982
+ if (refusals.length === 0) return;
8983
+ const refusedAtUtcMs = Date.now();
8984
+ await this.refusalStorage.record(refusals.map((refusal) => ({
8985
+ remoteName: refusal.remoteName,
8986
+ documentId: refusal.documentId,
8987
+ branch: refusal.branch,
8988
+ refusedAtUtcMs
8989
+ })));
8990
+ for (const refusal of refusals) await this.eventBus.emit(SyncEventTypes.PURGE_REFUSED, refusal).catch(() => {});
8991
+ }
7906
8992
  agreement() {
7907
8993
  return createPeerAgreement({
7908
8994
  local: this.manifest,
@@ -7918,16 +9004,36 @@ var SyncManager = class {
7918
9004
  }
7919
9005
  return held;
7920
9006
  }
9007
+ /** Once per id: a second pass would drop the holds its marker earned. */
9008
+ tombstone(documentId) {
9009
+ if (this.purgedDocumentIds.has(documentId)) return;
9010
+ this.purgedDocumentIds.add(documentId);
9011
+ this.quarantinedDocumentIds.delete(documentId);
9012
+ const prefix = holdKey(documentId, "");
9013
+ for (const key of [...this.versionCache.keys()]) if (key.startsWith(prefix)) this.versionCache.delete(key);
9014
+ this.forgetDocument?.(documentId);
9015
+ for (const held of this.heldKeys.values()) for (const [key, record] of [...held]) if (record.documentId === documentId) held.delete(key);
9016
+ for (const remote of this.remotes.values()) {
9017
+ const stale = remote.channel.outbox.items.filter((item) => item.documentId === documentId && item.status < 2 && !carriesMarker(item));
9018
+ for (const item of stale) item.executed();
9019
+ if (stale.length > 0) remote.channel.outbox.remove(...stale);
9020
+ const dead = remote.channel.deadLetter.items.filter((item) => item.documentId === documentId);
9021
+ if (dead.length > 0) remote.channel.deadLetter.remove(...dead);
9022
+ }
9023
+ }
7921
9024
  async forgetRemote(name) {
7922
9025
  this.records.delete(name);
7923
9026
  this.heldKeys.delete(name);
7924
9027
  await this.holds.removeRemote(name);
9028
+ await Promise.allSettled(this.markerWritesOf([name]));
9029
+ await this.markerStorage.removeRemote(name);
7925
9030
  }
7926
9031
  peerSupportsOf(remote) {
7927
9032
  return peerSupports(remote.meta.peer?.manifest ?? null, this.capabilities);
7928
9033
  }
7929
- /** Cached once found: a document's versions never change. */
9034
+ /** Cached once found: a document's versions change only by a purge. */
7930
9035
  async versionsOf(documentId, branch) {
9036
+ if (this.purgedDocumentIds.has(documentId)) return purgedProtocolVersions();
7931
9037
  const key = holdKey(documentId, branch);
7932
9038
  const cached = this.versionCache.get(key);
7933
9039
  if (cached) return cached;
@@ -7972,6 +9078,11 @@ var SyncManager = class {
7972
9078
  try {
7973
9079
  await this.holds.upsert(record);
7974
9080
  } catch (error) {
9081
+ if (DocumentPurgedError.isError(error)) {
9082
+ held.delete(key);
9083
+ this.tombstone(documentId);
9084
+ return;
9085
+ }
7975
9086
  this.logger.error("Failed to persist a sync hold (@remote, @documentId): @error", remote.meta.name, documentId, error instanceof Error ? error.message : String(error));
7976
9087
  }
7977
9088
  this.eventBus.emit(SyncEventTypes.SYNC_HELD, {
@@ -8036,7 +9147,7 @@ var SyncManager = class {
8036
9147
  page = await page.next();
8037
9148
  }
8038
9149
  let operations = entries.filter((entry) => entry.sourceRemote !== remote.meta.name).map((entry) => toOperationWithContext(entry));
8039
- operations = filterOperations(operations, remote.meta.filter);
9150
+ operations = filterForRemote(operations, remote.meta.filter);
8040
9151
  if (operations.length === 0) return;
8041
9152
  operations.sort((a, b) => {
8042
9153
  if (a.context.scope !== b.context.scope) {
@@ -8069,6 +9180,7 @@ var SyncManager = class {
8069
9180
  const peer = this.peerSupportsOf(remote);
8070
9181
  const refused = /* @__PURE__ */ new Set();
8071
9182
  for (const syncOp of syncOps) {
9183
+ if (carriesMarker(syncOp)) continue;
8072
9184
  const versions = createdVersions(syncOp.operations, syncOp.documentId) ?? await this.versionsOf(syncOp.documentId, syncOp.branch);
8073
9185
  if (versions && this.refuse(remote, syncOp, versions, peer)) refused.add(syncOp);
8074
9186
  }
@@ -8166,6 +9278,8 @@ var SyncManager = class {
8166
9278
  this.remotes.set(name, remote);
8167
9279
  this.records.set(name, meta);
8168
9280
  await this.loadDeadLetters(remote);
9281
+ await this.restoreReceivedMarkers(remote);
9282
+ const unheard = [...remote.channel.inbox.items];
8169
9283
  this.wireChannelCallbacks(remote);
8170
9284
  try {
8171
9285
  await channel.init();
@@ -8173,6 +9287,7 @@ var SyncManager = class {
8173
9287
  await this.dropRemoteAfterFailedInit(remote, !isCredentialOrNetworkError(error));
8174
9288
  throw error;
8175
9289
  }
9290
+ if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
8176
9291
  await this.peerUpdates.get(name);
8177
9292
  this.owe(name, await this.watermarkHead());
8178
9293
  const backfillController = new AbortController();
@@ -8196,6 +9311,7 @@ var SyncManager = class {
8196
9311
  if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
8197
9312
  this.removing.add(name);
8198
9313
  this.owed.delete(name);
9314
+ this.derivedThrough.delete(name);
8199
9315
  try {
8200
9316
  await this.teardownRemoteResources(remote);
8201
9317
  await this.remoteStorage.remove(name);
@@ -8253,7 +9369,14 @@ var SyncManager = class {
8253
9369
  this.connectionStateUnsubscribes.delete(name);
8254
9370
  }
8255
9371
  this.evictedOutboxFloors.delete(name);
9372
+ this.derivedThrough.delete(name);
8256
9373
  this.prunePending.delete(name);
9374
+ this.receivedMarkers.delete(name);
9375
+ for (const [id, retry] of [...this.markerRetries]) {
9376
+ if (retry.remoteName !== name) continue;
9377
+ clearTimeout(retry.timer);
9378
+ this.markerRetries.delete(id);
9379
+ }
8257
9380
  this.peerUnsubscribes.get(name)?.();
8258
9381
  this.peerUnsubscribes.delete(name);
8259
9382
  this.peerUpdates.delete(name);
@@ -8271,6 +9394,10 @@ var SyncManager = class {
8271
9394
  onSyncStatusChange(callback) {
8272
9395
  return this.syncStatusTracker.onChange(callback);
8273
9396
  }
9397
+ /** Settles once the remotes' received markers are stored; rejects if one failed. */
9398
+ async receiptsStored(remoteNames) {
9399
+ await Promise.all(this.markerWritesOf(remoteNames));
9400
+ }
8274
9401
  recordPlanKeyMapping(planKey, jobId) {
8275
9402
  if (!this.planKeyToJobUuid.has(planKey) && this.planKeyToJobUuid.size >= PLAN_KEY_TO_JOB_UUID_CAP) {
8276
9403
  const oldest = this.planKeyToJobUuid.keys().next().value;
@@ -8280,6 +9407,7 @@ var SyncManager = class {
8280
9407
  }
8281
9408
  wireChannelCallbacks(remote) {
8282
9409
  remote.channel.inbox.onAdded((syncOps) => this.handleInboxAdded(remote, syncOps));
9410
+ remote.channel.inbox.onRemoved((syncOps) => this.handleInboxRemoved(remote, syncOps));
8283
9411
  remote.channel.setLocalManifest?.(() => this.manifest);
8284
9412
  const unsubscribePeer = remote.channel.onPeerManifest?.((manifest, undelivered) => this.queuePeerManifest(remote, manifest, undelivered).catch(() => {}));
8285
9413
  if (unsubscribePeer) this.peerUnsubscribes.set(remote.meta.name, unsubscribePeer);
@@ -8300,7 +9428,25 @@ var SyncManager = class {
8300
9428
  remote.channel.deadLetter.remove(...refusals);
8301
9429
  for (const syncOp of refusals) this.holdRefused(remote, syncOp).catch(() => {});
8302
9430
  }
8303
- const syncOps = added.filter((syncOp) => !refusals.includes(syncOp));
9431
+ const remaining = added.filter((syncOp) => !refusals.includes(syncOp));
9432
+ const purged = remaining.filter((syncOp) => this.purgedDocumentIds.has(syncOp.documentId));
9433
+ for (const syncOp of purged) {
9434
+ this.logger.warn("Dead letter for purged document (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, syncOp.error?.message ?? "unknown");
9435
+ if (syncOp.error?.source !== "outbox") continue;
9436
+ if (syncOperationErrorType(syncOp.error) !== "MARKER_REFUSED") continue;
9437
+ const refusal = {
9438
+ remoteName: remote.meta.name,
9439
+ documentId: syncOp.documentId,
9440
+ branch: syncOp.branch,
9441
+ errorMessage: syncOp.error.error.message
9442
+ };
9443
+ this.recordPurgeRefusal(refusal).catch((error) => {
9444
+ this.logger.error("Recording a refused marker failed (@remote, @documentId, @error)", refusal.remoteName, refusal.documentId, error instanceof Error ? error.message : String(error));
9445
+ remote.channel.forgetMarkerRefusal?.(refusal.documentId, refusal.branch);
9446
+ });
9447
+ }
9448
+ if (purged.length > 0) remote.channel.deadLetter.remove(...purged);
9449
+ const syncOps = remaining.filter((syncOp) => !purged.includes(syncOp));
8304
9450
  for (const syncOp of syncOps) {
8305
9451
  this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
8306
9452
  const errorType = syncOperationErrorType(syncOp.error);
@@ -8313,12 +9459,16 @@ var SyncManager = class {
8313
9459
  documentId: syncOp.documentId,
8314
9460
  scopes: syncOp.scopes,
8315
9461
  branch: syncOp.branch,
8316
- operations: syncOp.operations,
9462
+ operations: carriesMarker(syncOp) ? syncOp.operations.filter((op) => isPurgeMarker$1(op)) : syncOp.operations,
8317
9463
  errorSource: syncOp.error?.source ?? "none",
8318
9464
  errorMessage: syncOp.error?.error.message ?? "unknown",
8319
9465
  errorType
8320
9466
  };
8321
9467
  this.deadLetterStorage.add(record).catch((err) => {
9468
+ if (DocumentPurgedError.isError(err)) {
9469
+ this.tombstone(record.documentId);
9470
+ return;
9471
+ }
8322
9472
  this.logger.error("Failed to persist dead letter (@id, @error)", record.id, err instanceof Error ? err.message : String(err));
8323
9473
  });
8324
9474
  this.eventBus.emit(SyncEventTypes.DEAD_LETTER_ADDED, {
@@ -8365,6 +9515,7 @@ var SyncManager = class {
8365
9515
  }
8366
9516
  async processCompleteBatch(batch) {
8367
9517
  if (this.isShutdown) return;
9518
+ for (const { event } of batch.entries) for (const op of event.operations) if (isPurgeMarker$1(op)) this.tombstone(op.context.documentId);
8368
9519
  const collectionIds = [...new Set(Object.values(batch.collectionMemberships).flatMap((collections) => collections))];
8369
9520
  const affectedRemotes = [];
8370
9521
  for (const collectionId of collectionIds) {
@@ -8386,6 +9537,7 @@ var SyncManager = class {
8386
9537
  async deriveSettled() {
8387
9538
  if (this.isShutdown) return;
8388
9539
  const through = this.watermark.settledThrough;
9540
+ await this.oweSettledRange(through);
8389
9541
  for (const [name, upTo] of [...this.owed]) {
8390
9542
  const remote = this.remotes.get(name);
8391
9543
  if (!remote || this.removing.has(name)) {
@@ -8397,6 +9549,27 @@ var SyncManager = class {
8397
9549
  }
8398
9550
  await this.drainPrunes();
8399
9551
  }
9552
+ /** Owes the remotes of every collection that moved, event or not. */
9553
+ async oweSettledRange(through) {
9554
+ if (through <= this.sweptThrough) return;
9555
+ const among = [...new Set([...this.remotes.values()].filter((remote) => !this.removing.has(remote.meta.name) && (this.derivedThrough.get(remote.meta.name) ?? -1) < through).map((remote) => remote.meta.collectionId.key))];
9556
+ if (among.length === 0) {
9557
+ this.sweptThrough = through;
9558
+ return;
9559
+ }
9560
+ let collectionIds;
9561
+ try {
9562
+ collectionIds = await this.operationIndex.getCollectionsInRange(this.sweptThrough, through, among, this.abortController.signal);
9563
+ } catch (error) {
9564
+ this.logger.warn("Settled range sweep failed; the next advance retries it: @error", error);
9565
+ return;
9566
+ }
9567
+ for (const collectionId of collectionIds) for (const remote of this.getRemotesForCollection(collectionId)) {
9568
+ const name = remote.meta.name;
9569
+ if ((this.derivedThrough.get(name) ?? -1) < through) this.owe(name, through);
9570
+ }
9571
+ this.sweptThrough = through;
9572
+ }
8400
9573
  owe(name, upTo) {
8401
9574
  this.owed.set(name, Math.max(this.owed.get(name) ?? 0, upTo));
8402
9575
  }
@@ -8411,7 +9584,30 @@ var SyncManager = class {
8411
9584
  }
8412
9585
  handleInboxAdded(remote, syncOps) {
8413
9586
  if (this.isShutdown) return;
8414
- const eligible = syncOps.filter((op) => !this.quarantinedDocumentIds.has(op.documentId));
9587
+ const eligible = [];
9588
+ const dropped = [];
9589
+ const received = this.receivedMarkersOf(remote.meta.name);
9590
+ const loading = (id, syncOp) => {
9591
+ const item = received.get(id);
9592
+ return item !== void 0 && item !== syncOp && item.status !== 2 && remote.channel.inbox.get(item.id) === item;
9593
+ };
9594
+ for (const syncOp of syncOps) if (carriesMarker(syncOp)) {
9595
+ const ids = markerIdsOf(syncOp);
9596
+ if (ids.every((id) => loading(id, syncOp))) {
9597
+ this.storeReceivedMarker(remote, syncOp);
9598
+ dropped.push(syncOp);
9599
+ } else {
9600
+ for (const id of ids) received.set(id, syncOp);
9601
+ this.storeReceivedMarker(remote, syncOp);
9602
+ eligible.push(syncOp);
9603
+ }
9604
+ } else if (this.purgedDocumentIds.has(syncOp.documentId)) dropped.push(syncOp);
9605
+ else if (!this.quarantinedDocumentIds.has(syncOp.documentId)) eligible.push(syncOp);
9606
+ for (const syncOp of dropped) {
9607
+ this.logger.debug("Dropping received operations of a purged or already-loading document (@remote, @documentId)", remote.meta.name, syncOp.documentId);
9608
+ syncOp.executed();
9609
+ }
9610
+ if (dropped.length > 0) remote.channel.inbox.remove(...dropped);
8415
9611
  if (eligible.length === 0) return;
8416
9612
  const keyed = [];
8417
9613
  const nonKeyed = [];
@@ -8426,6 +9622,79 @@ var SyncManager = class {
8426
9622
  this.processInboxChunks(chunks);
8427
9623
  }
8428
9624
  }
9625
+ /** Forgets a marker once the item loading it leaves the inbox. */
9626
+ handleInboxRemoved(remote, syncOps) {
9627
+ const received = this.receivedMarkers.get(remote.meta.name);
9628
+ if (received === void 0 || received.size === 0) return;
9629
+ const name = remote.meta.name;
9630
+ for (const syncOp of syncOps) for (const id of markerIdsOf(syncOp)) {
9631
+ if (received.get(id) !== syncOp) continue;
9632
+ received.delete(id);
9633
+ this.writeMarker(name, id, () => this.markerStorage.remove(name, id));
9634
+ }
9635
+ }
9636
+ markerWritesOf(remoteNames) {
9637
+ if (remoteNames === void 0) return [...this.markerWrites.values()];
9638
+ const prefixes = [...remoteNames].map((name) => `${name}\u0000`);
9639
+ return [...this.markerWrites].filter(([key]) => prefixes.some((prefix) => key.startsWith(prefix))).map(([, write]) => write);
9640
+ }
9641
+ /** Kept until its outcome, so a restart does not ack past it unapplied. */
9642
+ storeReceivedMarker(remote, syncOp) {
9643
+ if (this.restoredMarkers.has(syncOp)) return;
9644
+ if (this.purgedDocumentIds.has(syncOp.documentId)) return;
9645
+ const name = remote.meta.name;
9646
+ for (const operation of syncOp.operations) {
9647
+ if (!isPurgeMarker$1(operation)) continue;
9648
+ const markerId = operation.operation.id;
9649
+ this.writeMarker(name, markerId, () => this.markerStorage.upsert({
9650
+ remoteName: name,
9651
+ markerId,
9652
+ documentId: syncOp.documentId,
9653
+ branch: syncOp.branch,
9654
+ operation,
9655
+ receivedAtUtcMs: Date.now()
9656
+ }));
9657
+ }
9658
+ }
9659
+ writeMarker(remoteName, markerId, write) {
9660
+ const key = `${remoteName}\u0000${markerId}`;
9661
+ const next = (this.markerWrites.get(key) ?? Promise.resolve()).catch(() => void 0).then(write);
9662
+ this.markerWrites.set(key, next);
9663
+ const settle = () => {
9664
+ if (this.markerWrites.get(key) === next) this.markerWrites.delete(key);
9665
+ };
9666
+ next.then(settle, (error) => {
9667
+ this.logger.error("Failed to store received marker (@remote, @markerId, @error)", remoteName, markerId, error instanceof Error ? error.message : String(error));
9668
+ settle();
9669
+ });
9670
+ }
9671
+ /** Queued before init resets latestOrdinal, so a puller is re-served above it. */
9672
+ async restoreReceivedMarkers(remote) {
9673
+ const name = remote.meta.name;
9674
+ let records;
9675
+ try {
9676
+ records = await this.markerStorage.list(name);
9677
+ } catch (error) {
9678
+ this.logger.error("Failed to load received markers for remote (@name, @error)", name, error instanceof Error ? error.message : String(error));
9679
+ return;
9680
+ }
9681
+ if (records.length === 0) return;
9682
+ const syncOps = records.map((record) => {
9683
+ const syncOp = new SyncOperation(crypto.randomUUID(), "", [], name, record.documentId, [record.operation.context.scope], record.branch, [record.operation]);
9684
+ syncOp.transported();
9685
+ this.restoredMarkers.add(syncOp);
9686
+ return syncOp;
9687
+ });
9688
+ remote.channel.inbox.add(...syncOps);
9689
+ }
9690
+ receivedMarkersOf(name) {
9691
+ let received = this.receivedMarkers.get(name);
9692
+ if (received === void 0) {
9693
+ received = /* @__PURE__ */ new Map();
9694
+ this.receivedMarkers.set(name, received);
9695
+ }
9696
+ return received;
9697
+ }
8429
9698
  processInboxChunks(chunks) {
8430
9699
  const next = this.inboxChunkChain.then(async () => {
8431
9700
  for (const chunk of chunks) {
@@ -8448,12 +9717,17 @@ var SyncManager = class {
8448
9717
  if (this.isShutdown) return;
8449
9718
  const err = error instanceof Error ? error : new Error(String(error));
8450
9719
  this.logger.error("Failed to load operations from inbox (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, err.message);
9720
+ if (carriesMarker(syncOp)) {
9721
+ this.retryMarker(remote, syncOp, err.message);
9722
+ return;
9723
+ }
8451
9724
  const channelError = new ChannelError("inbox", err);
8452
9725
  syncOp.failed(channelError);
8453
9726
  remote.channel.deadLetter.add(syncOp);
8454
9727
  remote.channel.inbox.remove(syncOp);
8455
9728
  return;
8456
9729
  }
9730
+ if (syncOp.jobId) this.recordPlanKeyMapping(syncOp.jobId, jobInfo.id);
8457
9731
  let completedJobInfo;
8458
9732
  try {
8459
9733
  completedJobInfo = await this.awaiter.waitForJob(jobInfo.id, this.abortController.signal);
@@ -8461,6 +9735,10 @@ var SyncManager = class {
8461
9735
  if (this.isShutdown) return;
8462
9736
  const err = error instanceof Error ? error : new Error(String(error));
8463
9737
  this.logger.error("Failed to wait for job completion (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, jobInfo.id, err.message);
9738
+ if (carriesMarker(syncOp)) {
9739
+ this.retryMarker(remote, syncOp, err.message);
9740
+ return;
9741
+ }
8464
9742
  const channelError = new ChannelError("inbox", err);
8465
9743
  syncOp.failed(channelError);
8466
9744
  remote.channel.deadLetter.add(syncOp);
@@ -8468,14 +9746,57 @@ var SyncManager = class {
8468
9746
  return;
8469
9747
  }
8470
9748
  if (this.isShutdown) return;
8471
- if (completedJobInfo.status === "FAILED") {
9749
+ if (completedJobInfo.status !== "FAILED") {
9750
+ syncOp.executed();
9751
+ if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
9752
+ } else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
9753
+ this.tombstone(syncOp.documentId);
9754
+ syncOp.executed();
9755
+ } else {
8472
9756
  const errorMessage = completedJobInfo.error?.message || "Unknown error";
8473
9757
  this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
8474
- syncOp.failed(this.inboxFailure(completedJobInfo.error));
9758
+ if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
9759
+ this.retryMarker(remote, syncOp, errorMessage);
9760
+ return;
9761
+ }
9762
+ syncOp.failed(this.inboxFailure(syncOp, completedJobInfo.error));
8475
9763
  remote.channel.deadLetter.add(syncOp);
8476
- } else syncOp.executed();
9764
+ }
9765
+ this.markerRetries.delete(syncOp.id);
8477
9766
  remote.channel.inbox.remove(syncOp);
8478
9767
  }
9768
+ /** Reloads a marker with backoff; it stays in the inbox, not dead-lettered. */
9769
+ retryMarker(remote, syncOp, reason) {
9770
+ if (this.isShutdown) return;
9771
+ const attempt = (this.markerRetries.get(syncOp.id)?.attempt ?? 0) + 1;
9772
+ const delay = calculateBackoffDelay(attempt, this.config.markerRetryBaseDelayMs, this.config.markerRetryMaxDelayMs, Math.random());
9773
+ this.logger.warn("Purge marker load failed; retrying (@remote, @documentId, @attempt, @delayMs, @error)", remote.meta.name, syncOp.documentId, attempt, Math.round(delay), reason);
9774
+ const timer = setTimeout(() => {
9775
+ this.redeliverMarker(remote, syncOp).catch((error) => {
9776
+ this.logger.error("Purge marker retry failed (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, error instanceof Error ? error.message : String(error));
9777
+ });
9778
+ }, delay);
9779
+ if (typeof timer === "object" && "unref" in timer) timer.unref();
9780
+ this.markerRetries.set(syncOp.id, {
9781
+ remoteName: remote.meta.name,
9782
+ attempt,
9783
+ timer
9784
+ });
9785
+ }
9786
+ async redeliverMarker(remote, syncOp) {
9787
+ const name = remote.meta.name;
9788
+ if (this.isShutdown || this.remotes.get(name) !== remote || this.removing.has(name) || remote.channel.inbox.get(syncOp.id) !== syncOp) {
9789
+ this.markerRetries.delete(syncOp.id);
9790
+ return;
9791
+ }
9792
+ if (this.purgedDocumentIds.has(syncOp.documentId)) {
9793
+ this.markerRetries.delete(syncOp.id);
9794
+ syncOp.executed();
9795
+ remote.channel.inbox.remove(syncOp);
9796
+ return;
9797
+ }
9798
+ await this.applyInboxJob(remote, syncOp);
9799
+ }
8479
9800
  async applyInboxBatch(received) {
8480
9801
  const refused = /* @__PURE__ */ new Set();
8481
9802
  for (const { remote, syncOp } of received) {
@@ -8518,6 +9839,10 @@ var SyncManager = class {
8518
9839
  if (this.isShutdown) return;
8519
9840
  for (const { remote, syncOp } of items) {
8520
9841
  const err = error instanceof Error ? error : new Error(String(error));
9842
+ if (carriesMarker(syncOp)) {
9843
+ this.retryMarker(remote, syncOp, err.message);
9844
+ continue;
9845
+ }
8521
9846
  syncOp.failed(new ChannelError("inbox", err));
8522
9847
  remote.channel.deadLetter.add(syncOp);
8523
9848
  remote.channel.inbox.remove(syncOp);
@@ -8548,16 +9873,29 @@ var SyncManager = class {
8548
9873
  } catch (error) {
8549
9874
  if (this.isShutdown) continue;
8550
9875
  const err = error instanceof Error ? error : new Error(String(error));
9876
+ if (carriesMarker(syncOp)) {
9877
+ this.retryMarker(remote, syncOp, err.message);
9878
+ continue;
9879
+ }
8551
9880
  syncOp.failed(new ChannelError("inbox", err));
8552
9881
  remote.channel.deadLetter.add(syncOp);
8553
9882
  remote.channel.inbox.remove(syncOp);
8554
9883
  continue;
8555
9884
  }
8556
9885
  if (this.isShutdown) return;
8557
- if (completedJobInfo.status === "FAILED") {
8558
- syncOp.failed(this.inboxFailure(completedJobInfo.error));
9886
+ if (completedJobInfo.status !== "FAILED") {
9887
+ syncOp.executed();
9888
+ if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
9889
+ } else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
9890
+ this.tombstone(syncOp.documentId);
9891
+ syncOp.executed();
9892
+ } else if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
9893
+ this.retryMarker(remote, syncOp, completedJobInfo.error?.message || "Unknown error");
9894
+ continue;
9895
+ } else {
9896
+ syncOp.failed(this.inboxFailure(syncOp, completedJobInfo.error));
8559
9897
  remote.channel.deadLetter.add(syncOp);
8560
- } else syncOp.executed();
9898
+ }
8561
9899
  remote.channel.inbox.remove(syncOp);
8562
9900
  }
8563
9901
  }
@@ -8571,9 +9909,9 @@ var SyncManager = class {
8571
9909
  * keep syncing, because reconciling the two policies needs the traffic a
8572
9910
  * quarantine would stop.
8573
9911
  */
8574
- inboxFailure(error) {
9912
+ inboxFailure(syncOp, error) {
8575
9913
  const message = error?.message || "Unknown error";
8576
- return new ChannelError("inbox", /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), classifyJobFailure(error?.name ?? "Error"));
9914
+ return new ChannelError("inbox", /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), carriesMarker(syncOp) ? "MARKER_REFUSED" : classifyJobFailure(error?.name ?? "Error"));
8577
9915
  }
8578
9916
  /**
8579
9917
  * Where the next outbox derivation starts: at the evicted floor when there is
@@ -8705,6 +10043,7 @@ var SyncManager = class {
8705
10043
  }
8706
10044
  }
8707
10045
  emitBatches(remote, operations, mode, chain) {
10046
+ operations = operations.filter((op) => isPurgeMarker$1(op) || !this.purgedDocumentIds.has(op.context.documentId));
8708
10047
  if (operations.length === 0) return;
8709
10048
  const batches = batchOperationsByDocument(operations);
8710
10049
  const syncOps = [];
@@ -8725,13 +10064,14 @@ var SyncManager = class {
8725
10064
  async deriveOutbox(remote, ackOrdinal, mode, signal) {
8726
10065
  const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
8727
10066
  const startOrdinal = this.refillOrdinal(remote, ackOrdinal);
10067
+ const throughOrdinal = this.watermark.settledThrough;
8728
10068
  let maxOrdinal = startOrdinal;
8729
10069
  const chain = { lastJobByDoc: /* @__PURE__ */ new Map() };
8730
10070
  const sinceTimestamp = remote.meta.options.sinceTimestampUtcMs;
8731
10071
  const emitBatches = (operations) => this.emitBatches(remote, operations, mode, chain);
8732
10072
  let page = await this.operationIndex.find(remote.meta.collectionId.key, startOrdinal, {
8733
10073
  excludeSourceRemote: remote.meta.name,
8734
- throughOrdinal: this.watermark.settledThrough
10074
+ throughOrdinal
8735
10075
  }, void 0, composedSignal);
8736
10076
  let carry = [];
8737
10077
  let hasMore;
@@ -8743,9 +10083,11 @@ var SyncManager = class {
8743
10083
  operations = [...carry, ...operations];
8744
10084
  carry = [];
8745
10085
  }
8746
- if (sinceTimestamp && sinceTimestamp !== "0") operations = operations.filter((op) => op.operation.timestampUtcMs >= sinceTimestamp);
8747
- operations = filterOperations(operations, remote.meta.filter);
8748
- operations = operations.filter((op) => !this.quarantinedDocumentIds.has(op.context.documentId));
10086
+ for (const op of operations) if (isPurgeMarker$1(op)) this.tombstone(op.context.documentId);
10087
+ operations = operations.filter((op) => isPurgeMarker$1(op) || !this.purgedDocumentIds.has(op.context.documentId));
10088
+ if (sinceTimestamp && sinceTimestamp !== "0") operations = operations.filter((op) => isPurgeMarker$1(op) || op.operation.timestampUtcMs >= sinceTimestamp);
10089
+ operations = filterForRemote(operations, remote.meta.filter);
10090
+ operations = operations.filter((op) => isPurgeMarker$1(op) || !this.quarantinedDocumentIds.has(op.context.documentId));
8749
10091
  if (this.gates(remote)) operations = await this.gateOutbound(remote, operations);
8750
10092
  hasMore = !!page.next;
8751
10093
  if (operations.length > 0) {
@@ -8772,6 +10114,9 @@ var SyncManager = class {
8772
10114
  } while (hasMore);
8773
10115
  if (carry.length > 0) emitBatches(carry);
8774
10116
  remote.channel.outbox.advanceOrdinal(maxOrdinal);
10117
+ const name = remote.meta.name;
10118
+ if (this.evictedOutboxFloors.has(name)) this.derivedThrough.delete(name);
10119
+ else this.derivedThrough.set(name, Math.max(this.derivedThrough.get(name) ?? -1, throughOrdinal));
8775
10120
  }
8776
10121
  };
8777
10122
  //#endregion
@@ -8782,6 +10127,8 @@ var SyncBuilder = class {
8782
10127
  cursorStorage;
8783
10128
  deadLetterStorage;
8784
10129
  holdStorage;
10130
+ receivedMarkerStorage;
10131
+ purgeRefusalStorage;
8785
10132
  config = {};
8786
10133
  withChannelFactory(factory) {
8787
10134
  this.channelFactory = factory;
@@ -8803,6 +10150,14 @@ var SyncBuilder = class {
8803
10150
  this.holdStorage = storage;
8804
10151
  return this;
8805
10152
  }
10153
+ withReceivedMarkerStorage(storage) {
10154
+ this.receivedMarkerStorage = storage;
10155
+ return this;
10156
+ }
10157
+ withPurgeRefusalStorage(storage) {
10158
+ this.purgeRefusalStorage = storage;
10159
+ return this;
10160
+ }
8806
10161
  withMaxDeadLettersPerRemote(limit) {
8807
10162
  this.config.maxDeadLettersPerRemote = limit;
8808
10163
  return this;
@@ -8828,7 +10183,9 @@ var SyncBuilder = class {
8828
10183
  const cursorStorage = this.cursorStorage ?? new KyselySyncCursorStorage(db);
8829
10184
  const deadLetterStorage = this.deadLetterStorage ?? new KyselySyncDeadLetterStorage(db);
8830
10185
  const holdStorage = this.holdStorage ?? new KyselySyncHoldStorage(db);
8831
- const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, watermark, this.config, localPeer, holdStorage);
10186
+ const receivedMarkerStorage = this.receivedMarkerStorage ?? new KyselySyncReceivedMarkerStorage(db);
10187
+ const purgeRefusalStorage = this.purgeRefusalStorage ?? new KyselySyncPurgeRefusalStorage(db);
10188
+ const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, watermark, this.config, localPeer, holdStorage, { listPurged: () => listPurged(db) }, receivedMarkerStorage, { at: (documentId, ordinal) => deliveryAt(db, documentId, ordinal) }, purgeRefusalStorage);
8832
10189
  return {
8833
10190
  remoteStorage,
8834
10191
  cursorStorage,
@@ -8893,6 +10250,8 @@ var AbortError = class extends Error {
8893
10250
  * for the new Reactor architecture.
8894
10251
  */
8895
10252
  var Reactor = class {
10253
+ disposers;
10254
+ closers;
8896
10255
  logger;
8897
10256
  documentModelRegistry;
8898
10257
  shutdownStatus;
@@ -8908,7 +10267,9 @@ var Reactor = class {
8908
10267
  eventBus;
8909
10268
  executorManager;
8910
10269
  catchUp;
8911
- constructor(logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp) {
10270
+ constructor(logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp, disposers = [], closers = []) {
10271
+ this.disposers = disposers;
10272
+ this.closers = closers;
8912
10273
  this.catchUp = catchUp;
8913
10274
  this.logger = logger;
8914
10275
  this.documentModelRegistry = documentModelRegistry;
@@ -8936,9 +10297,15 @@ var Reactor = class {
8936
10297
  this.setShutdown(true);
8937
10298
  const shutdownAsync = async () => {
8938
10299
  await this.executorManager.stop(true);
10300
+ for (const dispose of this.disposers) dispose();
8939
10301
  await this.catchUp?.stop();
8940
10302
  this.readModelCoordinator.stop();
8941
10303
  this.jobTracker.shutdown();
10304
+ for (const close of this.closers) try {
10305
+ await close();
10306
+ } catch (error) {
10307
+ this.logger.error("Shutdown step failed: @Error", error);
10308
+ }
8942
10309
  };
8943
10310
  this.setCompleted(shutdownAsync());
8944
10311
  return this.shutdownStatus;
@@ -9823,6 +11190,13 @@ var ReactorBuilder = class {
9823
11190
  });
9824
11191
  const documentMetaCache = new DocumentMetaCache(operationStore, { maxDocuments: 1e3 });
9825
11192
  await documentMetaCache.startup();
11193
+ const unsubscribeMarkerEviction = eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, (_type, event) => {
11194
+ for (const item of event.operations) {
11195
+ if (!isPurgeMarker$1(item)) continue;
11196
+ writeCache.invalidate(item.context.documentId);
11197
+ documentMetaCache.invalidate(item.context.documentId);
11198
+ }
11199
+ });
9826
11200
  const collectionMembershipCache = new CollectionMembershipCache(operationIndex);
9827
11201
  const executionScope = new KyselyExecutionScope(database, operationStore, operationIndex, keyframeStore, writeCache, documentMetaCache, collectionMembershipCache);
9828
11202
  let executorManager = this.executorManager;
@@ -9918,7 +11292,7 @@ var ReactorBuilder = class {
9918
11292
  const indexedReadModels = readModelCoordinator.indexedReadModels?.bind(readModelCoordinator);
9919
11293
  if (indexedReadModels) catchUp.addSource(() => indexedReadModels().filter((model) => isCatchUpConsumer(model)), "host");
9920
11294
  else this.logger.warn("The read model coordinator does not report indexedReadModels; the host runs no read-side catch-up sweep");
9921
- const reactor = new Reactor(this.logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, this.features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp);
11295
+ const reactor = new Reactor(this.logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, this.features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp, [unsubscribeMarkerEviction], [() => processorManager.shutdown()]);
9922
11296
  const localPeer = {
9923
11297
  capabilities: this.getPeerCapabilities(),
9924
11298
  flags: featureFlags,
@@ -9929,6 +11303,9 @@ var ReactorBuilder = class {
9929
11303
  } catch {
9930
11304
  return;
9931
11305
  }
11306
+ },
11307
+ forgetDocument: (documentId) => {
11308
+ documentMetaCache.invalidate(documentId);
9932
11309
  }
9933
11310
  };
9934
11311
  let syncModule = void 0;
@@ -9973,7 +11350,8 @@ var ReactorBuilder = class {
9973
11350
  pools: this.instrumentedPools,
9974
11351
  degradedComponents,
9975
11352
  catchUp,
9976
- settledWatermark
11353
+ settledWatermark,
11354
+ documentPurgeService: new DocumentPurgeService(queue, jobTracker, eventBus)
9977
11355
  };
9978
11356
  catchUp.start();
9979
11357
  if (degradedComponents.length > 0) this.logger.warn("Reactor started with @count degraded component(s): @components", degradedComponents.length, degradedComponents.map(({ component }) => component).join(", "));
@@ -10040,7 +11418,7 @@ var ReactorBuilder = class {
10040
11418
  poolInstrumentations.push(forwarder);
10041
11419
  this.instrumentedPools.push(forwarder);
10042
11420
  }
10043
- const { ProjectionShardManager } = await import("./projection-shard-manager-CJIdjsd8.js");
11421
+ const { ProjectionShardManager } = await import("./projection-shard-manager-BSK-GEyP.js");
10044
11422
  const manager = new ProjectionShardManager({
10045
11423
  shardCount: config.shardCount,
10046
11424
  db,
@@ -10085,7 +11463,7 @@ var ReactorBuilder = class {
10085
11463
  */
10086
11464
  async createDefaultWorkerFactory(numWorkers, db) {
10087
11465
  const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
10088
- import("./worker-handle-DjbEQkq-.js"),
11466
+ import("./worker-handle-weNGCZme.js"),
10089
11467
  import("./transport-CZDHzfvy.js"),
10090
11468
  Promise.resolve().then(() => worker_exports)
10091
11469
  ]);
@@ -10834,6 +12212,6 @@ var DocumentIntegrityService = class {
10834
12212
  }
10835
12213
  };
10836
12214
  //#endregion
10837
- export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
12215
+ export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isPurgeMarker, isRecoverableGraphQLError, listPurged, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
10838
12216
 
10839
12217
  //# sourceMappingURL=index.js.map