@powerhousedao/reactor 6.2.3-dev.34 → 6.2.3-dev.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{build-worker-executor-BonPsKdx.js → build-worker-executor-B9IfcRij.js} +8 -2
- package/dist/build-worker-executor-B9IfcRij.js.map +1 -0
- package/dist/{document-indexer-I6i_YzsR.js → document-indexer-CZqIpdz2.js} +124 -26
- package/dist/document-indexer-CZqIpdz2.js.map +1 -0
- package/dist/entry.js +5 -2
- package/dist/entry.js.map +1 -1
- package/dist/{forwarding-logger-BkwlvD9W.js → forwarding-logger-DMqDs9tE.js} +980 -148
- package/dist/forwarding-logger-DMqDs9tE.js.map +1 -0
- package/dist/index.d.ts +1006 -517
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1593 -215
- package/dist/index.js.map +1 -1
- package/dist/projection-entry.js +42 -6
- package/dist/projection-entry.js.map +1 -1
- package/dist/{projection-shard-manager-CJIdjsd8.js → projection-shard-manager-BSK-GEyP.js} +2 -2
- package/dist/{projection-shard-manager-CJIdjsd8.js.map → projection-shard-manager-BSK-GEyP.js.map} +1 -1
- package/dist/{worker-handle-DjbEQkq-.js → worker-handle-weNGCZme.js} +9 -2
- package/dist/worker-handle-weNGCZme.js.map +1 -0
- package/package.json +4 -4
- package/dist/build-worker-executor-BonPsKdx.js.map +0 -1
- package/dist/document-indexer-I6i_YzsR.js.map +0 -1
- package/dist/forwarding-logger-BkwlvD9W.js.map +0 -1
- package/dist/worker-handle-DjbEQkq-.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as selectDecisionModel, A as ProbeSettler, At as matchesScope, B as JobExecutorEventTypes, C as KyselyWriteCache, Ct as RelationshipNotFoundError, D as acquirePurgeLocks, Dt as UpgradePreconditionFailedError, E as PURGE_NS, Et as UnsupportedStoredProtocolError, F as snapshotFunctionsFor, G as GATED_DOCUMENT_ACTIONS, H as DuplicateModuleError, I as DocumentMetaCache, J as createEmptyConsistencyToken, K as createConsistencyToken, L as CollectionMembershipCache, M as createKyselyWatermarkProbe, Mt as throwIfAborted, N as describeWaitingSessions, O as findPurged, Ot as EventBusAggregateError, P as parseSnapshot, Pt as __exportAll, Q as decideAtHead, R as DEFAULT_DEFERRED_JOB_TTL_MS, S as EventBus, St as PurgeTooLargeError, T as PURGE_LOCK_BUCKETS, Tt as UnsupportedProtocolVersionError, U as InvalidModuleError, V as DuplicateManifestError, W as ModuleNotFoundError, X as summarizeSubmittedActions, Y as submittedActionIds, Z as targetDocumentId, _ as KyselyExecutionScope, _t as DocumentPurgedError, a as DEFAULT_DRIVE_CONTAINER_TYPES, at as AppendConditionFailedError, b as appliedDelete, bt as InvalidOperationTimestampError, c as runMigrations, ct as DuplicateOperationError, d as KyselyOperationStore, dt as AuthEnforcementDisabledError, et as documentDecisionModel, f as KyselyKeyframeStore, ft as AuthTimestampNotMonotonicError, g as PassthroughSigner, gt as DocumentNotFoundError, h as verifyActionSignature, ht as DocumentNotDeletedError, it as APPEND_CONDITION_FAILED_PREFIX, j as SettledWatermark, jt as parsePagingOptions, k as listPurged, kt as ReactorEventTypes, l as createForwardingPoolInstrumentation, lt as OptimisticLockError, m as SimpleJobExecutor, mt as DocumentDeletedError, n as errorToInfo, nt as buildDecisionModel, o as REACTOR_SCHEMA, ot as DocumentAlreadyExistsError, p as DocumentModelRegistry, pt as AuthorizationDeniedError, q as createDocumentFromAction, r as sanitizeArg, rt as DriveCollectionId, s as getMigrationStatus, st as DocumentExistence, t as createForwardingLogger, tt as authDecisionModel, u as instrumentPgPool, ut as RevisionMismatchError, v as DEFAULT_PURGE_DELETE_BATCH, vt as ExcessiveReshuffleError, w as KyselyOperationIndex, wt as ReservedActionError, x as resolveFeatureFlags, xt as InvalidSignatureError, y as KyselyDocumentPurger, yt as GroupInUseError, z as DEFAULT_MAX_PURGE_OPERATIONS } from "./forwarding-logger-DMqDs9tE.js";
|
|
2
2
|
import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-BM_ULVWd.js";
|
|
3
|
-
import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-
|
|
4
|
-
import { DEFAULT_SIGNATURE_POLICY, DowngradeNotSupportedError, MAX_AUTH_GRANTS, PEER_CAPABILITIES, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
3
|
+
import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-CZqIpdz2.js";
|
|
4
|
+
import { DEFAULT_SIGNATURE_POLICY, DOCUMENT_PURGE_PROTOCOL, DowngradeNotSupportedError, MARKER_REFUSAL_FEATURE, MAX_AUTH_GRANTS, PEER_CAPABILITIES, PURGE_DOCUMENT, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, isPurgeMarker, isPurgeMarker as isPurgeMarker$1, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, purgedProtocolVersions, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
5
5
|
import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
|
|
6
6
|
import { v4 } from "uuid";
|
|
7
7
|
import { ConsoleLogger, childLogger } from "document-model";
|
|
@@ -326,11 +326,13 @@ var DriveClient = class {
|
|
|
326
326
|
logger;
|
|
327
327
|
reactor;
|
|
328
328
|
signer;
|
|
329
|
-
|
|
329
|
+
resolveReference;
|
|
330
|
+
constructor(client, logger, reactor, signer, resolveReference) {
|
|
330
331
|
this.client = client;
|
|
331
332
|
this.logger = logger;
|
|
332
333
|
this.reactor = reactor;
|
|
333
334
|
this.signer = signer;
|
|
335
|
+
this.resolveReference = resolveReference;
|
|
334
336
|
}
|
|
335
337
|
async create(input, signal) {
|
|
336
338
|
this.logger.verbose("drives.create(@input)", input);
|
|
@@ -351,6 +353,7 @@ var DriveClient = class {
|
|
|
351
353
|
async addFile(driveIdentifier, document, parentFolder, signal) {
|
|
352
354
|
this.logger.verbose("drives.addFile(@driveIdentifier, @document, @parentFolder)", driveIdentifier, document.header.id, parentFolder);
|
|
353
355
|
const documentId = document.header.id;
|
|
356
|
+
const driveId = await this.resolveReference(driveIdentifier, signal);
|
|
354
357
|
const createInput = {
|
|
355
358
|
model: document.header.documentType,
|
|
356
359
|
version: 0,
|
|
@@ -377,7 +380,7 @@ var DriveClient = class {
|
|
|
377
380
|
toVersion: normalizeDocumentModelVersion(document.state.document?.version),
|
|
378
381
|
initialState: document.state
|
|
379
382
|
}),
|
|
380
|
-
addRelationshipAction(
|
|
383
|
+
addRelationshipAction(driveId, documentId, "child")
|
|
381
384
|
], this.signer, {
|
|
382
385
|
documentId,
|
|
383
386
|
branch: "main"
|
|
@@ -388,7 +391,7 @@ var DriveClient = class {
|
|
|
388
391
|
documentType: document.header.documentType,
|
|
389
392
|
parentFolder
|
|
390
393
|
})], this.signer, {
|
|
391
|
-
documentId:
|
|
394
|
+
documentId: driveId,
|
|
392
395
|
branch: "main"
|
|
393
396
|
}, signal);
|
|
394
397
|
await this.runJobs([{
|
|
@@ -401,7 +404,7 @@ var DriveClient = class {
|
|
|
401
404
|
}], signal);
|
|
402
405
|
await this.runJobs([{
|
|
403
406
|
key: "drive",
|
|
404
|
-
documentId:
|
|
407
|
+
documentId: driveId,
|
|
405
408
|
scope: getSharedActionScope(driveActions),
|
|
406
409
|
branch: "main",
|
|
407
410
|
actions: driveActions,
|
|
@@ -420,22 +423,24 @@ var DriveClient = class {
|
|
|
420
423
|
if (!node || !isFolderNode(node)) throw new Error("Folder creation failed");
|
|
421
424
|
return node;
|
|
422
425
|
}
|
|
423
|
-
async removeNode(driveIdentifier,
|
|
424
|
-
this.logger.verbose("drives.removeNode(@driveIdentifier, @
|
|
425
|
-
const
|
|
426
|
+
async removeNode(driveIdentifier, nodeIdentifier, signal) {
|
|
427
|
+
this.logger.verbose("drives.removeNode(@driveIdentifier, @nodeIdentifier)", driveIdentifier, nodeIdentifier);
|
|
428
|
+
const driveId = await this.resolveReference(driveIdentifier, signal);
|
|
429
|
+
const nodeId = await this.resolveReference(nodeIdentifier, signal);
|
|
430
|
+
const drive = await this.client.get(driveId, void 0, signal);
|
|
426
431
|
const node = drive.state.global.nodes.find((n) => n.id === nodeId);
|
|
427
432
|
if (!node) {
|
|
428
433
|
if (!await this.documentExists(nodeId, signal)) throw new Error(`Node ${nodeId} not found in drive ${driveIdentifier}`);
|
|
429
|
-
await this.removeFileNode(
|
|
434
|
+
await this.removeFileNode(driveId, nodeId, signal);
|
|
430
435
|
return;
|
|
431
436
|
}
|
|
432
437
|
if (isFolderNode(node)) {
|
|
433
438
|
const fileDescendants = getDescendants(node, drive.state.global.nodes).filter(isFileNode);
|
|
434
|
-
for (const file of fileDescendants) await this.removeFileNode(
|
|
435
|
-
await this.client.execute(
|
|
439
|
+
for (const file of fileDescendants) await this.removeFileNode(driveId, file.id, signal);
|
|
440
|
+
await this.client.execute(driveId, "main", [deleteNode({ id: nodeId })], signal);
|
|
436
441
|
return;
|
|
437
442
|
}
|
|
438
|
-
await this.removeFileNode(
|
|
443
|
+
await this.removeFileNode(driveId, nodeId, signal);
|
|
439
444
|
}
|
|
440
445
|
async renameNode(driveIdentifier, nodeId, name, signal) {
|
|
441
446
|
this.logger.verbose("drives.renameNode(@driveIdentifier, @nodeId, @name)", driveIdentifier, nodeId, name);
|
|
@@ -546,34 +551,41 @@ var DriveClient = class {
|
|
|
546
551
|
}
|
|
547
552
|
}
|
|
548
553
|
async removeFileNode(driveId, fileId, signal) {
|
|
549
|
-
const
|
|
554
|
+
const driveActions = await signActions([deleteNode({ id: fileId })], this.signer, {
|
|
550
555
|
documentId: driveId,
|
|
551
556
|
branch: "main"
|
|
552
557
|
}, signal);
|
|
553
|
-
|
|
558
|
+
await this.runJobs([{
|
|
559
|
+
key: "drive",
|
|
554
560
|
documentId: driveId,
|
|
561
|
+
scope: getSharedActionScope(driveActions),
|
|
562
|
+
branch: "main",
|
|
563
|
+
actions: driveActions,
|
|
564
|
+
dependsOn: []
|
|
565
|
+
}], signal);
|
|
566
|
+
const deleteActions = await signActions([deleteDocumentAction(fileId)], this.signer, {
|
|
567
|
+
documentId: fileId,
|
|
555
568
|
branch: "main"
|
|
556
569
|
}, signal);
|
|
557
|
-
const
|
|
558
|
-
key: "relationship",
|
|
570
|
+
const relationshipActions = await signActions([removeRelationshipAction(driveId, fileId, "child")], this.signer, {
|
|
559
571
|
documentId: driveId,
|
|
560
|
-
|
|
572
|
+
branch: "main"
|
|
573
|
+
}, signal);
|
|
574
|
+
await this.runJobs([{
|
|
575
|
+
key: "delete",
|
|
576
|
+
documentId: fileId,
|
|
577
|
+
scope: getSharedActionScope(deleteActions),
|
|
561
578
|
branch: "main",
|
|
562
|
-
actions:
|
|
579
|
+
actions: deleteActions,
|
|
563
580
|
dependsOn: []
|
|
564
581
|
}, {
|
|
565
|
-
key: "
|
|
582
|
+
key: "relationship",
|
|
566
583
|
documentId: driveId,
|
|
567
|
-
scope: getSharedActionScope(
|
|
584
|
+
scope: getSharedActionScope(relationshipActions),
|
|
568
585
|
branch: "main",
|
|
569
|
-
actions:
|
|
570
|
-
dependsOn: ["
|
|
571
|
-
}]
|
|
572
|
-
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.client.waitForJob(job, signal)));
|
|
573
|
-
for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
|
|
574
|
-
const deleteJob = await this.reactor.deleteDocument(fileId, this.signer, signal);
|
|
575
|
-
const deleteCompleted = await this.client.waitForJob(deleteJob, signal);
|
|
576
|
-
if (deleteCompleted.status === "FAILED") throw new Error(deleteCompleted.error?.message);
|
|
586
|
+
actions: relationshipActions,
|
|
587
|
+
dependsOn: ["delete"]
|
|
588
|
+
}], signal);
|
|
577
589
|
}
|
|
578
590
|
};
|
|
579
591
|
//#endregion
|
|
@@ -849,8 +861,8 @@ var BareReadGate = class {
|
|
|
849
861
|
}
|
|
850
862
|
};
|
|
851
863
|
/**
|
|
852
|
-
* Rethrows a failed read unless the document is confirmed not live.
|
|
853
|
-
*
|
|
864
|
+
* Rethrows a failed read unless the document is confirmed not live. Any error
|
|
865
|
+
* other than DocumentNotFoundError is checked against `exists` rather than
|
|
854
866
|
* inferred from the message, and a failed check surfaces the read's own error.
|
|
855
867
|
*/
|
|
856
868
|
async function assertAbsent(documentView, documentId, error, signal) {
|
|
@@ -885,7 +897,7 @@ var SeededStateReader = class {
|
|
|
885
897
|
* A stream this replica does not hold has to reach buildDecisionModel as the
|
|
886
898
|
* absence it recognises, or the whole read fails instead of leaving the group
|
|
887
899
|
* out of the model, where its principal does not match and the policy fails
|
|
888
|
-
* closed.
|
|
900
|
+
* closed. An error other than DocumentNotFoundError has its absence
|
|
889
901
|
* confirmed rather than inferred from the message: a transient failure must
|
|
890
902
|
* surface, not silently deny.
|
|
891
903
|
*/
|
|
@@ -1203,6 +1215,9 @@ function keepReadableScopes(scopes, readable) {
|
|
|
1203
1215
|
}
|
|
1204
1216
|
//#endregion
|
|
1205
1217
|
//#region src/client/reactor-client.ts
|
|
1218
|
+
function deletePlanKey(documentId) {
|
|
1219
|
+
return `delete:${documentId}`;
|
|
1220
|
+
}
|
|
1206
1221
|
/**
|
|
1207
1222
|
* The document a candidate is decided against. Routed on the action type alone,
|
|
1208
1223
|
* which is what the executor routes on: every action the reactor reduces onto
|
|
@@ -1275,7 +1290,7 @@ var ReactorClient = class {
|
|
|
1275
1290
|
this.eventReads = new EventReadsSource(reactor, documentView, readGate);
|
|
1276
1291
|
this.actionEvaluation = actionEvaluation;
|
|
1277
1292
|
this.createSignaturePolicy = createSignaturePolicy;
|
|
1278
|
-
this.drives = new DriveClient(this, logger, reactor, signer);
|
|
1293
|
+
this.drives = new DriveClient(this, logger, reactor, signer, (id, signal) => this.resolveReference(id, "main", signal));
|
|
1279
1294
|
this.logger.verbose("ReactorClient initialized");
|
|
1280
1295
|
}
|
|
1281
1296
|
/**
|
|
@@ -1523,8 +1538,23 @@ var ReactorClient = class {
|
|
|
1523
1538
|
*/
|
|
1524
1539
|
async create(document, parentIdentifier, signal) {
|
|
1525
1540
|
this.logger.verbose("create(@id, @parentIdentifier)", document.header.id, parentIdentifier);
|
|
1541
|
+
const batchResult = await this.submitCreate(document, parentIdentifier, signal);
|
|
1542
|
+
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
|
|
1543
|
+
for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
|
|
1544
|
+
const created = await this.reactor.get(document.header.id);
|
|
1545
|
+
return this.gateDocument(created, void 0, signal);
|
|
1546
|
+
}
|
|
1547
|
+
/**
|
|
1548
|
+
* Submits a document's create batch without waiting for it
|
|
1549
|
+
*/
|
|
1550
|
+
async createAsync(document, parentIdentifier, signal) {
|
|
1551
|
+
this.logger.verbose("createAsync(@id, @parentIdentifier)", document.header.id, parentIdentifier);
|
|
1552
|
+
return this.submitCreate(document, parentIdentifier, signal);
|
|
1553
|
+
}
|
|
1554
|
+
async submitCreate(document, parentIdentifier, signal) {
|
|
1526
1555
|
const documentId = document.header.id;
|
|
1527
1556
|
const branch = document.header.branch || "main";
|
|
1557
|
+
const parentId = parentIdentifier ? await this.resolveReference(parentIdentifier, "main", signal) : void 0;
|
|
1528
1558
|
const createInput = {
|
|
1529
1559
|
model: document.header.documentType,
|
|
1530
1560
|
version: 0,
|
|
@@ -1560,31 +1590,39 @@ var ReactorClient = class {
|
|
|
1560
1590
|
actions: createActions,
|
|
1561
1591
|
dependsOn: []
|
|
1562
1592
|
}];
|
|
1563
|
-
if (
|
|
1564
|
-
const parentActions = await signActions([addRelationshipAction(
|
|
1565
|
-
documentId:
|
|
1593
|
+
if (parentId) {
|
|
1594
|
+
const parentActions = await signActions([addRelationshipAction(parentId, documentId, "child")], this.signer, {
|
|
1595
|
+
documentId: parentId,
|
|
1566
1596
|
branch: "main"
|
|
1567
1597
|
}, signal);
|
|
1568
1598
|
jobs.push({
|
|
1569
1599
|
key: "parent",
|
|
1570
|
-
documentId:
|
|
1600
|
+
documentId: parentId,
|
|
1571
1601
|
scope: getSharedActionScope(parentActions),
|
|
1572
1602
|
branch: "main",
|
|
1573
1603
|
actions: parentActions,
|
|
1574
1604
|
dependsOn: ["create"]
|
|
1575
1605
|
});
|
|
1576
1606
|
}
|
|
1577
|
-
|
|
1578
|
-
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
|
|
1579
|
-
for (const job of completedJobs) if (job.status === "FAILED") throw new Error(job.error?.message);
|
|
1580
|
-
const created = await this.reactor.get(documentId);
|
|
1581
|
-
return this.gateDocument(created, void 0, signal);
|
|
1607
|
+
return this.reactor.executeBatch({ jobs }, signal);
|
|
1582
1608
|
}
|
|
1583
1609
|
/**
|
|
1584
1610
|
* Creates an empty document and waits for completion
|
|
1585
1611
|
*/
|
|
1586
1612
|
async createEmpty(documentModelType, options, signal) {
|
|
1587
1613
|
this.logger.verbose("createEmpty(@documentModelType, @options)", documentModelType, options);
|
|
1614
|
+
const document = await this.emptyDocument(documentModelType, options, signal);
|
|
1615
|
+
return this.create(document, options?.parentIdentifier, signal);
|
|
1616
|
+
}
|
|
1617
|
+
/**
|
|
1618
|
+
* Submits an empty document's create batch without waiting for it
|
|
1619
|
+
*/
|
|
1620
|
+
async createEmptyAsync(documentModelType, options, signal) {
|
|
1621
|
+
this.logger.verbose("createEmptyAsync(@documentModelType, @options)", documentModelType, options);
|
|
1622
|
+
const document = await this.emptyDocument(documentModelType, options, signal);
|
|
1623
|
+
return this.submitCreate(document, options?.parentIdentifier, signal);
|
|
1624
|
+
}
|
|
1625
|
+
async emptyDocument(documentModelType, options, signal) {
|
|
1588
1626
|
const matchingModules = (await this.reactor.getDocumentModels(void 0, void 0, signal)).results.filter((m) => m.documentModel.global.id === documentModelType);
|
|
1589
1627
|
let module;
|
|
1590
1628
|
if (options?.documentModelVersion !== void 0) {
|
|
@@ -1604,7 +1642,7 @@ var ReactorClient = class {
|
|
|
1604
1642
|
...options?.protocolVersions
|
|
1605
1643
|
} });
|
|
1606
1644
|
document.state.document.version = normalizeDocumentModelVersion(module.version);
|
|
1607
|
-
return
|
|
1645
|
+
return document;
|
|
1608
1646
|
}
|
|
1609
1647
|
/**
|
|
1610
1648
|
* Upgrades a document to a newer document model version by dispatching an
|
|
@@ -1739,10 +1777,12 @@ var ReactorClient = class {
|
|
|
1739
1777
|
*/
|
|
1740
1778
|
async addRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch = "main", signal) {
|
|
1741
1779
|
this.logger.verbose("addRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @metadata, @branch)", sourceIdentifier, targetIdentifier, relationshipType, metadata, branch);
|
|
1742
|
-
const
|
|
1780
|
+
const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
|
|
1781
|
+
const targetId = await this.resolveReference(targetIdentifier, branch, signal);
|
|
1782
|
+
const jobInfo = await this.reactor.addRelationship(sourceId, targetId, relationshipType, metadata, branch, this.signer, signal);
|
|
1743
1783
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
1744
1784
|
if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
|
|
1745
|
-
const result = await this.reactor.getByIdOrSlug(
|
|
1785
|
+
const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
|
|
1746
1786
|
return this.gateDocument(result, { branch }, signal);
|
|
1747
1787
|
}
|
|
1748
1788
|
/**
|
|
@@ -1750,11 +1790,13 @@ var ReactorClient = class {
|
|
|
1750
1790
|
*/
|
|
1751
1791
|
async updateRelationship(sourceIdentifier, targetIdentifier, relationshipType, metadata, branch = "main", signal) {
|
|
1752
1792
|
this.logger.verbose("updateRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @metadata, @branch)", sourceIdentifier, targetIdentifier, relationshipType, metadata, branch);
|
|
1753
|
-
|
|
1754
|
-
const
|
|
1793
|
+
const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
|
|
1794
|
+
const targetId = await this.resolveReference(targetIdentifier, branch, signal);
|
|
1795
|
+
if (!await this.readRelationshipEdge(sourceId, targetId, relationshipType, { branch }, signal)) throw new RelationshipNotFoundError(sourceIdentifier, targetIdentifier, relationshipType);
|
|
1796
|
+
const jobInfo = await this.reactor.updateRelationship(sourceId, targetId, relationshipType, metadata, branch, this.signer, signal);
|
|
1755
1797
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
1756
1798
|
if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
|
|
1757
|
-
const result = await this.reactor.getByIdOrSlug(
|
|
1799
|
+
const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
|
|
1758
1800
|
return this.gateDocument(result, { branch }, signal);
|
|
1759
1801
|
}
|
|
1760
1802
|
/**
|
|
@@ -1762,10 +1804,12 @@ var ReactorClient = class {
|
|
|
1762
1804
|
*/
|
|
1763
1805
|
async removeRelationship(sourceIdentifier, targetIdentifier, relationshipType, branch = "main", signal) {
|
|
1764
1806
|
this.logger.verbose("removeRelationship(@sourceIdentifier, @targetIdentifier, @relationshipType, @branch)", sourceIdentifier, targetIdentifier, relationshipType, branch);
|
|
1765
|
-
const
|
|
1807
|
+
const sourceId = await this.resolveReference(sourceIdentifier, branch, signal);
|
|
1808
|
+
const targetId = await this.resolveReference(targetIdentifier, branch, signal);
|
|
1809
|
+
const jobInfo = await this.reactor.removeRelationship(sourceId, targetId, relationshipType, branch, this.signer, signal);
|
|
1766
1810
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
1767
1811
|
if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
|
|
1768
|
-
const result = await this.reactor.getByIdOrSlug(
|
|
1812
|
+
const result = await this.reactor.getByIdOrSlug(sourceId, { branch }, completedJob.consistencyToken, signal);
|
|
1769
1813
|
return this.gateDocument(result, { branch }, signal);
|
|
1770
1814
|
}
|
|
1771
1815
|
/**
|
|
@@ -1773,15 +1817,18 @@ var ReactorClient = class {
|
|
|
1773
1817
|
*/
|
|
1774
1818
|
async moveRelationship(sourceParentIdentifier, targetParentIdentifier, targetIdentifier, relationshipType, branch = "main", signal) {
|
|
1775
1819
|
this.logger.verbose("moveRelationship(@sourceParentIdentifier, @targetParentIdentifier, @targetIdentifier, @relationshipType, @branch)", sourceParentIdentifier, targetParentIdentifier, targetIdentifier, relationshipType, branch);
|
|
1776
|
-
const
|
|
1777
|
-
const
|
|
1820
|
+
const sourceParentId = await this.resolveReference(sourceParentIdentifier, branch, signal);
|
|
1821
|
+
const targetParentId = await this.resolveReference(targetParentIdentifier, branch, signal);
|
|
1822
|
+
const targetId = await this.resolveReference(targetIdentifier, branch, signal);
|
|
1823
|
+
const metadata = (await this.readRelationshipEdge(sourceParentId, targetId, relationshipType, { branch }, signal))?.metadata;
|
|
1824
|
+
const removeJobInfo = await this.reactor.removeRelationship(sourceParentId, targetId, relationshipType, branch, this.signer, signal);
|
|
1778
1825
|
const removeCompletedJob = await this.waitForJob(removeJobInfo, signal);
|
|
1779
1826
|
if (removeCompletedJob.status === "FAILED") throw new Error(removeCompletedJob.error?.message);
|
|
1780
|
-
const addJobInfo = await this.reactor.addRelationship(
|
|
1827
|
+
const addJobInfo = await this.reactor.addRelationship(targetParentId, targetId, relationshipType, metadata, branch, this.signer, signal);
|
|
1781
1828
|
const addCompletedJob = await this.waitForJob(addJobInfo, signal);
|
|
1782
1829
|
if (addCompletedJob.status === "FAILED") throw new Error(addCompletedJob.error?.message);
|
|
1783
|
-
const sourceResult = await this.reactor.getByIdOrSlug(
|
|
1784
|
-
const targetResult = await this.reactor.getByIdOrSlug(
|
|
1830
|
+
const sourceResult = await this.reactor.getByIdOrSlug(sourceParentId, { branch }, removeCompletedJob.consistencyToken, signal);
|
|
1831
|
+
const targetResult = await this.reactor.getByIdOrSlug(targetParentId, { branch }, addCompletedJob.consistencyToken, signal);
|
|
1785
1832
|
return {
|
|
1786
1833
|
source: await this.gateDocument(sourceResult, { branch }, signal),
|
|
1787
1834
|
target: await this.gateDocument(targetResult, { branch }, signal)
|
|
@@ -1801,9 +1848,9 @@ var ReactorClient = class {
|
|
|
1801
1848
|
*/
|
|
1802
1849
|
async deleteDocument(identifier, propagate, signal) {
|
|
1803
1850
|
this.logger.verbose("deleteDocument(@identifier, @propagate)", identifier, propagate);
|
|
1804
|
-
const
|
|
1851
|
+
const rootId = await this.resolveReference(identifier, "main", signal);
|
|
1852
|
+
const toDelete = /* @__PURE__ */ new Set([rootId]);
|
|
1805
1853
|
if (propagate === "cascade") {
|
|
1806
|
-
const toDelete = /* @__PURE__ */ new Set([identifier]);
|
|
1807
1854
|
let changed = true;
|
|
1808
1855
|
while (changed) {
|
|
1809
1856
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
@@ -1814,19 +1861,33 @@ var ReactorClient = class {
|
|
|
1814
1861
|
changed = true;
|
|
1815
1862
|
}
|
|
1816
1863
|
}
|
|
1817
|
-
for (const descendantId of toDelete) {
|
|
1818
|
-
if (descendantId === identifier) continue;
|
|
1819
|
-
const removalJobs = await this.removeAllIncomingRelationships(descendantId, signal);
|
|
1820
|
-
jobs.push(...removalJobs);
|
|
1821
|
-
const jobInfo = await this.reactor.deleteDocument(descendantId, this.signer, signal);
|
|
1822
|
-
jobs.push(jobInfo);
|
|
1823
|
-
}
|
|
1824
1864
|
}
|
|
1825
|
-
const
|
|
1826
|
-
|
|
1827
|
-
const
|
|
1828
|
-
|
|
1829
|
-
|
|
1865
|
+
const order = [...toDelete].reverse();
|
|
1866
|
+
const position = new Map(order.map((id, index) => [id, index]));
|
|
1867
|
+
const incoming = /* @__PURE__ */ new Map();
|
|
1868
|
+
for (const documentId of order) {
|
|
1869
|
+
const page = await this.documentIndexer.getIncoming(documentId, void 0, void 0, void 0, signal);
|
|
1870
|
+
incoming.set(documentId, page.results);
|
|
1871
|
+
}
|
|
1872
|
+
const waitsForTarget = (rel) => {
|
|
1873
|
+
const source = position.get(rel.sourceId);
|
|
1874
|
+
return source === void 0 || source > position.get(rel.targetId);
|
|
1875
|
+
};
|
|
1876
|
+
const leadingBySource = /* @__PURE__ */ new Map();
|
|
1877
|
+
for (const rels of incoming.values()) for (const rel of rels) {
|
|
1878
|
+
if (waitsForTarget(rel)) continue;
|
|
1879
|
+
const leading = leadingBySource.get(rel.sourceId) ?? [];
|
|
1880
|
+
leading.push(rel);
|
|
1881
|
+
leadingBySource.set(rel.sourceId, leading);
|
|
1882
|
+
}
|
|
1883
|
+
const plans = [];
|
|
1884
|
+
for (const documentId of order) {
|
|
1885
|
+
for (const rel of leadingBySource.get(documentId) ?? []) plans.push(await this.planRemoval(rel, [], plans.length, signal));
|
|
1886
|
+
plans.push(await this.planDelete(documentId, signal));
|
|
1887
|
+
for (const rel of incoming.get(documentId)) if (waitsForTarget(rel)) plans.push(await this.planRemoval(rel, [deletePlanKey(documentId)], plans.length, signal));
|
|
1888
|
+
}
|
|
1889
|
+
const batchResult = await this.reactor.executeBatch({ jobs: plans }, signal);
|
|
1890
|
+
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
|
|
1830
1891
|
for (const completedJob of completedJobs) if (completedJob.status === "FAILED") throw new Error(completedJob.error?.message);
|
|
1831
1892
|
}
|
|
1832
1893
|
/**
|
|
@@ -1896,12 +1957,16 @@ var ReactorClient = class {
|
|
|
1896
1957
|
return served("created", documents, reads);
|
|
1897
1958
|
})());
|
|
1898
1959
|
}, search);
|
|
1899
|
-
const unsubscribeDeleted = this.subscriptionManager.onDocumentDeleted((documentIds) => {
|
|
1960
|
+
const unsubscribeDeleted = this.subscriptionManager.onDocumentDeleted((documentIds, info) => {
|
|
1900
1961
|
const reads = this.eventReads.forEvent();
|
|
1962
|
+
const purged = info?.purged ? { purged: true } : {};
|
|
1901
1963
|
for (const childId of documentIds) deliver((async () => await this.servesEvery([childId], view, reads) ? {
|
|
1902
1964
|
type: "deleted",
|
|
1903
1965
|
documents: [],
|
|
1904
|
-
context: {
|
|
1966
|
+
context: {
|
|
1967
|
+
childId,
|
|
1968
|
+
...purged
|
|
1969
|
+
}
|
|
1905
1970
|
} : void 0)());
|
|
1906
1971
|
}, search);
|
|
1907
1972
|
const unsubscribeUpdated = this.subscriptionManager.onDocumentStateUpdated((result) => {
|
|
@@ -2102,11 +2167,18 @@ var ReactorClient = class {
|
|
|
2102
2167
|
}
|
|
2103
2168
|
/**
|
|
2104
2169
|
* The id a write on `identifier` is stored under, which its signatures bind.
|
|
2105
|
-
* A create names its own id
|
|
2106
|
-
* a document still in flight resolves to itself.
|
|
2170
|
+
* A create names its own id.
|
|
2107
2171
|
*/
|
|
2108
2172
|
async resolveWriteTarget(identifier, branch, actions, signal) {
|
|
2109
2173
|
if (actions.some((action) => action.type === "CREATE_DOCUMENT")) return identifier;
|
|
2174
|
+
return this.resolveReference(identifier, branch, signal);
|
|
2175
|
+
}
|
|
2176
|
+
/**
|
|
2177
|
+
* The id a document reference names. An id no slug maps to is taken as
|
|
2178
|
+
* given, so a document still in flight resolves to itself; an id that is
|
|
2179
|
+
* another document's slug is refused as ambiguous.
|
|
2180
|
+
*/
|
|
2181
|
+
async resolveReference(identifier, branch, signal) {
|
|
2110
2182
|
const view = { branch };
|
|
2111
2183
|
const bySlug = await this.documentView.resolveSlug(identifier, view, void 0, signal);
|
|
2112
2184
|
if (bySlug === void 0 || bySlug === identifier) return identifier;
|
|
@@ -2145,14 +2217,33 @@ var ReactorClient = class {
|
|
|
2145
2217
|
scopeStates: document.state ?? {}
|
|
2146
2218
|
};
|
|
2147
2219
|
}
|
|
2148
|
-
async
|
|
2149
|
-
const
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2220
|
+
async planDelete(documentId, signal) {
|
|
2221
|
+
const signed = await signActions([deleteDocumentAction(documentId)], this.signer, {
|
|
2222
|
+
documentId,
|
|
2223
|
+
branch: "main"
|
|
2224
|
+
}, signal);
|
|
2225
|
+
return {
|
|
2226
|
+
key: deletePlanKey(documentId),
|
|
2227
|
+
documentId,
|
|
2228
|
+
scope: "document",
|
|
2229
|
+
branch: "main",
|
|
2230
|
+
actions: signed,
|
|
2231
|
+
dependsOn: []
|
|
2232
|
+
};
|
|
2233
|
+
}
|
|
2234
|
+
async planRemoval(rel, dependsOn, index, signal) {
|
|
2235
|
+
const signed = await signActions([removeRelationshipAction(rel.sourceId, rel.targetId, rel.relationshipType)], this.signer, {
|
|
2236
|
+
documentId: rel.sourceId,
|
|
2237
|
+
branch: "main"
|
|
2238
|
+
}, signal);
|
|
2239
|
+
return {
|
|
2240
|
+
key: `remove:${index}`,
|
|
2241
|
+
documentId: rel.sourceId,
|
|
2242
|
+
scope: "document",
|
|
2243
|
+
branch: "main",
|
|
2244
|
+
actions: signed,
|
|
2245
|
+
dependsOn
|
|
2246
|
+
};
|
|
2156
2247
|
}
|
|
2157
2248
|
};
|
|
2158
2249
|
//#endregion
|
|
@@ -2178,11 +2269,15 @@ const QueueEventTypes = { JOB_AVAILABLE: 1e4 };
|
|
|
2178
2269
|
/** Conflict retries a job may take without charging its retry limit. */
|
|
2179
2270
|
const MAX_EXEMPT_CONFLICT_RETRIES = 20;
|
|
2180
2271
|
function toErrorInfo(error) {
|
|
2181
|
-
if (error instanceof Error)
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2272
|
+
if (error instanceof Error) {
|
|
2273
|
+
const documentId = error.documentId;
|
|
2274
|
+
return {
|
|
2275
|
+
name: error.name,
|
|
2276
|
+
message: error.message,
|
|
2277
|
+
stack: error.stack || (/* @__PURE__ */ new Error()).stack || "",
|
|
2278
|
+
...typeof documentId === "string" ? { documentId } : {}
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2186
2281
|
return {
|
|
2187
2282
|
name: "Error",
|
|
2188
2283
|
message: error,
|
|
@@ -2229,12 +2324,12 @@ var JobResultHandler = class {
|
|
|
2229
2324
|
return;
|
|
2230
2325
|
} catch {}
|
|
2231
2326
|
}
|
|
2232
|
-
if (result.error && DocumentNotFoundError.isError(result.error) && handle.job.kind === "load") {
|
|
2327
|
+
if (result.error && DocumentNotFoundError.isError(result.error) && !DocumentPurgedError.isError(result.error) && handle.job.kind === "load") {
|
|
2233
2328
|
handle.defer();
|
|
2234
2329
|
callbacks.deferJob(result.error.documentId, handle.job);
|
|
2235
2330
|
return;
|
|
2236
2331
|
}
|
|
2237
|
-
if (result.error && (DocumentDeletedError.isError(result.error) || DocumentAlreadyExistsError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || InvalidSignatureError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error) || UnsupportedProtocolVersionError.isError(result.error))) {
|
|
2332
|
+
if (result.error && (DocumentDeletedError.isError(result.error) || DocumentAlreadyExistsError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || InvalidSignatureError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error) || UnsupportedProtocolVersionError.isError(result.error) || DocumentPurgedError.isError(result.error) || DocumentNotDeletedError.isError(result.error) || GroupInUseError.isError(result.error) || PurgeTooLargeError.isError(result.error) || ReservedActionError.isError(result.error))) {
|
|
2238
2333
|
const errorInfo = toErrorInfo(result.error);
|
|
2239
2334
|
this.jobTracker.markFailed(handle.job.id, errorInfo, handle.job);
|
|
2240
2335
|
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
@@ -2654,9 +2749,12 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
2654
2749
|
};
|
|
2655
2750
|
this.eventBus.emit(JobExecutorEventTypes.JOB_FAILED, failedEvent).catch(() => {});
|
|
2656
2751
|
}
|
|
2657
|
-
if (outcome.result.success && outcome.writeReady)
|
|
2658
|
-
this.
|
|
2659
|
-
|
|
2752
|
+
if (outcome.result.success && outcome.writeReady) {
|
|
2753
|
+
this.evictPurged(outcome.writeReady);
|
|
2754
|
+
this.emitWriteReady(handle.job, outcome.writeReady).catch((error) => {
|
|
2755
|
+
this.logger.error("emitWriteReady failed for job @jobId: @Error", { jobId: handle.job.id }, error);
|
|
2756
|
+
});
|
|
2757
|
+
}
|
|
2660
2758
|
await this.resultHandler.handleResult(handle, outcome.result, {
|
|
2661
2759
|
deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
|
|
2662
2760
|
flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
|
|
@@ -2664,6 +2762,12 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
2664
2762
|
this.activeJobs--;
|
|
2665
2763
|
await this.tryDispatchFor(worker);
|
|
2666
2764
|
}
|
|
2765
|
+
/** Every worker drops a committed purge's id; routing covers only its own. */
|
|
2766
|
+
evictPurged(payload) {
|
|
2767
|
+
const documentIds = [...new Set(payload.operations.filter((operation) => isPurgeMarker$1(operation)).map((operation) => operation.context.documentId))];
|
|
2768
|
+
if (documentIds.length === 0) return;
|
|
2769
|
+
for (const worker of this.workers) worker.evictPurged(documentIds);
|
|
2770
|
+
}
|
|
2667
2771
|
async emitWriteReady(job, payload) {
|
|
2668
2772
|
const documentIds = [...new Set(payload.operations.map((op) => op.context.documentId))];
|
|
2669
2773
|
let collectionMemberships = {};
|
|
@@ -2755,8 +2859,78 @@ function fillMissingMemberships(documentIds, found) {
|
|
|
2755
2859
|
return result;
|
|
2756
2860
|
}
|
|
2757
2861
|
//#endregion
|
|
2862
|
+
//#region src/admin/document-purge-service.ts
|
|
2863
|
+
/** The only local origin of a purge: one purge job per erased document. */
|
|
2864
|
+
var DocumentPurgeService = class DocumentPurgeService {
|
|
2865
|
+
queue;
|
|
2866
|
+
jobTracker;
|
|
2867
|
+
eventBus;
|
|
2868
|
+
constructor(queue, jobTracker, eventBus) {
|
|
2869
|
+
this.queue = queue;
|
|
2870
|
+
this.jobTracker = jobTracker;
|
|
2871
|
+
this.eventBus = eventBus;
|
|
2872
|
+
}
|
|
2873
|
+
static fromModule(module) {
|
|
2874
|
+
return new DocumentPurgeService(module.queue, module.jobTracker, module.eventBus);
|
|
2875
|
+
}
|
|
2876
|
+
/** One purge job per id; resolves once enqueued, not once complete. */
|
|
2877
|
+
async enqueuePurge(ids, requestId, opts = {}) {
|
|
2878
|
+
const requestDocumentIds = [...new Set(ids)];
|
|
2879
|
+
const infos = [];
|
|
2880
|
+
for (const documentId of requestDocumentIds) {
|
|
2881
|
+
const jobId = v4();
|
|
2882
|
+
const createdAtUtcIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
2883
|
+
const meta = buildSingleJobMeta(jobId, {
|
|
2884
|
+
purgeRequestId: requestId,
|
|
2885
|
+
purgeRequestDocumentIds: requestDocumentIds
|
|
2886
|
+
});
|
|
2887
|
+
const job = {
|
|
2888
|
+
id: jobId,
|
|
2889
|
+
kind: "purge",
|
|
2890
|
+
documentId,
|
|
2891
|
+
scope: "document",
|
|
2892
|
+
branch: "main",
|
|
2893
|
+
actions: [],
|
|
2894
|
+
operations: [],
|
|
2895
|
+
createdAt: createdAtUtcIso,
|
|
2896
|
+
queueHint: [],
|
|
2897
|
+
maxRetries: 3,
|
|
2898
|
+
errorHistory: [],
|
|
2899
|
+
meta,
|
|
2900
|
+
purge: {
|
|
2901
|
+
requestId,
|
|
2902
|
+
allowLarge: opts.allowLarge ?? false
|
|
2903
|
+
}
|
|
2904
|
+
};
|
|
2905
|
+
const info = {
|
|
2906
|
+
id: jobId,
|
|
2907
|
+
documentId,
|
|
2908
|
+
status: "PENDING",
|
|
2909
|
+
createdAtUtcIso,
|
|
2910
|
+
consistencyToken: {
|
|
2911
|
+
version: 1,
|
|
2912
|
+
createdAtUtcIso,
|
|
2913
|
+
coordinates: []
|
|
2914
|
+
},
|
|
2915
|
+
meta
|
|
2916
|
+
};
|
|
2917
|
+
this.jobTracker.registerJob(info);
|
|
2918
|
+
const pending = {
|
|
2919
|
+
jobId,
|
|
2920
|
+
jobMeta: meta
|
|
2921
|
+
};
|
|
2922
|
+
this.eventBus.emit(ReactorEventTypes.JOB_PENDING, pending).catch(() => {});
|
|
2923
|
+
const dispatched = this.queue.enqueue(job);
|
|
2924
|
+
dispatched.catch(() => {});
|
|
2925
|
+
await Promise.race([dispatched, Promise.resolve()]);
|
|
2926
|
+
infos.push(info);
|
|
2927
|
+
}
|
|
2928
|
+
return infos;
|
|
2929
|
+
}
|
|
2930
|
+
};
|
|
2931
|
+
//#endregion
|
|
2758
2932
|
//#region src/storage/kysely/stored-protocol-versions.ts
|
|
2759
|
-
const created = sql.raw(`scope = 'document' and "index" = 0 and (action->'input'->'protocolVersions') is not null`);
|
|
2933
|
+
const created = sql.raw(`scope = 'document' and "index" = 0 and (action->'input'->'protocolVersions') is not null and action->>'type' <> 'PURGE_DOCUMENT'`);
|
|
2760
2934
|
const key = sql.raw(`md5((action->'input'->'protocolVersions')::text)`);
|
|
2761
2935
|
/** Distinct creation protocolVersions, one index probe per value. */
|
|
2762
2936
|
async function storedProtocolVersions(db, schema) {
|
|
@@ -3108,14 +3282,23 @@ var InMemoryJobTracker = class {
|
|
|
3108
3282
|
}
|
|
3109
3283
|
};
|
|
3110
3284
|
//#endregion
|
|
3285
|
+
//#region src/read-models/types.ts
|
|
3286
|
+
/** A cursor row whose factory is unregistered; it only records a drive's data. */
|
|
3287
|
+
const RELEASED_CURSOR_STATUS = "released";
|
|
3288
|
+
//#endregion
|
|
3111
3289
|
//#region src/processors/utils.ts
|
|
3112
3290
|
function isDriveDeletion(op) {
|
|
3113
|
-
|
|
3291
|
+
const type = op.operation.action.type;
|
|
3292
|
+
return type === "DELETE_DOCUMENT" || type === "PURGE_DOCUMENT";
|
|
3114
3293
|
}
|
|
3115
3294
|
function extractDriveHeader(op) {
|
|
3116
3295
|
if (!op.context.resultingState) return void 0;
|
|
3117
3296
|
return JSON.parse(op.context.resultingState).header;
|
|
3118
3297
|
}
|
|
3298
|
+
function extractCreationHeader(op) {
|
|
3299
|
+
if (op.operation.action.type !== "CREATE_DOCUMENT") return void 0;
|
|
3300
|
+
return extractDriveHeader(op) ?? createDocumentFromAction(op.operation.action).header;
|
|
3301
|
+
}
|
|
3119
3302
|
function extractDeletedDocumentId(op) {
|
|
3120
3303
|
return op.operation.action.input.documentId ?? op.context.documentId;
|
|
3121
3304
|
}
|
|
@@ -3183,11 +3366,25 @@ function lowestOf(ops) {
|
|
|
3183
3366
|
for (const op of ops) lowest = Math.min(lowest, op.context.ordinal);
|
|
3184
3367
|
return lowest;
|
|
3185
3368
|
}
|
|
3369
|
+
/** Drops operations of tombstoned ids; the marker always passes. */
|
|
3370
|
+
function withoutPurged(ops, purged, keep) {
|
|
3371
|
+
if (purged.size === 0) return ops;
|
|
3372
|
+
return ops.filter((op) => isPurgeMarker$1(op.operation) || !purged.has(op.context.documentId) || keep?.has(op.context.ordinal) === true);
|
|
3373
|
+
}
|
|
3374
|
+
/** The ids whose tombstone would drop an operation of `ops`. */
|
|
3375
|
+
function purgeCandidates(ops) {
|
|
3376
|
+
const ids = /* @__PURE__ */ new Set();
|
|
3377
|
+
for (const op of ops) if (!isPurgeMarker$1(op.operation)) ids.add(op.context.documentId);
|
|
3378
|
+
return [...ids];
|
|
3379
|
+
}
|
|
3186
3380
|
function highestOf(ops) {
|
|
3187
3381
|
let highest = 0;
|
|
3188
3382
|
for (const op of ops) highest = Math.max(highest, op.context.ordinal);
|
|
3189
3383
|
return highest;
|
|
3190
3384
|
}
|
|
3385
|
+
const LOOKUP_RETRY_MS = 100;
|
|
3386
|
+
const MAX_LOOKUP_RETRY_MS = 3e4;
|
|
3387
|
+
const LOOKUP_RETRIES = 10;
|
|
3191
3388
|
var ProcessorQueue = class {
|
|
3192
3389
|
options;
|
|
3193
3390
|
tasks = [];
|
|
@@ -3195,17 +3392,24 @@ var ProcessorQueue = class {
|
|
|
3195
3392
|
closed = false;
|
|
3196
3393
|
replaysAhead = 0;
|
|
3197
3394
|
unrouted = /* @__PURE__ */ new Set();
|
|
3395
|
+
wake;
|
|
3396
|
+
parked = false;
|
|
3198
3397
|
constructor(options) {
|
|
3199
3398
|
this.options = options;
|
|
3200
3399
|
}
|
|
3201
3400
|
get isClosed() {
|
|
3202
3401
|
return this.closed;
|
|
3203
3402
|
}
|
|
3403
|
+
/** True while a backfill or retry is queued or running. */
|
|
3404
|
+
get replaying() {
|
|
3405
|
+
return this.replaysAhead > 0;
|
|
3406
|
+
}
|
|
3204
3407
|
/** Resolves at once behind a replay, so a pass never waits out a backfill. */
|
|
3205
|
-
live(ops) {
|
|
3408
|
+
live(ops, check) {
|
|
3206
3409
|
const delivered = this.push((done) => ({
|
|
3207
3410
|
kind: "live",
|
|
3208
3411
|
ops,
|
|
3412
|
+
check,
|
|
3209
3413
|
done
|
|
3210
3414
|
}));
|
|
3211
3415
|
return this.replaysAhead > 0 ? Promise.resolve() : delivered;
|
|
@@ -3223,6 +3427,21 @@ var ProcessorQueue = class {
|
|
|
3223
3427
|
done
|
|
3224
3428
|
}));
|
|
3225
3429
|
}
|
|
3430
|
+
/** `share` if active, then `deletion` whatever the status; true once delivered. */
|
|
3431
|
+
erase(share, deletion, check) {
|
|
3432
|
+
if (this.closed) return Promise.resolve(false);
|
|
3433
|
+
let delivered = false;
|
|
3434
|
+
const task = {
|
|
3435
|
+
kind: "erase",
|
|
3436
|
+
ops: share,
|
|
3437
|
+
deletion,
|
|
3438
|
+
check,
|
|
3439
|
+
run: async () => {
|
|
3440
|
+
delivered = await this.runErase(task);
|
|
3441
|
+
}
|
|
3442
|
+
};
|
|
3443
|
+
return this.push((done) => Object.assign(task, { done })).then(() => delivered);
|
|
3444
|
+
}
|
|
3226
3445
|
/** Replays from the cursor as it stands when the task runs. */
|
|
3227
3446
|
backfill() {
|
|
3228
3447
|
return this.replay("backfill", () => this.runBackfill());
|
|
@@ -3250,6 +3469,7 @@ var ProcessorQueue = class {
|
|
|
3250
3469
|
}
|
|
3251
3470
|
});
|
|
3252
3471
|
this.closed = true;
|
|
3472
|
+
this.wake?.();
|
|
3253
3473
|
return disconnect;
|
|
3254
3474
|
}
|
|
3255
3475
|
replay(kind, run) {
|
|
@@ -3320,19 +3540,121 @@ var ProcessorQueue = class {
|
|
|
3320
3540
|
await this.raiseCursor(through);
|
|
3321
3541
|
return;
|
|
3322
3542
|
}
|
|
3323
|
-
if (cursor.status !== "active") {
|
|
3543
|
+
if (cursor.status !== "active" || this.parked) {
|
|
3324
3544
|
await this.parkBelow(fresh);
|
|
3325
3545
|
return;
|
|
3326
3546
|
}
|
|
3327
|
-
|
|
3547
|
+
const live = await this.dropPurgedLive(batch, fresh);
|
|
3548
|
+
if (live === void 0) {
|
|
3549
|
+
await this.parkBelow(fresh);
|
|
3550
|
+
return;
|
|
3551
|
+
}
|
|
3552
|
+
if (live.length > 0 && !await this.deliver(live)) {
|
|
3328
3553
|
await this.parkBelow(fresh);
|
|
3329
3554
|
return;
|
|
3330
3555
|
}
|
|
3331
3556
|
await this.raiseCursor(Math.max(highestOf(fresh), through));
|
|
3332
3557
|
}
|
|
3558
|
+
/** Undefined once closed, or parked after the lookup's last retry. */
|
|
3559
|
+
async dropPurgedLive(batch, fresh) {
|
|
3560
|
+
const keep = /* @__PURE__ */ new Set();
|
|
3561
|
+
const checks = [];
|
|
3562
|
+
for (const task of batch) {
|
|
3563
|
+
if (task.kind !== "live" || !task.check) continue;
|
|
3564
|
+
checks.push(task.check.purged);
|
|
3565
|
+
for (const ordinal of task.check.keep ?? []) keep.add(ordinal);
|
|
3566
|
+
}
|
|
3567
|
+
let purged;
|
|
3568
|
+
try {
|
|
3569
|
+
const sets = await Promise.all(checks);
|
|
3570
|
+
purged = new Set(sets.flatMap((set) => [...set]));
|
|
3571
|
+
} catch (error) {
|
|
3572
|
+
purged = await this.retryLookup(fresh, error);
|
|
3573
|
+
}
|
|
3574
|
+
return purged && withoutPurged(fresh, purged, keep);
|
|
3575
|
+
}
|
|
3576
|
+
async retryLookup(ops, error) {
|
|
3577
|
+
const ids = purgeCandidates(ops);
|
|
3578
|
+
let wait = this.options.lookupRetryMs ?? LOOKUP_RETRY_MS;
|
|
3579
|
+
const retries = this.options.lookupRetries ?? LOOKUP_RETRIES;
|
|
3580
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
3581
|
+
this.options.logger.error("Processor '@ProcessorId' failed checking tombstones, retrying in @Wait ms: @Error", this.options.processorId, wait, error);
|
|
3582
|
+
await this.sleep(wait);
|
|
3583
|
+
if (this.closed) return void 0;
|
|
3584
|
+
try {
|
|
3585
|
+
return await this.options.purged(ids);
|
|
3586
|
+
} catch (retryError) {
|
|
3587
|
+
error = retryError;
|
|
3588
|
+
wait = Math.min(wait * 2, MAX_LOOKUP_RETRY_MS);
|
|
3589
|
+
}
|
|
3590
|
+
}
|
|
3591
|
+
await this.park(ops, ids, wait, error);
|
|
3592
|
+
}
|
|
3593
|
+
async park(ops, ids, wait, error) {
|
|
3594
|
+
this.parked = true;
|
|
3595
|
+
this.noteLookupError(error);
|
|
3596
|
+
this.options.logger.error("Processor '@ProcessorId' parked: tombstones unread after @Retries retries, retrying every @Wait ms: @Error", this.options.processorId, this.options.lookupRetries ?? LOOKUP_RETRIES, wait, error);
|
|
3597
|
+
this.replay("backfill", () => this.resume(ids, wait));
|
|
3598
|
+
await this.parkBelow(ops);
|
|
3599
|
+
}
|
|
3600
|
+
async resume(ids, wait) {
|
|
3601
|
+
for (;;) {
|
|
3602
|
+
if (this.closed) return;
|
|
3603
|
+
await this.sleep(wait);
|
|
3604
|
+
if (this.closed) return;
|
|
3605
|
+
try {
|
|
3606
|
+
await this.options.purged(ids);
|
|
3607
|
+
break;
|
|
3608
|
+
} catch (error) {
|
|
3609
|
+
this.noteLookupError(error);
|
|
3610
|
+
this.options.logger.error("Processor '@ProcessorId' still parked, tombstones unread; retrying in @Wait ms: @Error", this.options.processorId, wait, error);
|
|
3611
|
+
}
|
|
3612
|
+
}
|
|
3613
|
+
this.parked = false;
|
|
3614
|
+
const { cursor } = this.options;
|
|
3615
|
+
cursor.lastError = void 0;
|
|
3616
|
+
cursor.lastErrorTimestamp = void 0;
|
|
3617
|
+
await this.persist();
|
|
3618
|
+
this.options.logger.info("Processor '@ProcessorId' resumed from ordinal @Ordinal", this.options.processorId, cursor.lastOrdinal);
|
|
3619
|
+
await this.runBackfill();
|
|
3620
|
+
}
|
|
3621
|
+
noteLookupError(error) {
|
|
3622
|
+
const { cursor } = this.options;
|
|
3623
|
+
cursor.lastError = errorMessage(error);
|
|
3624
|
+
cursor.lastErrorTimestamp = /* @__PURE__ */ new Date();
|
|
3625
|
+
}
|
|
3626
|
+
sleep(ms) {
|
|
3627
|
+
return new Promise((resolve) => {
|
|
3628
|
+
const done = () => {
|
|
3629
|
+
clearTimeout(timer);
|
|
3630
|
+
this.wake = void 0;
|
|
3631
|
+
resolve();
|
|
3632
|
+
};
|
|
3633
|
+
const timer = setTimeout(done, ms);
|
|
3634
|
+
timer.unref?.();
|
|
3635
|
+
this.wake = done;
|
|
3636
|
+
});
|
|
3637
|
+
}
|
|
3638
|
+
async runErase(task) {
|
|
3639
|
+
const { cursor, floor } = this.options;
|
|
3640
|
+
const fresh = task.ops.filter((op) => {
|
|
3641
|
+
const ordinal = op.context.ordinal;
|
|
3642
|
+
if (ordinal <= floor) return false;
|
|
3643
|
+
return !this.unrouted.delete(ordinal);
|
|
3644
|
+
});
|
|
3645
|
+
const deletion = task.deletion && !this.unrouted.delete(task.deletion.context.ordinal) ? task.deletion : void 0;
|
|
3646
|
+
let ops = [];
|
|
3647
|
+
if (cursor.status === "active" && !this.parked && fresh.length > 0) try {
|
|
3648
|
+
ops = withoutPurged(fresh, await task.check.purged);
|
|
3649
|
+
} catch (error) {
|
|
3650
|
+
this.options.logger.error("Processor '@ProcessorId' skipped its drive's last batch, tombstones unread: @Error", this.options.processorId, error);
|
|
3651
|
+
}
|
|
3652
|
+
if (deletion) ops.push(deletion);
|
|
3653
|
+
return ops.length === 0 || await this.deliver(ops);
|
|
3654
|
+
}
|
|
3333
3655
|
async runBackfill() {
|
|
3334
3656
|
const { cursor, filter, floor } = this.options;
|
|
3335
|
-
if (cursor.status !== "active") return;
|
|
3657
|
+
if (cursor.status !== "active" || this.parked) return;
|
|
3336
3658
|
let page;
|
|
3337
3659
|
try {
|
|
3338
3660
|
page = await this.options.readSince(cursor.lastOrdinal);
|
|
@@ -3343,13 +3665,28 @@ var ProcessorQueue = class {
|
|
|
3343
3665
|
while (page.results.length > 0) {
|
|
3344
3666
|
if (this.closed) return;
|
|
3345
3667
|
const matching = page.results.filter((op) => op.context.ordinal > floor && matchesFilter(op, filter));
|
|
3668
|
+
let live = matching;
|
|
3346
3669
|
if (matching.length > 0) {
|
|
3347
|
-
|
|
3670
|
+
const ids = purgeCandidates(matching);
|
|
3671
|
+
let purged;
|
|
3672
|
+
try {
|
|
3673
|
+
purged = await this.options.purged(ids);
|
|
3674
|
+
} catch (error) {
|
|
3675
|
+
purged = await this.retryLookup(matching, error);
|
|
3676
|
+
}
|
|
3677
|
+
if (purged === void 0) {
|
|
3348
3678
|
await this.persist();
|
|
3349
3679
|
return;
|
|
3350
3680
|
}
|
|
3351
|
-
|
|
3681
|
+
live = withoutPurged(matching, purged);
|
|
3352
3682
|
}
|
|
3683
|
+
if (live.length > 0) {
|
|
3684
|
+
if (!await this.deliver(live)) {
|
|
3685
|
+
await this.persist();
|
|
3686
|
+
return;
|
|
3687
|
+
}
|
|
3688
|
+
}
|
|
3689
|
+
if (matching.length > 0) this.dedupeQueued(matching);
|
|
3353
3690
|
await this.raiseCursor(highestOf(page.results));
|
|
3354
3691
|
if (!page.next) break;
|
|
3355
3692
|
try {
|
|
@@ -3370,7 +3707,10 @@ var ProcessorQueue = class {
|
|
|
3370
3707
|
}
|
|
3371
3708
|
if (queued.size === 0) return;
|
|
3372
3709
|
for (const task of this.tasks) {
|
|
3373
|
-
if (task.kind
|
|
3710
|
+
if (task.kind === "erase") {
|
|
3711
|
+
const { deletion } = task;
|
|
3712
|
+
if (deletion && queued.has(deletion.context.ordinal)) task.deletion = void 0;
|
|
3713
|
+
} else if (task.kind !== "live") continue;
|
|
3374
3714
|
task.ops = task.ops.filter((op) => !queued.has(op.context.ordinal));
|
|
3375
3715
|
}
|
|
3376
3716
|
}
|
|
@@ -3398,7 +3738,8 @@ var ProcessorQueue = class {
|
|
|
3398
3738
|
async raiseCursor(through) {
|
|
3399
3739
|
const { cursor } = this.options;
|
|
3400
3740
|
const capped = Math.min(through, this.options.confirmedThrough());
|
|
3401
|
-
if (cursor.status !== "active" ||
|
|
3741
|
+
if (cursor.status !== "active" || this.parked) return;
|
|
3742
|
+
if (capped <= cursor.lastOrdinal) return;
|
|
3402
3743
|
cursor.lastOrdinal = capped;
|
|
3403
3744
|
await this.persist();
|
|
3404
3745
|
}
|
|
@@ -3424,6 +3765,8 @@ var ProcessorQueue = class {
|
|
|
3424
3765
|
};
|
|
3425
3766
|
//#endregion
|
|
3426
3767
|
//#region src/processors/processor-manager.ts
|
|
3768
|
+
const DRIVE_STREAM_PAGE = 500;
|
|
3769
|
+
const SHUTDOWN_GRACE_MS = 5e3;
|
|
3427
3770
|
var ProcessorManager = class extends BaseReadModel {
|
|
3428
3771
|
factoryRegistry = /* @__PURE__ */ new Map();
|
|
3429
3772
|
processorsByDrive = /* @__PURE__ */ new Map();
|
|
@@ -3435,6 +3778,14 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3435
3778
|
advancing = Promise.resolve();
|
|
3436
3779
|
cursorWrites = /* @__PURE__ */ new Map();
|
|
3437
3780
|
draining = /* @__PURE__ */ new Map();
|
|
3781
|
+
deletedDrives = /* @__PURE__ */ new Map();
|
|
3782
|
+
erasing = /* @__PURE__ */ new Map();
|
|
3783
|
+
driveHeaders = /* @__PURE__ */ new Map();
|
|
3784
|
+
headerReads = /* @__PURE__ */ new Map();
|
|
3785
|
+
headerSlots = 8;
|
|
3786
|
+
headerWaiters = [];
|
|
3787
|
+
stopped = false;
|
|
3788
|
+
owedFailures = /* @__PURE__ */ new Map();
|
|
3438
3789
|
logger;
|
|
3439
3790
|
driveContainerTypes;
|
|
3440
3791
|
legacyProcessorIds;
|
|
@@ -3442,7 +3793,8 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3442
3793
|
super(db, operationIndex, writeCache, consistencyTracker, {
|
|
3443
3794
|
readModelId: "processor-manager",
|
|
3444
3795
|
rebuildStateOnInit: true,
|
|
3445
|
-
indexing: unchunkedReadModelIndexingConfig
|
|
3796
|
+
indexing: unchunkedReadModelIndexingConfig,
|
|
3797
|
+
purgeFence: "none"
|
|
3446
3798
|
});
|
|
3447
3799
|
this.logger = logger;
|
|
3448
3800
|
this.driveContainerTypes = driveContainerTypes;
|
|
@@ -3452,29 +3804,48 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3452
3804
|
await this.loadAllCursors();
|
|
3453
3805
|
await super.init();
|
|
3454
3806
|
await this.discoverExistingDrives();
|
|
3807
|
+
await this.discoverDeletedDrives();
|
|
3808
|
+
for (const [identifier, factory] of this.factoryRegistry) this.eraseOwed(identifier, factory);
|
|
3455
3809
|
}
|
|
3456
3810
|
async commitOperations(items) {
|
|
3457
3811
|
const { runs, reserved } = this.detectNewDrives(items);
|
|
3458
|
-
const
|
|
3459
|
-
const deliveries = this.enqueueRouted(items, reserved);
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
...deliveries
|
|
3464
|
-
]);
|
|
3812
|
+
const deletions = this.findDriveDeletions(items);
|
|
3813
|
+
const deliveries = this.enqueueRouted(items, reserved, deletions);
|
|
3814
|
+
this.detectDeletedDrives(items);
|
|
3815
|
+
this.forgetPurgedHeaders(items);
|
|
3816
|
+
await Promise.all([...runs.map((run) => run()), ...deliveries]);
|
|
3465
3817
|
}
|
|
3466
3818
|
async registerFactory(identifier, factory) {
|
|
3819
|
+
if (this.stopped) return;
|
|
3467
3820
|
const removals = this.removeFactory(identifier);
|
|
3468
3821
|
this.factoryRegistry.set(identifier, factory);
|
|
3469
3822
|
const previous = this.draining.get(identifier);
|
|
3470
3823
|
const creationOrdinal = this.highWater() + 1;
|
|
3471
3824
|
const runs = [];
|
|
3472
|
-
for (const [driveId, documentType] of this.knownDrives) runs.push(this.reserveSlot(identifier, factory, driveId,
|
|
3825
|
+
for (const [driveId, documentType] of this.knownDrives) runs.push(this.reserveSlot(identifier, factory, driveId, this.headerOf(driveId, documentType), creationOrdinal, void 0, false, previous).run);
|
|
3826
|
+
this.eraseOwed(identifier, factory, previous);
|
|
3473
3827
|
await Promise.all([...removals, ...runs.map((run) => run())]);
|
|
3474
3828
|
}
|
|
3475
3829
|
async unregisterFactory(identifier) {
|
|
3476
3830
|
await Promise.all(this.removeFactory(identifier));
|
|
3477
3831
|
}
|
|
3832
|
+
/** Closes every processor queue; the reactor's kill() calls it. */
|
|
3833
|
+
async shutdown() {
|
|
3834
|
+
if (this.stopped) return;
|
|
3835
|
+
this.stopped = true;
|
|
3836
|
+
this.pendingSlots.clear();
|
|
3837
|
+
this.owedFailures.clear();
|
|
3838
|
+
const closing = [...this.allBound()].map(({ queue }) => queue.close());
|
|
3839
|
+
this.processorsByDrive.clear();
|
|
3840
|
+
let timer;
|
|
3841
|
+
const grace = new Promise((resolve) => {
|
|
3842
|
+
timer = setTimeout(() => resolve(false), SHUTDOWN_GRACE_MS);
|
|
3843
|
+
timer.unref?.();
|
|
3844
|
+
});
|
|
3845
|
+
const closed = await Promise.race([Promise.all(closing).then(() => true), grace]);
|
|
3846
|
+
clearTimeout(timer);
|
|
3847
|
+
if (!closed) this.logger.warn("Processors still busy @Ms ms into shutdown; not waiting for them", SHUTDOWN_GRACE_MS);
|
|
3848
|
+
}
|
|
3478
3849
|
get(processorId) {
|
|
3479
3850
|
for (const tracked of this.allTrackedProcessors()) if (tracked.processorId === processorId) return tracked;
|
|
3480
3851
|
}
|
|
@@ -3492,6 +3863,7 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3492
3863
|
this.knownDrives.set(driveId, op.context.documentType);
|
|
3493
3864
|
const driveHeader = extractDriveHeader(op);
|
|
3494
3865
|
if (!driveHeader) continue;
|
|
3866
|
+
this.driveHeaders.set(driveId, driveHeader);
|
|
3495
3867
|
for (const [identifier, factory] of this.factoryRegistry) {
|
|
3496
3868
|
const { slot, run } = this.reserveSlot(identifier, factory, driveId, driveHeader, op.context.ordinal, items, true);
|
|
3497
3869
|
reserved.add(slot);
|
|
@@ -3503,23 +3875,85 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3503
3875
|
reserved
|
|
3504
3876
|
};
|
|
3505
3877
|
}
|
|
3506
|
-
/** Synchronous: drops a deleted drive's processors
|
|
3878
|
+
/** Synchronous: drops a deleted drive's processors; their rows stay owed. */
|
|
3507
3879
|
detectDeletedDrives(items) {
|
|
3508
|
-
const pending = [];
|
|
3509
3880
|
for (const op of items) {
|
|
3510
3881
|
if (!isDriveDeletion(op)) continue;
|
|
3511
3882
|
const driveId = extractDeletedDocumentId(op);
|
|
3512
3883
|
if (!driveId || !this.knownDrives.has(driveId)) continue;
|
|
3513
3884
|
this.knownDrives.delete(driveId);
|
|
3885
|
+
this.deletedDrives.set(driveId, op);
|
|
3514
3886
|
for (const slot of this.pendingSlots) if (slot.driveId === driveId) this.pendingSlots.delete(slot);
|
|
3515
3887
|
for (const { queue } of this.processorsByDrive.get(driveId) ?? []) queue.close();
|
|
3516
3888
|
this.processorsByDrive.delete(driveId);
|
|
3517
|
-
|
|
3889
|
+
this.pruneDeletedDrive(driveId);
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
/** Synchronous: a purged drive's header is erased data. */
|
|
3893
|
+
forgetPurgedHeaders(items) {
|
|
3894
|
+
for (const op of items) if (isPurgeMarker$1(op.operation)) {
|
|
3895
|
+
this.driveHeaders.delete(op.context.documentId);
|
|
3896
|
+
const read = this.headerReads.get(op.context.documentId);
|
|
3897
|
+
if (read) read.purged = true;
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
/** A drive's creation header; minimal once the drive's stream is purged. */
|
|
3901
|
+
headerOf(driveId, documentType) {
|
|
3902
|
+
const cached = this.driveHeaders.get(driveId);
|
|
3903
|
+
if (cached) return Promise.resolve(cached);
|
|
3904
|
+
const inFlight = this.headerReads.get(driveId)?.header;
|
|
3905
|
+
if (inFlight) return inFlight;
|
|
3906
|
+
const read = { purged: false };
|
|
3907
|
+
this.headerReads.set(driveId, read);
|
|
3908
|
+
read.header = this.readHeader(driveId, documentType, read);
|
|
3909
|
+
return read.header;
|
|
3910
|
+
}
|
|
3911
|
+
async readHeader(driveId, documentType, read) {
|
|
3912
|
+
let first;
|
|
3913
|
+
await this.takeHeaderSlot();
|
|
3914
|
+
try {
|
|
3915
|
+
[first] = await this.operationIndex.getStreamAfter({
|
|
3916
|
+
documentId: driveId,
|
|
3917
|
+
scope: "document",
|
|
3918
|
+
branch: "main"
|
|
3919
|
+
}, 0, void 0, 1);
|
|
3920
|
+
} catch (error) {
|
|
3921
|
+
this.logger.error("Failed reading drive '@DriveId' header: @Error", driveId, error);
|
|
3922
|
+
} finally {
|
|
3923
|
+
this.releaseHeaderSlot();
|
|
3924
|
+
if (this.headerReads.get(driveId) === read) this.headerReads.delete(driveId);
|
|
3518
3925
|
}
|
|
3519
|
-
|
|
3926
|
+
const header = first && extractCreationHeader(first);
|
|
3927
|
+
if (!header || read.purged) return createMinimalDriveHeader(driveId, documentType);
|
|
3928
|
+
if (this.knownDrives.has(driveId) || this.deletedDrives.has(driveId)) this.driveHeaders.set(driveId, header);
|
|
3929
|
+
return header;
|
|
3930
|
+
}
|
|
3931
|
+
async takeHeaderSlot() {
|
|
3932
|
+
if (this.headerSlots > 0) {
|
|
3933
|
+
this.headerSlots--;
|
|
3934
|
+
return;
|
|
3935
|
+
}
|
|
3936
|
+
await new Promise((resolve) => this.headerWaiters.push(resolve));
|
|
3937
|
+
}
|
|
3938
|
+
releaseHeaderSlot() {
|
|
3939
|
+
const next = this.headerWaiters.shift();
|
|
3940
|
+
if (next) next();
|
|
3941
|
+
else this.headerSlots++;
|
|
3942
|
+
}
|
|
3943
|
+
/** Synchronous: the known drives this batch deletes. */
|
|
3944
|
+
findDriveDeletions(items) {
|
|
3945
|
+
const deletions = /* @__PURE__ */ new Map();
|
|
3946
|
+
for (const op of items) {
|
|
3947
|
+
if (!isDriveDeletion(op)) continue;
|
|
3948
|
+
const driveId = extractDeletedDocumentId(op);
|
|
3949
|
+
if (!driveId || !this.knownDrives.has(driveId)) continue;
|
|
3950
|
+
const first = deletions.get(driveId);
|
|
3951
|
+
if (!first || op.context.ordinal < first.context.ordinal) deletions.set(driveId, op);
|
|
3952
|
+
}
|
|
3953
|
+
return deletions;
|
|
3520
3954
|
}
|
|
3521
3955
|
/** Synchronous: puts each processor's share of the batch on its queue. */
|
|
3522
|
-
enqueueRouted(items, reserved) {
|
|
3956
|
+
enqueueRouted(items, reserved, deletions = /* @__PURE__ */ new Map()) {
|
|
3523
3957
|
if (items.length === 0) return [];
|
|
3524
3958
|
let lowest = items[0].context.ordinal;
|
|
3525
3959
|
let highest = 0;
|
|
@@ -3532,14 +3966,179 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3532
3966
|
if (reserved.has(slot)) continue;
|
|
3533
3967
|
slot.lowestRoutedOrdinal = Math.min(slot.lowestRoutedOrdinal ?? lowest, lowest);
|
|
3534
3968
|
}
|
|
3969
|
+
let check;
|
|
3970
|
+
const checkOf = () => check ??= this.checkPurged(items);
|
|
3535
3971
|
const deliveries = [];
|
|
3536
3972
|
for (const { tracked, queue } of this.allBound()) {
|
|
3973
|
+
const deletion = deletions.get(tracked.driveId);
|
|
3974
|
+
if (deletion) {
|
|
3975
|
+
deliveries.push(this.deliverDeletion(tracked, queue, items, deletion, checkOf()));
|
|
3976
|
+
continue;
|
|
3977
|
+
}
|
|
3537
3978
|
const matching = items.filter((op) => matchesFilter(op, tracked.record.filter));
|
|
3538
|
-
if (matching.length > 0) deliveries.push(queue.live(matching));
|
|
3979
|
+
if (matching.length > 0) deliveries.push(queue.live(matching, checkOf()));
|
|
3539
3980
|
else queue.advance(highest);
|
|
3540
3981
|
}
|
|
3541
3982
|
return deliveries;
|
|
3542
3983
|
}
|
|
3984
|
+
deliverDeletion(tracked, queue, items, deletion, check) {
|
|
3985
|
+
const through = deletion.context.ordinal;
|
|
3986
|
+
const share = items.filter((op) => op.context.ordinal < through && matchesFilter(op, tracked.record.filter));
|
|
3987
|
+
const behindReplay = queue.replaying;
|
|
3988
|
+
const settled = queue.erase(share, deletion, check).then((delivered) => this.settleErased(tracked, delivered, deletion));
|
|
3989
|
+
this.holdErasure(tracked.factoryId, tracked.driveId, settled);
|
|
3990
|
+
return behindReplay ? Promise.resolve() : settled;
|
|
3991
|
+
}
|
|
3992
|
+
/** A delivered deletion deletes the row; a failed one keeps it, errored. */
|
|
3993
|
+
settleErased(tracked, delivered, deletion) {
|
|
3994
|
+
if (!delivered) {
|
|
3995
|
+
this.deletedDrives.set(tracked.driveId, deletion);
|
|
3996
|
+
this.trackOwedFailure(tracked, tracked.record, tracked);
|
|
3997
|
+
return this.writeCursor(tracked, tracked);
|
|
3998
|
+
}
|
|
3999
|
+
const deleted = this.deleteCursors((row) => row.processorId === tracked.processorId);
|
|
4000
|
+
this.pruneDeletedDrive(tracked.driveId);
|
|
4001
|
+
return Promise.all(deleted).then(() => void 0);
|
|
4002
|
+
}
|
|
4003
|
+
/** Keeps a failed deletion in getAll(); retry() delivers it again. */
|
|
4004
|
+
trackOwedFailure(owner, record, state) {
|
|
4005
|
+
const { processorId, factoryId, driveId, processorIndex } = owner;
|
|
4006
|
+
this.owedFailures.set(processorId, {
|
|
4007
|
+
processorId,
|
|
4008
|
+
factoryId,
|
|
4009
|
+
driveId,
|
|
4010
|
+
processorIndex,
|
|
4011
|
+
record,
|
|
4012
|
+
lastOrdinal: state.lastOrdinal,
|
|
4013
|
+
status: "errored",
|
|
4014
|
+
lastError: state.lastError,
|
|
4015
|
+
lastErrorTimestamp: state.lastErrorTimestamp,
|
|
4016
|
+
retry: () => this.retryOwed(factoryId, driveId)
|
|
4017
|
+
});
|
|
4018
|
+
}
|
|
4019
|
+
async retryOwed(factoryId, driveId) {
|
|
4020
|
+
const factory = this.factoryRegistry.get(factoryId);
|
|
4021
|
+
const deletion = this.deletedDrives.get(driveId);
|
|
4022
|
+
if (!factory || !deletion || this.stopped) return;
|
|
4023
|
+
await this.eraseDrive(factoryId, factory, driveId, deletion);
|
|
4024
|
+
}
|
|
4025
|
+
/** Forgets a deleted drive once no cursor row owes its deletion. */
|
|
4026
|
+
pruneDeletedDrive(driveId) {
|
|
4027
|
+
for (const row of this.cursorCache.values()) if (row.driveId === driveId) return;
|
|
4028
|
+
this.deletedDrives.delete(driveId);
|
|
4029
|
+
if (!this.knownDrives.has(driveId)) this.driveHeaders.delete(driveId);
|
|
4030
|
+
}
|
|
4031
|
+
/** Delivers each deleted drive's deletion its rows of `factoryId` still owe. */
|
|
4032
|
+
eraseOwed(factoryId, factory, previous) {
|
|
4033
|
+
if (![...this.cursorCache.values()].some((row) => row.factoryId === factoryId && this.deletedDrives.has(row.driveId)) || this.stopped) return;
|
|
4034
|
+
(async () => {
|
|
4035
|
+
await previous;
|
|
4036
|
+
if (this.factoryRegistry.get(factoryId) !== factory) return;
|
|
4037
|
+
const owed = /* @__PURE__ */ new Set();
|
|
4038
|
+
for (const row of this.cursorCache.values()) if (row.factoryId === factoryId && this.deletedDrives.has(row.driveId)) owed.add(row.driveId);
|
|
4039
|
+
await Promise.all([...owed].map((driveId) => this.eraseOwedDrive(factoryId, factory, driveId)));
|
|
4040
|
+
})().catch((error) => {
|
|
4041
|
+
this.logger.error("Owed deletions of '@FactoryId' failed: @Error", factoryId, error);
|
|
4042
|
+
});
|
|
4043
|
+
}
|
|
4044
|
+
/** After any delivery of the pair's deletion in flight, if still owed. */
|
|
4045
|
+
async eraseOwedDrive(factoryId, factory, driveId) {
|
|
4046
|
+
const key = `${factoryId}:${driveId}`;
|
|
4047
|
+
for (let held = this.erasing.get(key); held; held = this.erasing.get(key)) await held;
|
|
4048
|
+
if (this.stopped || this.factoryRegistry.get(factoryId) !== factory) return;
|
|
4049
|
+
const owes = [...this.cursorCache.values()].some((row) => row.factoryId === factoryId && row.driveId === driveId);
|
|
4050
|
+
const deletion = this.deletedDrives.get(driveId);
|
|
4051
|
+
if (!owes || !deletion) return;
|
|
4052
|
+
await this.eraseDrive(factoryId, factory, driveId, deletion);
|
|
4053
|
+
}
|
|
4054
|
+
/** Marks the pair's deletion as being delivered until `erasure` settles. */
|
|
4055
|
+
holdErasure(factoryId, driveId, erasure) {
|
|
4056
|
+
const key = `${factoryId}:${driveId}`;
|
|
4057
|
+
const held = Promise.allSettled([this.erasing.get(key), erasure]).then(() => void 0);
|
|
4058
|
+
this.erasing.set(key, held);
|
|
4059
|
+
held.then(() => {
|
|
4060
|
+
if (this.erasing.get(key) === held) this.erasing.delete(key);
|
|
4061
|
+
});
|
|
4062
|
+
}
|
|
4063
|
+
/** Runs `erase` unless the pair's deletion is already being delivered. */
|
|
4064
|
+
async claimErasure(factoryId, driveId, erase) {
|
|
4065
|
+
if (this.erasing.has(`${factoryId}:${driveId}`)) return false;
|
|
4066
|
+
let release;
|
|
4067
|
+
this.holdErasure(factoryId, driveId, new Promise((resolve) => {
|
|
4068
|
+
release = resolve;
|
|
4069
|
+
}));
|
|
4070
|
+
try {
|
|
4071
|
+
await erase();
|
|
4072
|
+
} finally {
|
|
4073
|
+
release();
|
|
4074
|
+
}
|
|
4075
|
+
return true;
|
|
4076
|
+
}
|
|
4077
|
+
async eraseDrive(factoryId, factory, driveId, deletion) {
|
|
4078
|
+
await this.claimErasure(factoryId, driveId, async () => {
|
|
4079
|
+
const header = await this.headerOf(driveId, deletion.context.documentType);
|
|
4080
|
+
const slot = {
|
|
4081
|
+
factoryId,
|
|
4082
|
+
driveId
|
|
4083
|
+
};
|
|
4084
|
+
const records = await this.runFactory(slot, factory, header);
|
|
4085
|
+
if (!records) return;
|
|
4086
|
+
await this.eraseRecords(factoryId, driveId, deletion, records);
|
|
4087
|
+
});
|
|
4088
|
+
}
|
|
4089
|
+
/** The deletion to each record a row owes it to, then every disconnect. */
|
|
4090
|
+
async eraseRecords(factoryId, driveId, deletion, records) {
|
|
4091
|
+
const ids = resolveProcessorSlots(records, this.legacyProcessorIds);
|
|
4092
|
+
const delivered = /* @__PURE__ */ new Set();
|
|
4093
|
+
const failed = /* @__PURE__ */ new Map();
|
|
4094
|
+
const recordOf = /* @__PURE__ */ new Map();
|
|
4095
|
+
await Promise.all(records.map(async (record, i) => {
|
|
4096
|
+
const processorId = `${factoryId}:${driveId}:${ids[i]}`;
|
|
4097
|
+
if (this.cursorCache.has(processorId)) {
|
|
4098
|
+
recordOf.set(processorId, record);
|
|
4099
|
+
try {
|
|
4100
|
+
await record.processor.onOperations([deletion]);
|
|
4101
|
+
delivered.add(processorId);
|
|
4102
|
+
} catch (error) {
|
|
4103
|
+
failed.set(processorId, error);
|
|
4104
|
+
this.logger.error("Processor '@ProcessorId' failed on its drive's deletion: @Error", processorId, error);
|
|
4105
|
+
}
|
|
4106
|
+
}
|
|
4107
|
+
await this.discard({
|
|
4108
|
+
factoryId,
|
|
4109
|
+
driveId
|
|
4110
|
+
}, record.processor.onDisconnect.bind(record.processor));
|
|
4111
|
+
}));
|
|
4112
|
+
const writes = [];
|
|
4113
|
+
for (const row of this.cursorCache.values()) {
|
|
4114
|
+
if (row.factoryId !== factoryId || row.driveId !== driveId) continue;
|
|
4115
|
+
if (delivered.has(row.processorId)) continue;
|
|
4116
|
+
if (!failed.has(row.processorId)) {
|
|
4117
|
+
this.logger.warn("Factory '@FactoryId' made no processor '@ProcessorId' for deleted drive '@DriveId'; its deletion stays owed", factoryId, row.processorId, driveId);
|
|
4118
|
+
continue;
|
|
4119
|
+
}
|
|
4120
|
+
const error = failed.get(row.processorId);
|
|
4121
|
+
const state = {
|
|
4122
|
+
lastOrdinal: row.lastOrdinal,
|
|
4123
|
+
status: "errored",
|
|
4124
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
4125
|
+
lastErrorTimestamp: /* @__PURE__ */ new Date()
|
|
4126
|
+
};
|
|
4127
|
+
this.trackOwedFailure(row, recordOf.get(row.processorId), state);
|
|
4128
|
+
writes.push(this.writeCursor(row, state));
|
|
4129
|
+
}
|
|
4130
|
+
for (const processorId of delivered) this.owedFailures.delete(processorId);
|
|
4131
|
+
writes.push(...this.deleteCursors((row) => delivered.has(row.processorId)));
|
|
4132
|
+
this.pruneDeletedDrive(driveId);
|
|
4133
|
+
await Promise.all(writes);
|
|
4134
|
+
}
|
|
4135
|
+
/** Started at routing; takes no purge lock, since processors hold none. */
|
|
4136
|
+
checkPurged(items) {
|
|
4137
|
+
const ids = purgeCandidates(items);
|
|
4138
|
+
const purged = ids.length === 0 ? Promise.resolve(/* @__PURE__ */ new Set()) : findPurged(this.db, ids);
|
|
4139
|
+
purged.catch(() => void 0);
|
|
4140
|
+
return { purged };
|
|
4141
|
+
}
|
|
3543
4142
|
/** Synchronous: binds records, or discards them if the slot was cancelled. */
|
|
3544
4143
|
bind(slot, records, creationOrdinal, creationItems) {
|
|
3545
4144
|
const released = this.pendingSlots.delete(slot);
|
|
@@ -3550,9 +4149,8 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3550
4149
|
delivered: settled
|
|
3551
4150
|
};
|
|
3552
4151
|
}
|
|
3553
|
-
if (!released) {
|
|
3554
|
-
const
|
|
3555
|
-
const settled = Promise.all(discarded).then(() => void 0);
|
|
4152
|
+
if (!released || this.stopped) {
|
|
4153
|
+
const settled = this.discardRecords(slot, records);
|
|
3556
4154
|
return {
|
|
3557
4155
|
persisted: settled,
|
|
3558
4156
|
delivered: settled
|
|
@@ -3571,16 +4169,24 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3571
4169
|
const missed = tracked.lastOrdinal < slot.reservedAt || lowestRoutedOrdinal !== void 0;
|
|
3572
4170
|
if (tracked.status === "active" && missed) deliveries.push(queue.backfill());
|
|
3573
4171
|
const matching = creationItems?.filter((op) => matchesFilter(op, tracked.record.filter));
|
|
3574
|
-
if (matching && matching.length > 0) deliveries.push(queue.live(matching));
|
|
4172
|
+
if (matching && matching.length > 0) deliveries.push(queue.live(matching, this.checkPurged(matching)));
|
|
3575
4173
|
}
|
|
3576
4174
|
return {
|
|
3577
4175
|
persisted: Promise.all(writes).then(() => void 0),
|
|
3578
4176
|
delivered: Promise.all(deliveries).then(() => void 0)
|
|
3579
4177
|
};
|
|
3580
4178
|
}
|
|
3581
|
-
/**
|
|
4179
|
+
/** A cancelled slot's records; on a deleted drive they pay what rows owe. */
|
|
4180
|
+
async discardRecords(slot, records) {
|
|
4181
|
+
const { factoryId, driveId } = slot;
|
|
4182
|
+
const deletion = this.stopped ? void 0 : this.deletedDrives.get(driveId);
|
|
4183
|
+
if (deletion !== void 0 && await this.claimErasure(factoryId, driveId, () => this.eraseRecords(factoryId, driveId, deletion, records))) return;
|
|
4184
|
+
await Promise.all(records.map((record) => this.discard(slot, record.processor.onDisconnect.bind(record.processor))));
|
|
4185
|
+
}
|
|
4186
|
+
/** Synchronous: removes a factory's slots and processors; releases its rows. */
|
|
3582
4187
|
removeFactory(identifier) {
|
|
3583
4188
|
if (!this.factoryRegistry.delete(identifier)) return [];
|
|
4189
|
+
for (const [processorId, failure] of this.owedFailures) if (failure.factoryId === identifier) this.owedFailures.delete(processorId);
|
|
3584
4190
|
const closing = [];
|
|
3585
4191
|
for (const slot of this.pendingSlots) {
|
|
3586
4192
|
if (slot.factoryId !== identifier) continue;
|
|
@@ -3601,7 +4207,25 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3601
4207
|
if (this.draining.get(identifier) === drained) this.draining.delete(identifier);
|
|
3602
4208
|
});
|
|
3603
4209
|
}
|
|
3604
|
-
return this.
|
|
4210
|
+
return this.releaseCursors(identifier);
|
|
4211
|
+
}
|
|
4212
|
+
releaseCursors(factoryId) {
|
|
4213
|
+
const writes = [];
|
|
4214
|
+
for (const row of this.cursorCache.values()) {
|
|
4215
|
+
if (row.factoryId !== factoryId) continue;
|
|
4216
|
+
if (row.status === "released") continue;
|
|
4217
|
+
if (this.deletedDrives.has(row.driveId)) continue;
|
|
4218
|
+
const released = {
|
|
4219
|
+
...row,
|
|
4220
|
+
status: RELEASED_CURSOR_STATUS
|
|
4221
|
+
};
|
|
4222
|
+
this.cursorCache.set(row.processorId, released);
|
|
4223
|
+
writes.push(this.lane(row.processorId, () => this.db.updateTable("ProcessorCursor").set({
|
|
4224
|
+
status: RELEASED_CURSOR_STATUS,
|
|
4225
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
4226
|
+
}).where("processorId", "=", row.processorId).execute()));
|
|
4227
|
+
}
|
|
4228
|
+
return writes;
|
|
3605
4229
|
}
|
|
3606
4230
|
reserveSlot(factoryId, factory, driveId, driveHeader, creationOrdinal, creationItems, awaitDelivery, previous) {
|
|
3607
4231
|
const slot = {
|
|
@@ -3615,7 +4239,8 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3615
4239
|
const run = async () => {
|
|
3616
4240
|
const bound = (async () => {
|
|
3617
4241
|
await previous;
|
|
3618
|
-
const
|
|
4242
|
+
const header = await driveHeader;
|
|
4243
|
+
const records = await this.runFactory(slot, factory, header);
|
|
3619
4244
|
return this.bind(slot, records, creationOrdinal, creationItems);
|
|
3620
4245
|
})();
|
|
3621
4246
|
slot.settled = bound.then(({ persisted }) => persisted);
|
|
@@ -3639,13 +4264,15 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3639
4264
|
const cached = this.cursorCache.get(processorId);
|
|
3640
4265
|
let floor = 0;
|
|
3641
4266
|
let cursor;
|
|
3642
|
-
if (cached
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
4267
|
+
if (cached && cached.status !== "released") {
|
|
4268
|
+
const errored = cached.status !== "active";
|
|
4269
|
+
cursor = {
|
|
4270
|
+
lastOrdinal: cached.lastOrdinal,
|
|
4271
|
+
status: cached.status,
|
|
4272
|
+
lastError: errored ? cached.lastError ?? void 0 : void 0,
|
|
4273
|
+
lastErrorTimestamp: errored ? cached.lastErrorTimestamp ?? void 0 : void 0
|
|
4274
|
+
};
|
|
4275
|
+
} else {
|
|
3649
4276
|
if (record.startFrom === "current") floor = creationOrdinal - 1;
|
|
3650
4277
|
cursor = {
|
|
3651
4278
|
lastOrdinal: floor,
|
|
@@ -3674,6 +4301,7 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3674
4301
|
routedThrough: () => this.highWater(),
|
|
3675
4302
|
confirmedThrough: () => this.lastOrdinal,
|
|
3676
4303
|
persist: (state) => this.writeCursor(tracked, state),
|
|
4304
|
+
purged: (ids) => findPurged(this.db, ids),
|
|
3677
4305
|
logger: this.logger
|
|
3678
4306
|
});
|
|
3679
4307
|
return {
|
|
@@ -3706,6 +4334,7 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3706
4334
|
}
|
|
3707
4335
|
*allTrackedProcessors() {
|
|
3708
4336
|
for (const { tracked } of this.allBound()) yield tracked;
|
|
4337
|
+
yield* this.owedFailures.values();
|
|
3709
4338
|
}
|
|
3710
4339
|
isDriveCreation(op) {
|
|
3711
4340
|
return op.operation.action.type === "CREATE_DOCUMENT" && this.driveContainerTypes.has(op.context.documentType);
|
|
@@ -3714,6 +4343,38 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
3714
4343
|
const drives = await this.db.selectFrom("DocumentSnapshot").select(["documentId", "documentType"]).where("documentType", "in", [...this.driveContainerTypes]).where("isDeleted", "=", false).execute();
|
|
3715
4344
|
for (const drive of drives) this.knownDrives.set(drive.documentId, drive.documentType);
|
|
3716
4345
|
}
|
|
4346
|
+
/** Cursor rows of a drive that is gone: find the deletion they still owe. */
|
|
4347
|
+
async discoverDeletedDrives() {
|
|
4348
|
+
const driveIds = /* @__PURE__ */ new Set();
|
|
4349
|
+
for (const row of this.cursorCache.values()) if (!this.knownDrives.has(row.driveId)) driveIds.add(row.driveId);
|
|
4350
|
+
for (const driveId of driveIds) try {
|
|
4351
|
+
await this.discoverDeletedDrive(driveId);
|
|
4352
|
+
} catch (error) {
|
|
4353
|
+
this.logger.error("Failed reading deleted drive '@DriveId': @Error", driveId, error);
|
|
4354
|
+
}
|
|
4355
|
+
}
|
|
4356
|
+
async discoverDeletedDrive(driveId) {
|
|
4357
|
+
const key = {
|
|
4358
|
+
documentId: driveId,
|
|
4359
|
+
scope: "document",
|
|
4360
|
+
branch: "main"
|
|
4361
|
+
};
|
|
4362
|
+
let header;
|
|
4363
|
+
let deletion;
|
|
4364
|
+
let after = 0;
|
|
4365
|
+
for (;;) {
|
|
4366
|
+
const page = await this.operationIndex.getStreamAfter(key, after, void 0, DRIVE_STREAM_PAGE);
|
|
4367
|
+
for (const op of page) {
|
|
4368
|
+
header ??= extractCreationHeader(op);
|
|
4369
|
+
if (!deletion && isDriveDeletion(op) && extractDeletedDocumentId(op) === driveId) deletion = op;
|
|
4370
|
+
}
|
|
4371
|
+
if (page.length < DRIVE_STREAM_PAGE) break;
|
|
4372
|
+
after = page.at(-1).context.ordinal;
|
|
4373
|
+
}
|
|
4374
|
+
if (!deletion) return;
|
|
4375
|
+
this.deletedDrives.set(driveId, deletion);
|
|
4376
|
+
if (header) this.driveHeaders.set(driveId, header);
|
|
4377
|
+
}
|
|
3717
4378
|
async loadAllCursors() {
|
|
3718
4379
|
const rows = await this.db.selectFrom("ProcessorCursor").selectAll().execute();
|
|
3719
4380
|
for (const row of rows) {
|
|
@@ -4576,11 +5237,12 @@ var ReactorSubscriptionManager = class {
|
|
|
4576
5237
|
/**
|
|
4577
5238
|
* Notify subscribers about deleted documents
|
|
4578
5239
|
*/
|
|
4579
|
-
notifyDocumentsDeleted(documentIds, documentTypes, parentIds) {
|
|
5240
|
+
notifyDocumentsDeleted(documentIds, documentTypes, parentIds, info) {
|
|
4580
5241
|
for (const subscription of this.deletedSubscriptions.values()) {
|
|
4581
5242
|
const filteredIds = this.filterDocumentIds(documentIds, subscription.search, documentTypes, parentIds);
|
|
4582
5243
|
if (filteredIds.length > 0) try {
|
|
4583
|
-
subscription.callback(filteredIds);
|
|
5244
|
+
if (info) subscription.callback(filteredIds, info);
|
|
5245
|
+
else subscription.callback(filteredIds);
|
|
4584
5246
|
} catch (error) {
|
|
4585
5247
|
this.errorHandler.handleError(error, {
|
|
4586
5248
|
eventType: "deleted",
|
|
@@ -4695,14 +5357,19 @@ var SubscriptionNotificationReadModel = class {
|
|
|
4695
5357
|
if (operations.length === 0) return;
|
|
4696
5358
|
const created = [];
|
|
4697
5359
|
const deleted = [];
|
|
5360
|
+
const purgeDeleted = [];
|
|
4698
5361
|
const updatedIds = /* @__PURE__ */ new Set();
|
|
5362
|
+
const purgedIds = /* @__PURE__ */ new Set();
|
|
4699
5363
|
const documentTypes = /* @__PURE__ */ new Map();
|
|
4700
5364
|
const parentIds = /* @__PURE__ */ new Map();
|
|
4701
5365
|
for (const item of operations) {
|
|
4702
5366
|
const { operation, context } = item;
|
|
4703
5367
|
const actionType = operation.action.type;
|
|
4704
5368
|
documentTypes.set(context.documentId, context.documentType);
|
|
4705
|
-
if (
|
|
5369
|
+
if (isPurgeMarker$1(operation)) {
|
|
5370
|
+
purgedIds.add(context.documentId);
|
|
5371
|
+
if (context.appliedDeletion) purgeDeleted.push(context.documentId);
|
|
5372
|
+
} else if (actionType === "CREATE_DOCUMENT") created.push(context.documentId);
|
|
4706
5373
|
else if (actionType === "DELETE_DOCUMENT") {
|
|
4707
5374
|
const deletedId = operation.action.input.documentId ?? context.documentId;
|
|
4708
5375
|
deleted.push(deletedId);
|
|
@@ -4716,9 +5383,21 @@ var SubscriptionNotificationReadModel = class {
|
|
|
4716
5383
|
}
|
|
4717
5384
|
if (created.length > 0) this.subscriptionManager.notifyDocumentsCreated(created, documentTypes, parentIds);
|
|
4718
5385
|
if (deleted.length > 0) this.subscriptionManager.notifyDocumentsDeleted(deleted, documentTypes, parentIds);
|
|
5386
|
+
if (purgeDeleted.length > 0) this.subscriptionManager.notifyDocumentsDeleted(purgeDeleted, documentTypes, parentIds, { purged: true });
|
|
5387
|
+
for (const id of purgedIds) updatedIds.delete(id);
|
|
4719
5388
|
if (updatedIds.size > 0 && this.documentView) {
|
|
4720
|
-
const
|
|
4721
|
-
this.
|
|
5389
|
+
const documentView = this.documentView;
|
|
5390
|
+
const present = (await Promise.all(Array.from(updatedIds).map((id) => this.readUpdated(documentView, id)))).filter((document) => document !== void 0);
|
|
5391
|
+
if (present.length > 0) this.subscriptionManager.notifyDocumentsUpdated(present);
|
|
5392
|
+
}
|
|
5393
|
+
}
|
|
5394
|
+
/** Undefined for a document gone since its update; others still notify. */
|
|
5395
|
+
async readUpdated(documentView, id) {
|
|
5396
|
+
try {
|
|
5397
|
+
return await documentView.get(id);
|
|
5398
|
+
} catch (error) {
|
|
5399
|
+
if (DocumentNotFoundError.isError(error)) return void 0;
|
|
5400
|
+
throw error;
|
|
4722
5401
|
}
|
|
4723
5402
|
}
|
|
4724
5403
|
};
|
|
@@ -4750,7 +5429,8 @@ var GroupReevaluationTrigger = class extends BaseReadModel {
|
|
|
4750
5429
|
rebuildStateOnInit: false,
|
|
4751
5430
|
indexing: unchunkedReadModelIndexingConfig,
|
|
4752
5431
|
startFrom: "head",
|
|
4753
|
-
replayStreamSuffix: false
|
|
5432
|
+
replayStreamSuffix: false,
|
|
5433
|
+
purgeFence: "none"
|
|
4754
5434
|
});
|
|
4755
5435
|
this.logger = logger;
|
|
4756
5436
|
this.eventBus = eventBus;
|
|
@@ -4878,7 +5558,8 @@ const SyncEventTypes = {
|
|
|
4878
5558
|
DEAD_LETTER_ADDED: 20004,
|
|
4879
5559
|
CONNECTION_STATE_CHANGED: 20005,
|
|
4880
5560
|
SYNC_HELD: 20006,
|
|
4881
|
-
SYNC_RELEASED: 20007
|
|
5561
|
+
SYNC_RELEASED: 20007,
|
|
5562
|
+
PURGE_REFUSED: 20008
|
|
4882
5563
|
};
|
|
4883
5564
|
//#endregion
|
|
4884
5565
|
//#region src/sync/mailbox.ts
|
|
@@ -4892,7 +5573,10 @@ var MailboxAggregateError = class extends Error {
|
|
|
4892
5573
|
}
|
|
4893
5574
|
};
|
|
4894
5575
|
var Mailbox = class {
|
|
5576
|
+
holdAckBelowMarkers;
|
|
4895
5577
|
itemsMap = /* @__PURE__ */ new Map();
|
|
5578
|
+
/** Unapplied items carrying a marker, so the held ack reads only these. */
|
|
5579
|
+
heldMarkers = /* @__PURE__ */ new Set();
|
|
4896
5580
|
addedCallbacks = [];
|
|
4897
5581
|
removedCallbacks = [];
|
|
4898
5582
|
paused = false;
|
|
@@ -4900,6 +5584,9 @@ var Mailbox = class {
|
|
|
4900
5584
|
removedBuffer = [];
|
|
4901
5585
|
_ack = 0;
|
|
4902
5586
|
_latestOrdinal = 0;
|
|
5587
|
+
constructor(options = {}) {
|
|
5588
|
+
this.holdAckBelowMarkers = options.holdAckBelowMarkers ?? false;
|
|
5589
|
+
}
|
|
4903
5590
|
init(ackOrdinal) {
|
|
4904
5591
|
this._ack = this._latestOrdinal = ackOrdinal;
|
|
4905
5592
|
}
|
|
@@ -4910,7 +5597,13 @@ var Mailbox = class {
|
|
|
4910
5597
|
return Array.from(this.itemsMap.values());
|
|
4911
5598
|
}
|
|
4912
5599
|
get ackOrdinal() {
|
|
4913
|
-
return this._ack;
|
|
5600
|
+
if (!this.holdAckBelowMarkers) return this._ack;
|
|
5601
|
+
let floor = Number.POSITIVE_INFINITY;
|
|
5602
|
+
for (const item of this.heldMarkers) for (const op of item.operations) {
|
|
5603
|
+
const ordinal = op.context.ordinal;
|
|
5604
|
+
if (ordinal > 0 && ordinal < floor) floor = ordinal;
|
|
5605
|
+
}
|
|
5606
|
+
return Math.min(this._ack, floor - 1);
|
|
4914
5607
|
}
|
|
4915
5608
|
get latestOrdinal() {
|
|
4916
5609
|
return this._latestOrdinal;
|
|
@@ -4920,10 +5613,20 @@ var Mailbox = class {
|
|
|
4920
5613
|
}
|
|
4921
5614
|
add(...items) {
|
|
4922
5615
|
for (const item of items) {
|
|
5616
|
+
const replaced = this.itemsMap.get(item.id);
|
|
5617
|
+
if (replaced !== void 0) this.heldMarkers.delete(replaced);
|
|
4923
5618
|
this.itemsMap.set(item.id, item);
|
|
4924
|
-
|
|
5619
|
+
let marker = false;
|
|
5620
|
+
for (const op of item.operations) {
|
|
5621
|
+
this._latestOrdinal = Math.max(this._latestOrdinal, op.context.ordinal);
|
|
5622
|
+
if (isPurgeMarker$1(op)) marker = true;
|
|
5623
|
+
}
|
|
5624
|
+
if (this.holdAckBelowMarkers && marker && item.status !== 2) this.heldMarkers.add(item);
|
|
4925
5625
|
item.on((syncOp, _, next) => {
|
|
4926
|
-
if (next === 2)
|
|
5626
|
+
if (next === 2) {
|
|
5627
|
+
this.heldMarkers.delete(syncOp);
|
|
5628
|
+
for (const op of syncOp.operations) this._ack = Math.max(this._ack, op.context.ordinal);
|
|
5629
|
+
}
|
|
4927
5630
|
});
|
|
4928
5631
|
}
|
|
4929
5632
|
if (this.paused) {
|
|
@@ -4940,7 +5643,10 @@ var Mailbox = class {
|
|
|
4940
5643
|
if (errors.length > 0) throw new MailboxAggregateError(errors);
|
|
4941
5644
|
}
|
|
4942
5645
|
remove(...items) {
|
|
4943
|
-
for (const item of items)
|
|
5646
|
+
for (const item of items) {
|
|
5647
|
+
this.itemsMap.delete(item.id);
|
|
5648
|
+
this.heldMarkers.delete(item);
|
|
5649
|
+
}
|
|
4944
5650
|
if (this.paused) {
|
|
4945
5651
|
this.removedBuffer.push(...items);
|
|
4946
5652
|
return;
|
|
@@ -5151,14 +5857,20 @@ var GraphQLRequestError = class extends Error {
|
|
|
5151
5857
|
* difference between a channel that recovers and one that is dead for the
|
|
5152
5858
|
* process lifetime.
|
|
5153
5859
|
*/
|
|
5154
|
-
const RECOVERABLE_GRAPHQL_ERROR_CODES = {
|
|
5155
|
-
/**
|
|
5156
|
-
* A stored operation cannot be represented in the schema - an action with no
|
|
5157
|
-
* id, say. The document holding it needs repairing, but the channel serves
|
|
5158
|
-
* every other document, and a peer that stopped polling would stop receiving
|
|
5159
|
-
* those too.
|
|
5160
|
-
*/
|
|
5161
|
-
malformedStoredOperation: "MALFORMED_STORED_OPERATION"
|
|
5860
|
+
const RECOVERABLE_GRAPHQL_ERROR_CODES = {
|
|
5861
|
+
/**
|
|
5862
|
+
* A stored operation cannot be represented in the schema - an action with no
|
|
5863
|
+
* id, say. The document holding it needs repairing, but the channel serves
|
|
5864
|
+
* every other document, and a peer that stopped polling would stop receiving
|
|
5865
|
+
* those too.
|
|
5866
|
+
*/
|
|
5867
|
+
malformedStoredOperation: "MALFORMED_STORED_OPERATION",
|
|
5868
|
+
/**
|
|
5869
|
+
* The poll's marker refusals could not be recorded, so nothing was served.
|
|
5870
|
+
* The client keeps them pending and reports them on its next poll.
|
|
5871
|
+
*/
|
|
5872
|
+
refusalNotRecorded: "REFUSAL_NOT_RECORDED"
|
|
5873
|
+
};
|
|
5162
5874
|
const RECOVERABLE_CODES = new Set(Object.values(RECOVERABLE_GRAPHQL_ERROR_CODES));
|
|
5163
5875
|
/**
|
|
5164
5876
|
* True when every error the response carried named a recoverable code.
|
|
@@ -5344,6 +6056,11 @@ function filterOperations(operations, filter) {
|
|
|
5344
6056
|
return true;
|
|
5345
6057
|
});
|
|
5346
6058
|
}
|
|
6059
|
+
/** Markers pass every remote filter, as they pass sinceTimestamp. */
|
|
6060
|
+
function filterForRemote(operations, filter) {
|
|
6061
|
+
const kept = new Set(filterOperations(operations, filter));
|
|
6062
|
+
return operations.filter((op) => kept.has(op) || isPurgeMarker$1(op));
|
|
6063
|
+
}
|
|
5347
6064
|
/**
|
|
5348
6065
|
* Creates an idle channel health status.
|
|
5349
6066
|
*
|
|
@@ -5621,6 +6338,11 @@ function classifyJobFailure(errorName) {
|
|
|
5621
6338
|
case "InvalidSignatureError": return "SIGNATURE_INVALID";
|
|
5622
6339
|
case "HashMismatchError": return "HASH_MISMATCH";
|
|
5623
6340
|
case "UnsupportedProtocolVersionError": return "UNSUPPORTED_PROTOCOL";
|
|
6341
|
+
case "DocumentPurgedError": return "DOCUMENT_PURGED";
|
|
6342
|
+
case "DocumentNotDeletedError":
|
|
6343
|
+
case "GroupInUseError":
|
|
6344
|
+
case "PurgeTooLargeError": return "PURGE_PRECONDITION";
|
|
6345
|
+
case "ReservedActionError": return "RESERVED_ACTION";
|
|
5624
6346
|
default: return "UNCLASSIFIED";
|
|
5625
6347
|
}
|
|
5626
6348
|
}
|
|
@@ -5632,7 +6354,10 @@ function syncOperationErrorType(error) {
|
|
|
5632
6354
|
const NON_QUARANTINING_ERROR_TYPES = /* @__PURE__ */ new Set([
|
|
5633
6355
|
"AUTH_TIMESTAMP_NOT_MONOTONIC",
|
|
5634
6356
|
"UNSUPPORTED_PROTOCOL",
|
|
5635
|
-
"PEER_PROTOCOL_UNSUPPORTED"
|
|
6357
|
+
"PEER_PROTOCOL_UNSUPPORTED",
|
|
6358
|
+
"DOCUMENT_PURGED",
|
|
6359
|
+
"PURGE_PRECONDITION",
|
|
6360
|
+
"MARKER_REFUSED"
|
|
5636
6361
|
]);
|
|
5637
6362
|
/**
|
|
5638
6363
|
* A held auth operation must not quarantine: reconciling the two policies needs
|
|
@@ -5752,6 +6477,13 @@ var IntervalPollTimer = class {
|
|
|
5752
6477
|
}
|
|
5753
6478
|
};
|
|
5754
6479
|
//#endregion
|
|
6480
|
+
//#region src/sync/purge-refusals.ts
|
|
6481
|
+
/** The most refusals a poll reports; the rest wait for the next poll. */
|
|
6482
|
+
const MAX_POLLED_REFUSALS = 100;
|
|
6483
|
+
function supportsPurgeRefusals(x) {
|
|
6484
|
+
return typeof x === "object" && x !== null && typeof x.recordPurgeRefusal === "function" && typeof x.recordPolledMarkerRefusals === "function";
|
|
6485
|
+
}
|
|
6486
|
+
//#endregion
|
|
5755
6487
|
//#region src/sync/channels/utils.ts
|
|
5756
6488
|
let syncOpCounter = 0;
|
|
5757
6489
|
/**
|
|
@@ -5879,6 +6611,9 @@ const AGREEMENT_FIELDS = [
|
|
|
5879
6611
|
"refusals",
|
|
5880
6612
|
"SyncRefusalInput"
|
|
5881
6613
|
];
|
|
6614
|
+
function refusalKey(refusal) {
|
|
6615
|
+
return `${refusal.documentId}\u0000${refusal.branch}\u0000${refusal.kind ?? ""}`;
|
|
6616
|
+
}
|
|
5882
6617
|
/** How often a channel whose remote went silent asks again whether it serves agreement. */
|
|
5883
6618
|
const AGREEMENT_PROBE_INTERVAL_MS = 3e5;
|
|
5884
6619
|
function pushMutation(withGatedUnder) {
|
|
@@ -5932,6 +6667,10 @@ var GqlRequestChannel = class {
|
|
|
5932
6667
|
manifestRefresh;
|
|
5933
6668
|
/** Polled rows this reactor could not run, reported on the next poll. */
|
|
5934
6669
|
pendingRefusals = /* @__PURE__ */ new Map();
|
|
6670
|
+
/** When each pushed, unacknowledged marker entry was last pushed. */
|
|
6671
|
+
markerPushedAt = /* @__PURE__ */ new Map();
|
|
6672
|
+
/** Documents whose marker the remote refused; its report repeats per poll. */
|
|
6673
|
+
refusedMarkers = /* @__PURE__ */ new Set();
|
|
5935
6674
|
isRecovering = false;
|
|
5936
6675
|
connectionState = "connecting";
|
|
5937
6676
|
/** Latest unrecoverable error was an auth rejection; cleared on connect. */
|
|
@@ -5955,21 +6694,30 @@ var GqlRequestChannel = class {
|
|
|
5955
6694
|
};
|
|
5956
6695
|
this.isShutdown = false;
|
|
5957
6696
|
this.failureCount = 0;
|
|
5958
|
-
this.inbox = new Mailbox();
|
|
6697
|
+
this.inbox = new Mailbox({ holdAckBelowMarkers: true });
|
|
5959
6698
|
this.bufferedOutbox = new BufferedMailbox(500, 25);
|
|
5960
6699
|
this.outbox = this.bufferedOutbox;
|
|
5961
6700
|
this.deadLetter = new Mailbox();
|
|
5962
6701
|
this.deadLetter.onAdded((syncOps) => {
|
|
5963
6702
|
for (const syncOp of syncOps) {
|
|
5964
6703
|
this.logger.warn("Dead letter added for document @DocumentId on channel @ChannelId", syncOp.documentId, this.channelId);
|
|
5965
|
-
if (syncOp.error?.source
|
|
6704
|
+
if (syncOp.error?.source !== "inbox") continue;
|
|
6705
|
+
const errorType = syncOperationErrorType(syncOp.error);
|
|
6706
|
+
const refusal = errorType === "UNSUPPORTED_PROTOCOL" ? {
|
|
5966
6707
|
documentId: syncOp.documentId,
|
|
5967
6708
|
branch: syncOp.branch
|
|
5968
|
-
}
|
|
6709
|
+
} : errorType === "MARKER_REFUSED" ? {
|
|
6710
|
+
documentId: syncOp.documentId,
|
|
6711
|
+
branch: syncOp.branch,
|
|
6712
|
+
kind: "marker"
|
|
6713
|
+
} : void 0;
|
|
6714
|
+
if (refusal) this.pendingRefusals.set(refusalKey(refusal), refusal);
|
|
5969
6715
|
}
|
|
5970
6716
|
});
|
|
5971
|
-
this.outbox.onAdded((
|
|
6717
|
+
this.outbox.onAdded((added) => {
|
|
5972
6718
|
if (this.isShutdown) return;
|
|
6719
|
+
const syncOps = added.filter((op) => this.outbox.get(op.id) === op);
|
|
6720
|
+
if (syncOps.length === 0) return;
|
|
5973
6721
|
if (this.isPushing) {
|
|
5974
6722
|
this.pendingDrain = true;
|
|
5975
6723
|
return;
|
|
@@ -5982,6 +6730,7 @@ var GqlRequestChannel = class {
|
|
|
5982
6730
|
this.attemptPush(syncOps);
|
|
5983
6731
|
});
|
|
5984
6732
|
this.outbox.onRemoved((syncOps) => {
|
|
6733
|
+
for (const syncOp of syncOps) this.markerPushedAt.delete(syncOp.id);
|
|
5985
6734
|
const ordinal = Math.min(getLatestAppliedOrdinal(syncOps), this.unappliedFloor() - 1);
|
|
5986
6735
|
if (ordinal > this.lastPersistedOutboxOrdinal) {
|
|
5987
6736
|
this.lastPersistedOutboxOrdinal = ordinal;
|
|
@@ -5995,8 +6744,8 @@ var GqlRequestChannel = class {
|
|
|
5995
6744
|
});
|
|
5996
6745
|
}
|
|
5997
6746
|
});
|
|
5998
|
-
this.inbox.onRemoved((
|
|
5999
|
-
const maxOrdinal =
|
|
6747
|
+
this.inbox.onRemoved(() => {
|
|
6748
|
+
const maxOrdinal = this.inbox.ackOrdinal;
|
|
6000
6749
|
if (maxOrdinal > this.lastPersistedInboxOrdinal) {
|
|
6001
6750
|
this.lastPersistedInboxOrdinal = maxOrdinal;
|
|
6002
6751
|
this.cursorStorage.upsert({
|
|
@@ -6060,6 +6809,9 @@ var GqlRequestChannel = class {
|
|
|
6060
6809
|
this.peerManifestCallbacks.delete(callback);
|
|
6061
6810
|
};
|
|
6062
6811
|
}
|
|
6812
|
+
forgetMarkerRefusal(documentId, branch) {
|
|
6813
|
+
this.refusedMarkers.delete(`${documentId}\u0000${branch}`);
|
|
6814
|
+
}
|
|
6063
6815
|
async hearPeer(manifest, undelivered) {
|
|
6064
6816
|
if (this.peerManifest !== void 0 && (this.peerManifest?.revision ?? null) === (manifest?.revision ?? null) || isOlderManifest(manifest, this.peerManifest)) return;
|
|
6065
6817
|
this.peerManifest = manifest;
|
|
@@ -6146,6 +6898,8 @@ var GqlRequestChannel = class {
|
|
|
6146
6898
|
}
|
|
6147
6899
|
const { envelopes, ackOrdinal, deadLetters, hasMore, manifestRevision, peerManifestRevision } = response;
|
|
6148
6900
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
6901
|
+
this.retireRefusedMarkers(deadLetters);
|
|
6902
|
+
this.repushUnackedMarkers();
|
|
6149
6903
|
if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision)) return;
|
|
6150
6904
|
const allSyncOps = [];
|
|
6151
6905
|
for (const envelope of envelopes) if (envelope.type.toLowerCase() === "operations" && envelope.operations) {
|
|
@@ -6169,7 +6923,15 @@ var GqlRequestChannel = class {
|
|
|
6169
6923
|
* Handles dead letters reported by the remote server.
|
|
6170
6924
|
* Creates local dead letter SyncOperations so the channel quiesces.
|
|
6171
6925
|
*/
|
|
6172
|
-
handleRemoteDeadLetters(
|
|
6926
|
+
handleRemoteDeadLetters(wire) {
|
|
6927
|
+
const deadLetters = wire.filter((dl) => {
|
|
6928
|
+
if (dl.errorType !== "MARKER_REFUSED") return true;
|
|
6929
|
+
const key = `${dl.documentId}\u0000${dl.branch}`;
|
|
6930
|
+
if (this.refusedMarkers.has(key)) return false;
|
|
6931
|
+
this.refusedMarkers.add(key);
|
|
6932
|
+
return true;
|
|
6933
|
+
});
|
|
6934
|
+
if (deadLetters.length === 0) return;
|
|
6173
6935
|
for (const dl of deadLetters) this.logger.error("Remote dead letter on @ChannelId: document @DocumentId failed with: @Error", this.channelId, dl.documentId, dl.error);
|
|
6174
6936
|
const syncOps = [];
|
|
6175
6937
|
for (const dl of deadLetters) {
|
|
@@ -6275,7 +7037,7 @@ var GqlRequestChannel = class {
|
|
|
6275
7037
|
let refusals = [];
|
|
6276
7038
|
for (;;) {
|
|
6277
7039
|
const revision = this.peerServesAgreement ? this.localManifestProvider?.().revision : void 0;
|
|
6278
|
-
refusals = this.peerServesAgreement ? [...this.pendingRefusals.values()] : [];
|
|
7040
|
+
refusals = this.peerServesAgreement ? [...this.pendingRefusals.values()].filter((refusal) => refusal.kind === void 0 || this.peerTakesMarkerRefusals()).slice(0, 100) : [];
|
|
6279
7041
|
try {
|
|
6280
7042
|
response = await this.executeGraphQL(this.pollQuery(this.peerServesDecisionFields, this.peerServesAgreement), this.peerServesAgreement ? {
|
|
6281
7043
|
...variables,
|
|
@@ -6293,7 +7055,7 @@ var GqlRequestChannel = class {
|
|
|
6293
7055
|
this.peerServesDecisionFields = false;
|
|
6294
7056
|
}
|
|
6295
7057
|
}
|
|
6296
|
-
for (const refusal of refusals) this.pendingRefusals.delete(
|
|
7058
|
+
for (const refusal of refusals) this.pendingRefusals.delete(refusalKey(refusal));
|
|
6297
7059
|
return {
|
|
6298
7060
|
envelopes: response.pollSyncEnvelopes.envelopes,
|
|
6299
7061
|
ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
|
|
@@ -6303,11 +7065,15 @@ var GqlRequestChannel = class {
|
|
|
6303
7065
|
peerManifestRevision: response.pollSyncEnvelopes.peerManifestRevision
|
|
6304
7066
|
};
|
|
6305
7067
|
}
|
|
7068
|
+
/** A peer before the feature would reject the field; its refusals wait. */
|
|
7069
|
+
peerTakesMarkerRefusals() {
|
|
7070
|
+
return this.peerManifest?.features[MARKER_REFUSAL_FEATURE]?.includes(1) === true;
|
|
7071
|
+
}
|
|
6306
7072
|
rejectsAgreementFields(error) {
|
|
6307
7073
|
return this.peerServesAgreement && this.isAgreementRejection(error);
|
|
6308
7074
|
}
|
|
6309
7075
|
isAgreementRejection(error) {
|
|
6310
|
-
if (!(error instanceof GraphQLRequestError) || error.category !== "graphql") return false;
|
|
7076
|
+
if (!(error instanceof GraphQLRequestError) || error.category !== "graphql" || isRecoverableGraphQLError(error)) return false;
|
|
6311
7077
|
return AGREEMENT_FIELDS.some((field) => error.message.includes(field));
|
|
6312
7078
|
}
|
|
6313
7079
|
/** Resolves once what the silent peer cannot run is held. */
|
|
@@ -6491,6 +7257,25 @@ var GqlRequestChannel = class {
|
|
|
6491
7257
|
}
|
|
6492
7258
|
});
|
|
6493
7259
|
}
|
|
7260
|
+
/** Stops pushing a marker entry the remote refused; its wire has no op id. */
|
|
7261
|
+
retireRefusedMarkers(deadLetters) {
|
|
7262
|
+
const refused = deadLetters.filter((dl) => dl.errorType === "MARKER_REFUSED");
|
|
7263
|
+
if (refused.length === 0) return;
|
|
7264
|
+
const retired = this.outbox.items.filter((syncOp) => this.markerPushedAt.has(syncOp.id) && syncOp.status !== 2 && refused.some((dl) => dl.documentId === syncOp.documentId && dl.branch === syncOp.branch));
|
|
7265
|
+
if (retired.length === 0) return;
|
|
7266
|
+
for (const syncOp of retired) syncOp.failed(new ChannelError("outbox", /* @__PURE__ */ new Error(`Remote refused the purge marker of ${syncOp.documentId}`), "MARKER_REFUSED"));
|
|
7267
|
+
this.outbox.remove(...retired);
|
|
7268
|
+
}
|
|
7269
|
+
/** Re-pushes markers unacked for retryMaxDelayMs: a restarted remote lost them. */
|
|
7270
|
+
repushUnackedMarkers() {
|
|
7271
|
+
if (this.isPushing || this.pushBlocked || this.receivingPages) return;
|
|
7272
|
+
const due = Date.now() - this.config.retryMaxDelayMs;
|
|
7273
|
+
const stale = this.outbox.items.filter((syncOp) => {
|
|
7274
|
+
const pushedAt = this.markerPushedAt.get(syncOp.id);
|
|
7275
|
+
return pushedAt !== void 0 && pushedAt <= due && syncOp.status !== 2;
|
|
7276
|
+
});
|
|
7277
|
+
if (stale.length > 0) this.attemptPush(stale);
|
|
7278
|
+
}
|
|
6494
7279
|
/**
|
|
6495
7280
|
* Schedules a retry of all current outbox items using exponential backoff.
|
|
6496
7281
|
*/
|
|
@@ -6543,7 +7328,11 @@ var GqlRequestChannel = class {
|
|
|
6543
7328
|
* Creates one SyncEnvelope per SyncOperation with key/dependsOn for batch ordering.
|
|
6544
7329
|
*/
|
|
6545
7330
|
async pushSyncOperations(syncOps) {
|
|
6546
|
-
|
|
7331
|
+
const now = Date.now();
|
|
7332
|
+
for (const syncOp of syncOps) {
|
|
7333
|
+
syncOp.started();
|
|
7334
|
+
if (syncOp.operations.some((op) => isPurgeMarker$1(op))) this.markerPushedAt.set(syncOp.id, now);
|
|
7335
|
+
}
|
|
6547
7336
|
const gatedUnder = this.peerServesAgreement ? this.peerManifest?.revision : void 0;
|
|
6548
7337
|
try {
|
|
6549
7338
|
await this.executeGraphQL(pushMutation(gatedUnder !== void 0), {
|
|
@@ -6766,7 +7555,7 @@ var GqlResponseChannel = class {
|
|
|
6766
7555
|
this.remoteName = remoteName;
|
|
6767
7556
|
this.cursorStorage = cursorStorage;
|
|
6768
7557
|
this.isShutdown = false;
|
|
6769
|
-
this.inbox = new Mailbox();
|
|
7558
|
+
this.inbox = new Mailbox({ holdAckBelowMarkers: true });
|
|
6770
7559
|
this.outbox = new Mailbox();
|
|
6771
7560
|
this.deadLetter = new Mailbox();
|
|
6772
7561
|
this.outbox.onRemoved((syncOps) => {
|
|
@@ -6776,8 +7565,8 @@ var GqlResponseChannel = class {
|
|
|
6776
7565
|
this.outbox.onAdded((syncOps) => {
|
|
6777
7566
|
this.forgetEvictedBelow(syncOps);
|
|
6778
7567
|
});
|
|
6779
|
-
this.inbox.onRemoved((
|
|
6780
|
-
const maxOrdinal =
|
|
7568
|
+
this.inbox.onRemoved(() => {
|
|
7569
|
+
const maxOrdinal = this.inbox.ackOrdinal;
|
|
6781
7570
|
if (maxOrdinal > this.lastPersistedInboxOrdinal) {
|
|
6782
7571
|
this.lastPersistedInboxOrdinal = maxOrdinal;
|
|
6783
7572
|
this.cursorStorage.upsert({
|
|
@@ -6928,6 +7717,40 @@ var GqlResponseChannelFactory = class {
|
|
|
6928
7717
|
}
|
|
6929
7718
|
};
|
|
6930
7719
|
//#endregion
|
|
7720
|
+
//#region src/storage/kysely/delivery-lookup.ts
|
|
7721
|
+
/** The index row at `ordinal` and every membership of its document. */
|
|
7722
|
+
async function deliveryAt(db, documentId, ordinal) {
|
|
7723
|
+
const row = await db.selectFrom("operation_index_operations").selectAll().where("ordinal", "=", ordinal).where("documentId", "=", documentId).executeTakeFirst();
|
|
7724
|
+
if (!row) return void 0;
|
|
7725
|
+
const memberships = await db.selectFrom("document_collections").select([
|
|
7726
|
+
"collectionId",
|
|
7727
|
+
"joinedOrdinal",
|
|
7728
|
+
"leftOrdinal"
|
|
7729
|
+
]).where("documentId", "=", documentId).execute();
|
|
7730
|
+
return {
|
|
7731
|
+
entry: {
|
|
7732
|
+
ordinal: row.ordinal,
|
|
7733
|
+
documentId: row.documentId,
|
|
7734
|
+
documentType: row.documentType,
|
|
7735
|
+
branch: row.branch,
|
|
7736
|
+
scope: row.scope,
|
|
7737
|
+
index: row.index,
|
|
7738
|
+
timestampUtcMs: row.timestampUtcMs,
|
|
7739
|
+
hash: row.hash,
|
|
7740
|
+
skip: row.skip,
|
|
7741
|
+
action: row.action,
|
|
7742
|
+
deniedReason: row.deniedReason ?? void 0,
|
|
7743
|
+
id: row.opId,
|
|
7744
|
+
sourceRemote: row.sourceRemote
|
|
7745
|
+
},
|
|
7746
|
+
memberships: memberships.map((membership) => ({
|
|
7747
|
+
collectionId: membership.collectionId,
|
|
7748
|
+
joinedOrdinal: Number(membership.joinedOrdinal),
|
|
7749
|
+
leftOrdinal: membership.leftOrdinal === null ? null : Number(membership.leftOrdinal)
|
|
7750
|
+
}))
|
|
7751
|
+
};
|
|
7752
|
+
}
|
|
7753
|
+
//#endregion
|
|
6931
7754
|
//#region src/storage/kysely/sync-cursor-storage.ts
|
|
6932
7755
|
function rowToRemoteCursor(row) {
|
|
6933
7756
|
return {
|
|
@@ -7052,6 +7875,8 @@ var KyselySyncDeadLetterStorage = class {
|
|
|
7052
7875
|
async add(deadLetter, signal) {
|
|
7053
7876
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
7054
7877
|
await this.db.transaction().execute(async (trx) => {
|
|
7878
|
+
await acquirePurgeLocks(trx, [deadLetter.documentId], "shared");
|
|
7879
|
+
if ((await findPurged(trx, [deadLetter.documentId])).size > 0) throw new DocumentPurgedError(deadLetter.documentId);
|
|
7055
7880
|
const insertable = deadLetterRecordToRow(deadLetter);
|
|
7056
7881
|
await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").doNothing()).execute();
|
|
7057
7882
|
});
|
|
@@ -7109,14 +7934,20 @@ var KyselySyncHoldStorage = class {
|
|
|
7109
7934
|
version: hold.version,
|
|
7110
7935
|
held_at_utc_ms: hold.heldAtUtcMs
|
|
7111
7936
|
};
|
|
7112
|
-
await this.db.
|
|
7113
|
-
"
|
|
7114
|
-
|
|
7115
|
-
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7937
|
+
await this.db.transaction().execute(async (trx) => {
|
|
7938
|
+
await acquirePurgeLocks(trx, [hold.documentId], "shared");
|
|
7939
|
+
if (hold.protocol !== DOCUMENT_PURGE_PROTOCOL) {
|
|
7940
|
+
if ((await findPurged(trx, [hold.documentId])).size > 0) throw new DocumentPurgedError(hold.documentId);
|
|
7941
|
+
}
|
|
7942
|
+
await trx.insertInto("sync_holds").values(row).onConflict((oc) => oc.columns([
|
|
7943
|
+
"remote_name",
|
|
7944
|
+
"document_id",
|
|
7945
|
+
"branch"
|
|
7946
|
+
]).doUpdateSet({
|
|
7947
|
+
protocol: row.protocol,
|
|
7948
|
+
version: row.version
|
|
7949
|
+
})).execute();
|
|
7950
|
+
});
|
|
7120
7951
|
}
|
|
7121
7952
|
async remove(remoteName, documentId, branch) {
|
|
7122
7953
|
await this.db.deleteFrom("sync_holds").where("remote_name", "=", remoteName).where("document_id", "=", documentId).where("branch", "=", branch).execute();
|
|
@@ -7126,6 +7957,69 @@ var KyselySyncHoldStorage = class {
|
|
|
7126
7957
|
}
|
|
7127
7958
|
};
|
|
7128
7959
|
//#endregion
|
|
7960
|
+
//#region src/storage/kysely/sync-purge-refusal-storage.ts
|
|
7961
|
+
var KyselySyncPurgeRefusalStorage = class {
|
|
7962
|
+
db;
|
|
7963
|
+
constructor(db) {
|
|
7964
|
+
this.db = db;
|
|
7965
|
+
}
|
|
7966
|
+
async list(documentId) {
|
|
7967
|
+
return (await this.db.selectFrom("sync_purge_refusals").selectAll().where("document_id", "=", documentId).orderBy("refused_at_utc_ms").orderBy("remote_name").execute()).map((row) => ({
|
|
7968
|
+
remoteName: row.remote_name,
|
|
7969
|
+
documentId: row.document_id,
|
|
7970
|
+
branch: row.branch,
|
|
7971
|
+
refusedAtUtcMs: Number(row.refused_at_utc_ms)
|
|
7972
|
+
}));
|
|
7973
|
+
}
|
|
7974
|
+
async record(refusals) {
|
|
7975
|
+
if (refusals.length === 0) return;
|
|
7976
|
+
await this.db.insertInto("sync_purge_refusals").values(refusals.map((refusal) => ({
|
|
7977
|
+
remote_name: refusal.remoteName,
|
|
7978
|
+
document_id: refusal.documentId,
|
|
7979
|
+
branch: refusal.branch,
|
|
7980
|
+
refused_at_utc_ms: refusal.refusedAtUtcMs
|
|
7981
|
+
}))).onConflict((oc) => oc.columns([
|
|
7982
|
+
"remote_name",
|
|
7983
|
+
"document_id",
|
|
7984
|
+
"branch"
|
|
7985
|
+
]).doNothing()).execute();
|
|
7986
|
+
}
|
|
7987
|
+
};
|
|
7988
|
+
//#endregion
|
|
7989
|
+
//#region src/storage/kysely/sync-received-marker-storage.ts
|
|
7990
|
+
var KyselySyncReceivedMarkerStorage = class {
|
|
7991
|
+
db;
|
|
7992
|
+
constructor(db) {
|
|
7993
|
+
this.db = db;
|
|
7994
|
+
}
|
|
7995
|
+
async list(remoteName) {
|
|
7996
|
+
return (await this.db.selectFrom("sync_received_markers").selectAll().where("remote_name", "=", remoteName).orderBy("received_at_utc_ms").execute()).map((row) => ({
|
|
7997
|
+
remoteName: row.remote_name,
|
|
7998
|
+
markerId: row.marker_id,
|
|
7999
|
+
documentId: row.document_id,
|
|
8000
|
+
branch: row.branch,
|
|
8001
|
+
operation: row.operation,
|
|
8002
|
+
receivedAtUtcMs: Number(row.received_at_utc_ms)
|
|
8003
|
+
}));
|
|
8004
|
+
}
|
|
8005
|
+
async upsert(record) {
|
|
8006
|
+
await this.db.insertInto("sync_received_markers").values({
|
|
8007
|
+
remote_name: record.remoteName,
|
|
8008
|
+
marker_id: record.markerId,
|
|
8009
|
+
document_id: record.documentId,
|
|
8010
|
+
branch: record.branch,
|
|
8011
|
+
operation: JSON.stringify(record.operation),
|
|
8012
|
+
received_at_utc_ms: record.receivedAtUtcMs
|
|
8013
|
+
}).onConflict((oc) => oc.columns(["remote_name", "marker_id"]).doNothing()).execute();
|
|
8014
|
+
}
|
|
8015
|
+
async remove(remoteName, markerId) {
|
|
8016
|
+
await this.db.deleteFrom("sync_received_markers").where("remote_name", "=", remoteName).where("marker_id", "=", markerId).execute();
|
|
8017
|
+
}
|
|
8018
|
+
async removeRemote(remoteName) {
|
|
8019
|
+
await this.db.deleteFrom("sync_received_markers").where("remote_name", "=", remoteName).execute();
|
|
8020
|
+
}
|
|
8021
|
+
};
|
|
8022
|
+
//#endregion
|
|
7129
8023
|
//#region src/storage/kysely/sync-remote-storage.ts
|
|
7130
8024
|
function rowToPeer(row) {
|
|
7131
8025
|
if (row.peer_manifest_at_utc_ms === null) return;
|
|
@@ -7244,6 +8138,8 @@ var KyselySyncRemoteStorage = class {
|
|
|
7244
8138
|
//#region src/sync/batch-aggregator.ts
|
|
7245
8139
|
/** Queued between write-ready events when the settled watermark advances. */
|
|
7246
8140
|
const SETTLED = Symbol("settled");
|
|
8141
|
+
/** Batches remembered as finished, so a repeated JOB_FAILED is ignored. */
|
|
8142
|
+
const FINISHED_BATCH_MEMORY = 1024;
|
|
7247
8143
|
var BatchAggregator = class {
|
|
7248
8144
|
logger;
|
|
7249
8145
|
driveContainerTypes;
|
|
@@ -7252,6 +8148,7 @@ var BatchAggregator = class {
|
|
|
7252
8148
|
queue = [];
|
|
7253
8149
|
processing = false;
|
|
7254
8150
|
pendingBatches = /* @__PURE__ */ new Map();
|
|
8151
|
+
finishedBatches = /* @__PURE__ */ new Set();
|
|
7255
8152
|
constructor(logger, driveContainerTypes, onBatchReady, onSettled) {
|
|
7256
8153
|
this.logger = logger;
|
|
7257
8154
|
this.driveContainerTypes = driveContainerTypes;
|
|
@@ -7268,16 +8165,13 @@ var BatchAggregator = class {
|
|
|
7268
8165
|
await this.processQueue();
|
|
7269
8166
|
}
|
|
7270
8167
|
async handleJobFailed(event) {
|
|
7271
|
-
|
|
7272
|
-
|
|
7273
|
-
const pending = this.pendingBatches.get(batchId);
|
|
7274
|
-
if (!pending) return;
|
|
7275
|
-
this.pendingBatches.delete(batchId);
|
|
7276
|
-
if (pending.events.length > 0) await this.onBatchReady(this.prepareBatch(pending.events));
|
|
8168
|
+
this.queue.push({ failed: event });
|
|
8169
|
+
await this.processQueue();
|
|
7277
8170
|
}
|
|
7278
8171
|
clear() {
|
|
7279
8172
|
this.queue = [];
|
|
7280
8173
|
this.pendingBatches.clear();
|
|
8174
|
+
this.finishedBatches.clear();
|
|
7281
8175
|
}
|
|
7282
8176
|
async processQueue() {
|
|
7283
8177
|
if (this.processing) return;
|
|
@@ -7293,6 +8187,14 @@ var BatchAggregator = class {
|
|
|
7293
8187
|
}
|
|
7294
8188
|
continue;
|
|
7295
8189
|
}
|
|
8190
|
+
if ("failed" in item) {
|
|
8191
|
+
try {
|
|
8192
|
+
await this.handleFailed(item.failed);
|
|
8193
|
+
} catch (error) {
|
|
8194
|
+
this.logger.error("Failed to process job-failed event (@jobId, @error)", item.failed.jobId, error instanceof Error ? error.message : String(error));
|
|
8195
|
+
}
|
|
8196
|
+
continue;
|
|
8197
|
+
}
|
|
7296
8198
|
try {
|
|
7297
8199
|
await this.handleWriteReady(item);
|
|
7298
8200
|
} catch (error) {
|
|
@@ -7306,25 +8208,49 @@ var BatchAggregator = class {
|
|
|
7306
8208
|
}
|
|
7307
8209
|
async handleWriteReady(event) {
|
|
7308
8210
|
const { batchId, batchJobIds } = event.jobMeta;
|
|
7309
|
-
|
|
8211
|
+
const pending = batchJobIds.length > 1 ? this.pendingFor(batchId, batchJobIds) : void 0;
|
|
8212
|
+
if (!pending) {
|
|
7310
8213
|
await this.onBatchReady(this.prepareBatch([event]));
|
|
7311
8214
|
return;
|
|
7312
8215
|
}
|
|
8216
|
+
pending.resolvedJobIds.add(event.jobId);
|
|
8217
|
+
pending.events.push(event);
|
|
8218
|
+
if (this.finishIfResolved(batchId, pending)) await this.onBatchReady(this.prepareBatch(pending.events));
|
|
8219
|
+
}
|
|
8220
|
+
/** A failed job never arrives, so what its batch holds goes now. */
|
|
8221
|
+
async handleFailed(event) {
|
|
8222
|
+
const meta = event.job?.meta;
|
|
8223
|
+
if (!meta?.batchId || meta.batchJobIds.length <= 1) return;
|
|
8224
|
+
const pending = this.pendingFor(meta.batchId, meta.batchJobIds);
|
|
8225
|
+
if (!pending) return;
|
|
8226
|
+
pending.resolvedJobIds.add(event.jobId);
|
|
8227
|
+
this.finishIfResolved(meta.batchId, pending);
|
|
8228
|
+
const events = pending.events;
|
|
8229
|
+
pending.events = [];
|
|
8230
|
+
if (events.length > 0) await this.onBatchReady(this.prepareBatch(events));
|
|
8231
|
+
}
|
|
8232
|
+
pendingFor(batchId, batchJobIds) {
|
|
8233
|
+
if (this.finishedBatches.has(batchId)) return;
|
|
7313
8234
|
let pending = this.pendingBatches.get(batchId);
|
|
7314
8235
|
if (!pending) {
|
|
7315
8236
|
pending = {
|
|
7316
8237
|
expectedJobIds: new Set(batchJobIds),
|
|
7317
|
-
|
|
8238
|
+
resolvedJobIds: /* @__PURE__ */ new Set(),
|
|
7318
8239
|
events: []
|
|
7319
8240
|
};
|
|
7320
8241
|
this.pendingBatches.set(batchId, pending);
|
|
7321
8242
|
}
|
|
7322
|
-
pending
|
|
7323
|
-
|
|
7324
|
-
|
|
7325
|
-
|
|
7326
|
-
|
|
8243
|
+
return pending;
|
|
8244
|
+
}
|
|
8245
|
+
finishIfResolved(batchId, pending) {
|
|
8246
|
+
for (const id of pending.expectedJobIds) if (!pending.resolvedJobIds.has(id)) return false;
|
|
8247
|
+
this.pendingBatches.delete(batchId);
|
|
8248
|
+
this.finishedBatches.add(batchId);
|
|
8249
|
+
if (this.finishedBatches.size > FINISHED_BATCH_MEMORY) {
|
|
8250
|
+
const oldest = this.finishedBatches.values().next().value;
|
|
8251
|
+
this.finishedBatches.delete(oldest);
|
|
7327
8252
|
}
|
|
8253
|
+
return true;
|
|
7328
8254
|
}
|
|
7329
8255
|
prepareBatch(events) {
|
|
7330
8256
|
const collectionMemberships = this.mergeCollectionMemberships(events);
|
|
@@ -7365,8 +8291,42 @@ var BatchAggregator = class {
|
|
|
7365
8291
|
}
|
|
7366
8292
|
};
|
|
7367
8293
|
//#endregion
|
|
8294
|
+
//#region src/sync/delivery-tracking.ts
|
|
8295
|
+
function supportsDeliveryTracking(x) {
|
|
8296
|
+
return typeof x === "object" && x !== null && typeof x.pendingDelivery === "function";
|
|
8297
|
+
}
|
|
8298
|
+
/** The outbox's rule: remotes it would serve the row to, not yet acknowledged. */
|
|
8299
|
+
async function pendingDelivery(deps, documentId, ordinal) {
|
|
8300
|
+
const row = await deps.lookup.at(documentId, ordinal);
|
|
8301
|
+
if (!row) throw new Error(`No operation of document ${documentId} at ordinal ${ordinal}`);
|
|
8302
|
+
const { entry, memberships } = row;
|
|
8303
|
+
const operation = toOperationWithContext(entry);
|
|
8304
|
+
const holds = await deps.holds.list({ documentId });
|
|
8305
|
+
const pending = [];
|
|
8306
|
+
for (const remote of deps.remotes) {
|
|
8307
|
+
const name = remote.meta.name;
|
|
8308
|
+
const membership = memberships.find((candidate) => candidate.collectionId === remote.meta.collectionId.key && (candidate.leftOrdinal === null || ordinal < candidate.leftOrdinal));
|
|
8309
|
+
if (!membership) continue;
|
|
8310
|
+
if (entry.sourceRemote === name) continue;
|
|
8311
|
+
if (filterForRemote([operation], remote.meta.filter).length === 0) continue;
|
|
8312
|
+
if (holds.some((hold) => hold.remoteName === name && hold.branch === entry.branch)) {
|
|
8313
|
+
pending.push({
|
|
8314
|
+
remote: name,
|
|
8315
|
+
state: "held"
|
|
8316
|
+
});
|
|
8317
|
+
continue;
|
|
8318
|
+
}
|
|
8319
|
+
const owedThrough = Math.max(ordinal, membership.joinedOrdinal);
|
|
8320
|
+
if ((await deps.cursors.get(name, "outbox")).cursorOrdinal < owedThrough) pending.push({
|
|
8321
|
+
remote: name,
|
|
8322
|
+
state: remote.channel.getConnectionState().state
|
|
8323
|
+
});
|
|
8324
|
+
}
|
|
8325
|
+
return pending;
|
|
8326
|
+
}
|
|
8327
|
+
//#endregion
|
|
7368
8328
|
//#region src/sync/memory-hold-storage.ts
|
|
7369
|
-
const keyOf = (remoteName, documentId, branch) => `${remoteName}\u0000${documentId}\u0000${branch}`;
|
|
8329
|
+
const keyOf$2 = (remoteName, documentId, branch) => `${remoteName}\u0000${documentId}\u0000${branch}`;
|
|
7370
8330
|
/** For a sync manager built without a database. */
|
|
7371
8331
|
var InMemorySyncHoldStorage = class {
|
|
7372
8332
|
holds = /* @__PURE__ */ new Map();
|
|
@@ -7374,7 +8334,7 @@ var InMemorySyncHoldStorage = class {
|
|
|
7374
8334
|
return Promise.resolve([...this.holds.values()].filter((hold) => (filter.remoteName === void 0 || hold.remoteName === filter.remoteName) && (filter.documentId === void 0 || hold.documentId === filter.documentId)));
|
|
7375
8335
|
}
|
|
7376
8336
|
upsert(hold) {
|
|
7377
|
-
const key = keyOf(hold.remoteName, hold.documentId, hold.branch);
|
|
8337
|
+
const key = keyOf$2(hold.remoteName, hold.documentId, hold.branch);
|
|
7378
8338
|
const known = this.holds.get(key);
|
|
7379
8339
|
this.holds.set(key, {
|
|
7380
8340
|
...hold,
|
|
@@ -7383,7 +8343,7 @@ var InMemorySyncHoldStorage = class {
|
|
|
7383
8343
|
return Promise.resolve();
|
|
7384
8344
|
}
|
|
7385
8345
|
remove(remoteName, documentId, branch) {
|
|
7386
|
-
this.holds.delete(keyOf(remoteName, documentId, branch));
|
|
8346
|
+
this.holds.delete(keyOf$2(remoteName, documentId, branch));
|
|
7387
8347
|
return Promise.resolve();
|
|
7388
8348
|
}
|
|
7389
8349
|
removeRemote(remoteName) {
|
|
@@ -7392,6 +8352,46 @@ var InMemorySyncHoldStorage = class {
|
|
|
7392
8352
|
}
|
|
7393
8353
|
};
|
|
7394
8354
|
//#endregion
|
|
8355
|
+
//#region src/sync/memory-purge-refusal-storage.ts
|
|
8356
|
+
const keyOf$1 = (refusal) => `${refusal.remoteName}\u0000${refusal.documentId}\u0000${refusal.branch}`;
|
|
8357
|
+
/** For a sync manager built without a database. */
|
|
8358
|
+
var InMemorySyncPurgeRefusalStorage = class {
|
|
8359
|
+
records = /* @__PURE__ */ new Map();
|
|
8360
|
+
list(documentId) {
|
|
8361
|
+
return Promise.resolve([...this.records.values()].filter((record) => record.documentId === documentId));
|
|
8362
|
+
}
|
|
8363
|
+
record(refusals) {
|
|
8364
|
+
for (const refusal of refusals) {
|
|
8365
|
+
const key = keyOf$1(refusal);
|
|
8366
|
+
if (!this.records.has(key)) this.records.set(key, refusal);
|
|
8367
|
+
}
|
|
8368
|
+
return Promise.resolve();
|
|
8369
|
+
}
|
|
8370
|
+
};
|
|
8371
|
+
//#endregion
|
|
8372
|
+
//#region src/sync/memory-received-marker-storage.ts
|
|
8373
|
+
const keyOf = (remoteName, markerId) => `${remoteName}\u0000${markerId}`;
|
|
8374
|
+
/** For a sync manager built without a database. */
|
|
8375
|
+
var InMemorySyncReceivedMarkerStorage = class {
|
|
8376
|
+
records = /* @__PURE__ */ new Map();
|
|
8377
|
+
list(remoteName) {
|
|
8378
|
+
return Promise.resolve([...this.records.values()].filter((record) => record.remoteName === remoteName));
|
|
8379
|
+
}
|
|
8380
|
+
upsert(record) {
|
|
8381
|
+
const key = keyOf(record.remoteName, record.markerId);
|
|
8382
|
+
if (!this.records.has(key)) this.records.set(key, record);
|
|
8383
|
+
return Promise.resolve();
|
|
8384
|
+
}
|
|
8385
|
+
remove(remoteName, markerId) {
|
|
8386
|
+
this.records.delete(keyOf(remoteName, markerId));
|
|
8387
|
+
return Promise.resolve();
|
|
8388
|
+
}
|
|
8389
|
+
removeRemote(remoteName) {
|
|
8390
|
+
for (const [key, record] of this.records) if (record.remoteName === remoteName) this.records.delete(key);
|
|
8391
|
+
return Promise.resolve();
|
|
8392
|
+
}
|
|
8393
|
+
};
|
|
8394
|
+
//#endregion
|
|
7395
8395
|
//#region src/sync/peer-agreement.ts
|
|
7396
8396
|
/** Pure over the remote records it is given, live or not. */
|
|
7397
8397
|
function createPeerAgreement(basis, remotes) {
|
|
@@ -7637,7 +8637,9 @@ const defaultSyncManagerConfig = {
|
|
|
7637
8637
|
maxDeadLettersPerRemote: 100,
|
|
7638
8638
|
maxInboxBatchSize: 32,
|
|
7639
8639
|
maxHeldOperationsPerRemote: 1e4,
|
|
7640
|
-
staleRemotePollWindowMs: 3e5
|
|
8640
|
+
staleRemotePollWindowMs: 3e5,
|
|
8641
|
+
markerRetryBaseDelayMs: 1e3,
|
|
8642
|
+
markerRetryMaxDelayMs: 6e4
|
|
7641
8643
|
};
|
|
7642
8644
|
const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
|
|
7643
8645
|
/**
|
|
@@ -7655,6 +8657,20 @@ const VERSION_CACHE_CAP = 1e4;
|
|
|
7655
8657
|
function createdVersions(operations, documentId) {
|
|
7656
8658
|
for (const { operation, context } of operations) if (context.documentId === documentId && operation.action.type === "CREATE_DOCUMENT") return operation.action.input.protocolVersions ?? {};
|
|
7657
8659
|
}
|
|
8660
|
+
/** The load was refused because this very id is purged here: a drop. */
|
|
8661
|
+
function isPurgedFailure(error, documentId) {
|
|
8662
|
+
return error?.name === "DocumentPurgedError" && error.documentId === documentId;
|
|
8663
|
+
}
|
|
8664
|
+
function carriesMarker(syncOp) {
|
|
8665
|
+
return syncOp.operations.some((op) => isPurgeMarker$1(op));
|
|
8666
|
+
}
|
|
8667
|
+
function markerIdsOf(syncOp) {
|
|
8668
|
+
return syncOp.operations.filter((op) => isPurgeMarker$1(op)).map((op) => op.operation.id);
|
|
8669
|
+
}
|
|
8670
|
+
/** Refused, or purged under another id: final, unlike an outage. */
|
|
8671
|
+
function isRefusedMarker(error) {
|
|
8672
|
+
return error?.name === "InvalidSignatureError" || error?.name === "DocumentPurgedError";
|
|
8673
|
+
}
|
|
7658
8674
|
/** Where a sync operation's run of ordinals begins. */
|
|
7659
8675
|
function firstOrdinalOf(syncOp) {
|
|
7660
8676
|
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
@@ -7680,6 +8696,10 @@ var SyncManager = class {
|
|
|
7680
8696
|
config;
|
|
7681
8697
|
connectionStateUnsubscribes = /* @__PURE__ */ new Map();
|
|
7682
8698
|
quarantinedDocumentIds = /* @__PURE__ */ new Set();
|
|
8699
|
+
purgedDocumentIds = /* @__PURE__ */ new Set();
|
|
8700
|
+
purges;
|
|
8701
|
+
delivery;
|
|
8702
|
+
forgetDocument;
|
|
7683
8703
|
backfillAbortControllers = /* @__PURE__ */ new Map();
|
|
7684
8704
|
planKeyToJobUuid = /* @__PURE__ */ new Map();
|
|
7685
8705
|
evictedOutboxFloors = /* @__PURE__ */ new Map();
|
|
@@ -7691,6 +8711,8 @@ var SyncManager = class {
|
|
|
7691
8711
|
lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
|
|
7692
8712
|
watermark;
|
|
7693
8713
|
owed = /* @__PURE__ */ new Map();
|
|
8714
|
+
derivedThrough = /* @__PURE__ */ new Map();
|
|
8715
|
+
sweptThrough = 0;
|
|
7694
8716
|
settledUnsubscribe;
|
|
7695
8717
|
inboxChunkChain = Promise.resolve();
|
|
7696
8718
|
capabilities;
|
|
@@ -7705,17 +8727,29 @@ var SyncManager = class {
|
|
|
7705
8727
|
protocolVersionsOf;
|
|
7706
8728
|
peerUpdates = /* @__PURE__ */ new Map();
|
|
7707
8729
|
peerUnsubscribes = /* @__PURE__ */ new Map();
|
|
8730
|
+
receivedMarkers = /* @__PURE__ */ new Map();
|
|
8731
|
+
markerStorage;
|
|
8732
|
+
refusalStorage;
|
|
8733
|
+
markerWrites = /* @__PURE__ */ new Map();
|
|
8734
|
+
restoredMarkers = /* @__PURE__ */ new WeakSet();
|
|
8735
|
+
markerRetries = /* @__PURE__ */ new Map();
|
|
7708
8736
|
constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, watermark, config = {}, localPeer = {
|
|
7709
8737
|
capabilities: PEER_CAPABILITIES,
|
|
7710
8738
|
flags: {}
|
|
7711
|
-
}, holds = new InMemorySyncHoldStorage()) {
|
|
8739
|
+
}, holds = new InMemorySyncHoldStorage(), purges, receivedMarkers = new InMemorySyncReceivedMarkerStorage(), delivery, refusals = new InMemorySyncPurgeRefusalStorage()) {
|
|
8740
|
+
this.markerStorage = receivedMarkers;
|
|
8741
|
+
this.refusalStorage = refusals;
|
|
7712
8742
|
this.watermark = watermark;
|
|
8743
|
+
this.purges = purges;
|
|
8744
|
+
this.delivery = delivery;
|
|
8745
|
+
this.forgetDocument = localPeer.forgetDocument;
|
|
7713
8746
|
this.capabilities = localPeer.capabilities;
|
|
7714
8747
|
this.manifest = localPeerManifest(localPeer.capabilities, localPeer.flags, localPeer.appKey, localPeer.sequence ?? Date.now());
|
|
7715
8748
|
this.localSupport = localSupports(localPeer.capabilities, localPeer.flags);
|
|
7716
8749
|
this.legacy = legacySupports(localPeer.capabilities);
|
|
7717
8750
|
for (const capability of localPeer.capabilities) {
|
|
7718
8751
|
if (capability.kind !== "protocol") continue;
|
|
8752
|
+
if (capability.name === DOCUMENT_PURGE_PROTOCOL) continue;
|
|
7719
8753
|
const supported = capability.supported(localPeer.flags);
|
|
7720
8754
|
const wanted = capability.preferred?.(localPeer.flags) ?? (supported.length > 0 ? Math.max(...supported) : void 0);
|
|
7721
8755
|
if (wanted !== void 0) this.wanted[capability.name] = wanted;
|
|
@@ -7749,6 +8783,11 @@ var SyncManager = class {
|
|
|
7749
8783
|
} catch (error) {
|
|
7750
8784
|
this.logger.error("Failed to load quarantined document IDs (@error)", error instanceof Error ? error.message : String(error));
|
|
7751
8785
|
}
|
|
8786
|
+
try {
|
|
8787
|
+
for (const id of await this.purges?.listPurged() ?? []) this.purgedDocumentIds.add(id);
|
|
8788
|
+
} catch (error) {
|
|
8789
|
+
this.logger.error("Failed to load purged document IDs (@error)", error instanceof Error ? error.message : String(error));
|
|
8790
|
+
}
|
|
7752
8791
|
try {
|
|
7753
8792
|
for (const hold of await this.holds.list()) this.heldFor(hold.remoteName).set(holdKey(hold.documentId, hold.branch), hold);
|
|
7754
8793
|
} catch (error) {
|
|
@@ -7756,6 +8795,7 @@ var SyncManager = class {
|
|
|
7756
8795
|
}
|
|
7757
8796
|
const remoteRecords = await this.remoteStorage.list();
|
|
7758
8797
|
const head = await this.watermarkHead();
|
|
8798
|
+
this.sweptThrough = Math.max(this.sweptThrough, head);
|
|
7759
8799
|
for (const record of remoteRecords) {
|
|
7760
8800
|
const channel = this.channelFactory.instance(record.id, record.name, record.channelConfig, this.cursorStorage, record.collectionId, record.filter, this.operationIndex, record.options);
|
|
7761
8801
|
const remote = {
|
|
@@ -7774,6 +8814,8 @@ var SyncManager = class {
|
|
|
7774
8814
|
this.owe(record.name, head);
|
|
7775
8815
|
this.records.set(record.name, remote.meta);
|
|
7776
8816
|
await this.loadDeadLetters(remote);
|
|
8817
|
+
await this.restoreReceivedMarkers(remote);
|
|
8818
|
+
const unheard = [...remote.channel.inbox.items];
|
|
7777
8819
|
this.wireChannelCallbacks(remote);
|
|
7778
8820
|
try {
|
|
7779
8821
|
await channel.init();
|
|
@@ -7782,6 +8824,7 @@ var SyncManager = class {
|
|
|
7782
8824
|
await this.dropRemoteAfterFailedInit(remote, false);
|
|
7783
8825
|
continue;
|
|
7784
8826
|
}
|
|
8827
|
+
if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
|
|
7785
8828
|
await this.peerUpdates.get(record.name);
|
|
7786
8829
|
const outboxAckOrdinal = remote.channel.outbox.ackOrdinal;
|
|
7787
8830
|
if (outboxAckOrdinal > 0) {
|
|
@@ -7811,6 +8854,8 @@ var SyncManager = class {
|
|
|
7811
8854
|
this.pruneDrainDeferred = false;
|
|
7812
8855
|
this.batchAggregator.clear();
|
|
7813
8856
|
this.owed.clear();
|
|
8857
|
+
for (const retry of this.markerRetries.values()) clearTimeout(retry.timer);
|
|
8858
|
+
this.markerRetries.clear();
|
|
7814
8859
|
this.settledUnsubscribe?.();
|
|
7815
8860
|
this.settledUnsubscribe = void 0;
|
|
7816
8861
|
if (this.eventUnsubscribe) {
|
|
@@ -7903,6 +8948,47 @@ var SyncManager = class {
|
|
|
7903
8948
|
heldAtUtcMs: record.heldAtUtcMs
|
|
7904
8949
|
}));
|
|
7905
8950
|
}
|
|
8951
|
+
async pendingDelivery(documentId, ordinal) {
|
|
8952
|
+
if (!this.delivery) throw new Error("Delivery tracking needs a delivery lookup");
|
|
8953
|
+
return pendingDelivery({
|
|
8954
|
+
remotes: [...this.remotes.values()].filter((remote) => !this.removing.has(remote.meta.name)),
|
|
8955
|
+
cursors: this.cursorStorage,
|
|
8956
|
+
holds: this.holds,
|
|
8957
|
+
lookup: this.delivery
|
|
8958
|
+
}, documentId, ordinal);
|
|
8959
|
+
}
|
|
8960
|
+
recordPurgeRefusal(refusal) {
|
|
8961
|
+
return this.persistRefusals([refusal]);
|
|
8962
|
+
}
|
|
8963
|
+
async recordPolledMarkerRefusals(remoteName, refusals) {
|
|
8964
|
+
const remote = this.getByName(remoteName);
|
|
8965
|
+
const tombstoned = refusals.slice(0, 100).filter((refusal) => this.purgedDocumentIds.has(refusal.documentId));
|
|
8966
|
+
if (tombstoned.length === 0) return;
|
|
8967
|
+
const collections = await this.operationIndex.getCollectionsForDocuments([...new Set(tombstoned.map((refusal) => refusal.documentId))]);
|
|
8968
|
+
const key = remote.meta.collectionId.key;
|
|
8969
|
+
const owed = /* @__PURE__ */ new Map();
|
|
8970
|
+
for (const refusal of tombstoned) {
|
|
8971
|
+
if (!collections[refusal.documentId]?.includes(key)) continue;
|
|
8972
|
+
owed.set(`${refusal.documentId}\u0000${refusal.branch}`, {
|
|
8973
|
+
remoteName,
|
|
8974
|
+
documentId: refusal.documentId,
|
|
8975
|
+
branch: refusal.branch,
|
|
8976
|
+
errorMessage: `Marker refused by ${remoteName}`
|
|
8977
|
+
});
|
|
8978
|
+
}
|
|
8979
|
+
await this.persistRefusals([...owed.values()]);
|
|
8980
|
+
}
|
|
8981
|
+
async persistRefusals(refusals) {
|
|
8982
|
+
if (refusals.length === 0) return;
|
|
8983
|
+
const refusedAtUtcMs = Date.now();
|
|
8984
|
+
await this.refusalStorage.record(refusals.map((refusal) => ({
|
|
8985
|
+
remoteName: refusal.remoteName,
|
|
8986
|
+
documentId: refusal.documentId,
|
|
8987
|
+
branch: refusal.branch,
|
|
8988
|
+
refusedAtUtcMs
|
|
8989
|
+
})));
|
|
8990
|
+
for (const refusal of refusals) await this.eventBus.emit(SyncEventTypes.PURGE_REFUSED, refusal).catch(() => {});
|
|
8991
|
+
}
|
|
7906
8992
|
agreement() {
|
|
7907
8993
|
return createPeerAgreement({
|
|
7908
8994
|
local: this.manifest,
|
|
@@ -7918,16 +9004,36 @@ var SyncManager = class {
|
|
|
7918
9004
|
}
|
|
7919
9005
|
return held;
|
|
7920
9006
|
}
|
|
9007
|
+
/** Once per id: a second pass would drop the holds its marker earned. */
|
|
9008
|
+
tombstone(documentId) {
|
|
9009
|
+
if (this.purgedDocumentIds.has(documentId)) return;
|
|
9010
|
+
this.purgedDocumentIds.add(documentId);
|
|
9011
|
+
this.quarantinedDocumentIds.delete(documentId);
|
|
9012
|
+
const prefix = holdKey(documentId, "");
|
|
9013
|
+
for (const key of [...this.versionCache.keys()]) if (key.startsWith(prefix)) this.versionCache.delete(key);
|
|
9014
|
+
this.forgetDocument?.(documentId);
|
|
9015
|
+
for (const held of this.heldKeys.values()) for (const [key, record] of [...held]) if (record.documentId === documentId) held.delete(key);
|
|
9016
|
+
for (const remote of this.remotes.values()) {
|
|
9017
|
+
const stale = remote.channel.outbox.items.filter((item) => item.documentId === documentId && item.status < 2 && !carriesMarker(item));
|
|
9018
|
+
for (const item of stale) item.executed();
|
|
9019
|
+
if (stale.length > 0) remote.channel.outbox.remove(...stale);
|
|
9020
|
+
const dead = remote.channel.deadLetter.items.filter((item) => item.documentId === documentId);
|
|
9021
|
+
if (dead.length > 0) remote.channel.deadLetter.remove(...dead);
|
|
9022
|
+
}
|
|
9023
|
+
}
|
|
7921
9024
|
async forgetRemote(name) {
|
|
7922
9025
|
this.records.delete(name);
|
|
7923
9026
|
this.heldKeys.delete(name);
|
|
7924
9027
|
await this.holds.removeRemote(name);
|
|
9028
|
+
await Promise.allSettled(this.markerWritesOf([name]));
|
|
9029
|
+
await this.markerStorage.removeRemote(name);
|
|
7925
9030
|
}
|
|
7926
9031
|
peerSupportsOf(remote) {
|
|
7927
9032
|
return peerSupports(remote.meta.peer?.manifest ?? null, this.capabilities);
|
|
7928
9033
|
}
|
|
7929
|
-
/** Cached once found: a document's versions
|
|
9034
|
+
/** Cached once found: a document's versions change only by a purge. */
|
|
7930
9035
|
async versionsOf(documentId, branch) {
|
|
9036
|
+
if (this.purgedDocumentIds.has(documentId)) return purgedProtocolVersions();
|
|
7931
9037
|
const key = holdKey(documentId, branch);
|
|
7932
9038
|
const cached = this.versionCache.get(key);
|
|
7933
9039
|
if (cached) return cached;
|
|
@@ -7972,6 +9078,11 @@ var SyncManager = class {
|
|
|
7972
9078
|
try {
|
|
7973
9079
|
await this.holds.upsert(record);
|
|
7974
9080
|
} catch (error) {
|
|
9081
|
+
if (DocumentPurgedError.isError(error)) {
|
|
9082
|
+
held.delete(key);
|
|
9083
|
+
this.tombstone(documentId);
|
|
9084
|
+
return;
|
|
9085
|
+
}
|
|
7975
9086
|
this.logger.error("Failed to persist a sync hold (@remote, @documentId): @error", remote.meta.name, documentId, error instanceof Error ? error.message : String(error));
|
|
7976
9087
|
}
|
|
7977
9088
|
this.eventBus.emit(SyncEventTypes.SYNC_HELD, {
|
|
@@ -8036,7 +9147,7 @@ var SyncManager = class {
|
|
|
8036
9147
|
page = await page.next();
|
|
8037
9148
|
}
|
|
8038
9149
|
let operations = entries.filter((entry) => entry.sourceRemote !== remote.meta.name).map((entry) => toOperationWithContext(entry));
|
|
8039
|
-
operations =
|
|
9150
|
+
operations = filterForRemote(operations, remote.meta.filter);
|
|
8040
9151
|
if (operations.length === 0) return;
|
|
8041
9152
|
operations.sort((a, b) => {
|
|
8042
9153
|
if (a.context.scope !== b.context.scope) {
|
|
@@ -8069,6 +9180,7 @@ var SyncManager = class {
|
|
|
8069
9180
|
const peer = this.peerSupportsOf(remote);
|
|
8070
9181
|
const refused = /* @__PURE__ */ new Set();
|
|
8071
9182
|
for (const syncOp of syncOps) {
|
|
9183
|
+
if (carriesMarker(syncOp)) continue;
|
|
8072
9184
|
const versions = createdVersions(syncOp.operations, syncOp.documentId) ?? await this.versionsOf(syncOp.documentId, syncOp.branch);
|
|
8073
9185
|
if (versions && this.refuse(remote, syncOp, versions, peer)) refused.add(syncOp);
|
|
8074
9186
|
}
|
|
@@ -8166,6 +9278,8 @@ var SyncManager = class {
|
|
|
8166
9278
|
this.remotes.set(name, remote);
|
|
8167
9279
|
this.records.set(name, meta);
|
|
8168
9280
|
await this.loadDeadLetters(remote);
|
|
9281
|
+
await this.restoreReceivedMarkers(remote);
|
|
9282
|
+
const unheard = [...remote.channel.inbox.items];
|
|
8169
9283
|
this.wireChannelCallbacks(remote);
|
|
8170
9284
|
try {
|
|
8171
9285
|
await channel.init();
|
|
@@ -8173,6 +9287,7 @@ var SyncManager = class {
|
|
|
8173
9287
|
await this.dropRemoteAfterFailedInit(remote, !isCredentialOrNetworkError(error));
|
|
8174
9288
|
throw error;
|
|
8175
9289
|
}
|
|
9290
|
+
if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
|
|
8176
9291
|
await this.peerUpdates.get(name);
|
|
8177
9292
|
this.owe(name, await this.watermarkHead());
|
|
8178
9293
|
const backfillController = new AbortController();
|
|
@@ -8196,6 +9311,7 @@ var SyncManager = class {
|
|
|
8196
9311
|
if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
|
|
8197
9312
|
this.removing.add(name);
|
|
8198
9313
|
this.owed.delete(name);
|
|
9314
|
+
this.derivedThrough.delete(name);
|
|
8199
9315
|
try {
|
|
8200
9316
|
await this.teardownRemoteResources(remote);
|
|
8201
9317
|
await this.remoteStorage.remove(name);
|
|
@@ -8253,7 +9369,14 @@ var SyncManager = class {
|
|
|
8253
9369
|
this.connectionStateUnsubscribes.delete(name);
|
|
8254
9370
|
}
|
|
8255
9371
|
this.evictedOutboxFloors.delete(name);
|
|
9372
|
+
this.derivedThrough.delete(name);
|
|
8256
9373
|
this.prunePending.delete(name);
|
|
9374
|
+
this.receivedMarkers.delete(name);
|
|
9375
|
+
for (const [id, retry] of [...this.markerRetries]) {
|
|
9376
|
+
if (retry.remoteName !== name) continue;
|
|
9377
|
+
clearTimeout(retry.timer);
|
|
9378
|
+
this.markerRetries.delete(id);
|
|
9379
|
+
}
|
|
8257
9380
|
this.peerUnsubscribes.get(name)?.();
|
|
8258
9381
|
this.peerUnsubscribes.delete(name);
|
|
8259
9382
|
this.peerUpdates.delete(name);
|
|
@@ -8271,6 +9394,10 @@ var SyncManager = class {
|
|
|
8271
9394
|
onSyncStatusChange(callback) {
|
|
8272
9395
|
return this.syncStatusTracker.onChange(callback);
|
|
8273
9396
|
}
|
|
9397
|
+
/** Settles once the remotes' received markers are stored; rejects if one failed. */
|
|
9398
|
+
async receiptsStored(remoteNames) {
|
|
9399
|
+
await Promise.all(this.markerWritesOf(remoteNames));
|
|
9400
|
+
}
|
|
8274
9401
|
recordPlanKeyMapping(planKey, jobId) {
|
|
8275
9402
|
if (!this.planKeyToJobUuid.has(planKey) && this.planKeyToJobUuid.size >= PLAN_KEY_TO_JOB_UUID_CAP) {
|
|
8276
9403
|
const oldest = this.planKeyToJobUuid.keys().next().value;
|
|
@@ -8280,6 +9407,7 @@ var SyncManager = class {
|
|
|
8280
9407
|
}
|
|
8281
9408
|
wireChannelCallbacks(remote) {
|
|
8282
9409
|
remote.channel.inbox.onAdded((syncOps) => this.handleInboxAdded(remote, syncOps));
|
|
9410
|
+
remote.channel.inbox.onRemoved((syncOps) => this.handleInboxRemoved(remote, syncOps));
|
|
8283
9411
|
remote.channel.setLocalManifest?.(() => this.manifest);
|
|
8284
9412
|
const unsubscribePeer = remote.channel.onPeerManifest?.((manifest, undelivered) => this.queuePeerManifest(remote, manifest, undelivered).catch(() => {}));
|
|
8285
9413
|
if (unsubscribePeer) this.peerUnsubscribes.set(remote.meta.name, unsubscribePeer);
|
|
@@ -8300,7 +9428,25 @@ var SyncManager = class {
|
|
|
8300
9428
|
remote.channel.deadLetter.remove(...refusals);
|
|
8301
9429
|
for (const syncOp of refusals) this.holdRefused(remote, syncOp).catch(() => {});
|
|
8302
9430
|
}
|
|
8303
|
-
const
|
|
9431
|
+
const remaining = added.filter((syncOp) => !refusals.includes(syncOp));
|
|
9432
|
+
const purged = remaining.filter((syncOp) => this.purgedDocumentIds.has(syncOp.documentId));
|
|
9433
|
+
for (const syncOp of purged) {
|
|
9434
|
+
this.logger.warn("Dead letter for purged document (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, syncOp.error?.message ?? "unknown");
|
|
9435
|
+
if (syncOp.error?.source !== "outbox") continue;
|
|
9436
|
+
if (syncOperationErrorType(syncOp.error) !== "MARKER_REFUSED") continue;
|
|
9437
|
+
const refusal = {
|
|
9438
|
+
remoteName: remote.meta.name,
|
|
9439
|
+
documentId: syncOp.documentId,
|
|
9440
|
+
branch: syncOp.branch,
|
|
9441
|
+
errorMessage: syncOp.error.error.message
|
|
9442
|
+
};
|
|
9443
|
+
this.recordPurgeRefusal(refusal).catch((error) => {
|
|
9444
|
+
this.logger.error("Recording a refused marker failed (@remote, @documentId, @error)", refusal.remoteName, refusal.documentId, error instanceof Error ? error.message : String(error));
|
|
9445
|
+
remote.channel.forgetMarkerRefusal?.(refusal.documentId, refusal.branch);
|
|
9446
|
+
});
|
|
9447
|
+
}
|
|
9448
|
+
if (purged.length > 0) remote.channel.deadLetter.remove(...purged);
|
|
9449
|
+
const syncOps = remaining.filter((syncOp) => !purged.includes(syncOp));
|
|
8304
9450
|
for (const syncOp of syncOps) {
|
|
8305
9451
|
this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
|
|
8306
9452
|
const errorType = syncOperationErrorType(syncOp.error);
|
|
@@ -8313,12 +9459,16 @@ var SyncManager = class {
|
|
|
8313
9459
|
documentId: syncOp.documentId,
|
|
8314
9460
|
scopes: syncOp.scopes,
|
|
8315
9461
|
branch: syncOp.branch,
|
|
8316
|
-
operations: syncOp.operations,
|
|
9462
|
+
operations: carriesMarker(syncOp) ? syncOp.operations.filter((op) => isPurgeMarker$1(op)) : syncOp.operations,
|
|
8317
9463
|
errorSource: syncOp.error?.source ?? "none",
|
|
8318
9464
|
errorMessage: syncOp.error?.error.message ?? "unknown",
|
|
8319
9465
|
errorType
|
|
8320
9466
|
};
|
|
8321
9467
|
this.deadLetterStorage.add(record).catch((err) => {
|
|
9468
|
+
if (DocumentPurgedError.isError(err)) {
|
|
9469
|
+
this.tombstone(record.documentId);
|
|
9470
|
+
return;
|
|
9471
|
+
}
|
|
8322
9472
|
this.logger.error("Failed to persist dead letter (@id, @error)", record.id, err instanceof Error ? err.message : String(err));
|
|
8323
9473
|
});
|
|
8324
9474
|
this.eventBus.emit(SyncEventTypes.DEAD_LETTER_ADDED, {
|
|
@@ -8365,6 +9515,7 @@ var SyncManager = class {
|
|
|
8365
9515
|
}
|
|
8366
9516
|
async processCompleteBatch(batch) {
|
|
8367
9517
|
if (this.isShutdown) return;
|
|
9518
|
+
for (const { event } of batch.entries) for (const op of event.operations) if (isPurgeMarker$1(op)) this.tombstone(op.context.documentId);
|
|
8368
9519
|
const collectionIds = [...new Set(Object.values(batch.collectionMemberships).flatMap((collections) => collections))];
|
|
8369
9520
|
const affectedRemotes = [];
|
|
8370
9521
|
for (const collectionId of collectionIds) {
|
|
@@ -8386,6 +9537,7 @@ var SyncManager = class {
|
|
|
8386
9537
|
async deriveSettled() {
|
|
8387
9538
|
if (this.isShutdown) return;
|
|
8388
9539
|
const through = this.watermark.settledThrough;
|
|
9540
|
+
await this.oweSettledRange(through);
|
|
8389
9541
|
for (const [name, upTo] of [...this.owed]) {
|
|
8390
9542
|
const remote = this.remotes.get(name);
|
|
8391
9543
|
if (!remote || this.removing.has(name)) {
|
|
@@ -8397,6 +9549,27 @@ var SyncManager = class {
|
|
|
8397
9549
|
}
|
|
8398
9550
|
await this.drainPrunes();
|
|
8399
9551
|
}
|
|
9552
|
+
/** Owes the remotes of every collection that moved, event or not. */
|
|
9553
|
+
async oweSettledRange(through) {
|
|
9554
|
+
if (through <= this.sweptThrough) return;
|
|
9555
|
+
const among = [...new Set([...this.remotes.values()].filter((remote) => !this.removing.has(remote.meta.name) && (this.derivedThrough.get(remote.meta.name) ?? -1) < through).map((remote) => remote.meta.collectionId.key))];
|
|
9556
|
+
if (among.length === 0) {
|
|
9557
|
+
this.sweptThrough = through;
|
|
9558
|
+
return;
|
|
9559
|
+
}
|
|
9560
|
+
let collectionIds;
|
|
9561
|
+
try {
|
|
9562
|
+
collectionIds = await this.operationIndex.getCollectionsInRange(this.sweptThrough, through, among, this.abortController.signal);
|
|
9563
|
+
} catch (error) {
|
|
9564
|
+
this.logger.warn("Settled range sweep failed; the next advance retries it: @error", error);
|
|
9565
|
+
return;
|
|
9566
|
+
}
|
|
9567
|
+
for (const collectionId of collectionIds) for (const remote of this.getRemotesForCollection(collectionId)) {
|
|
9568
|
+
const name = remote.meta.name;
|
|
9569
|
+
if ((this.derivedThrough.get(name) ?? -1) < through) this.owe(name, through);
|
|
9570
|
+
}
|
|
9571
|
+
this.sweptThrough = through;
|
|
9572
|
+
}
|
|
8400
9573
|
owe(name, upTo) {
|
|
8401
9574
|
this.owed.set(name, Math.max(this.owed.get(name) ?? 0, upTo));
|
|
8402
9575
|
}
|
|
@@ -8411,7 +9584,30 @@ var SyncManager = class {
|
|
|
8411
9584
|
}
|
|
8412
9585
|
handleInboxAdded(remote, syncOps) {
|
|
8413
9586
|
if (this.isShutdown) return;
|
|
8414
|
-
const eligible =
|
|
9587
|
+
const eligible = [];
|
|
9588
|
+
const dropped = [];
|
|
9589
|
+
const received = this.receivedMarkersOf(remote.meta.name);
|
|
9590
|
+
const loading = (id, syncOp) => {
|
|
9591
|
+
const item = received.get(id);
|
|
9592
|
+
return item !== void 0 && item !== syncOp && item.status !== 2 && remote.channel.inbox.get(item.id) === item;
|
|
9593
|
+
};
|
|
9594
|
+
for (const syncOp of syncOps) if (carriesMarker(syncOp)) {
|
|
9595
|
+
const ids = markerIdsOf(syncOp);
|
|
9596
|
+
if (ids.every((id) => loading(id, syncOp))) {
|
|
9597
|
+
this.storeReceivedMarker(remote, syncOp);
|
|
9598
|
+
dropped.push(syncOp);
|
|
9599
|
+
} else {
|
|
9600
|
+
for (const id of ids) received.set(id, syncOp);
|
|
9601
|
+
this.storeReceivedMarker(remote, syncOp);
|
|
9602
|
+
eligible.push(syncOp);
|
|
9603
|
+
}
|
|
9604
|
+
} else if (this.purgedDocumentIds.has(syncOp.documentId)) dropped.push(syncOp);
|
|
9605
|
+
else if (!this.quarantinedDocumentIds.has(syncOp.documentId)) eligible.push(syncOp);
|
|
9606
|
+
for (const syncOp of dropped) {
|
|
9607
|
+
this.logger.debug("Dropping received operations of a purged or already-loading document (@remote, @documentId)", remote.meta.name, syncOp.documentId);
|
|
9608
|
+
syncOp.executed();
|
|
9609
|
+
}
|
|
9610
|
+
if (dropped.length > 0) remote.channel.inbox.remove(...dropped);
|
|
8415
9611
|
if (eligible.length === 0) return;
|
|
8416
9612
|
const keyed = [];
|
|
8417
9613
|
const nonKeyed = [];
|
|
@@ -8426,6 +9622,79 @@ var SyncManager = class {
|
|
|
8426
9622
|
this.processInboxChunks(chunks);
|
|
8427
9623
|
}
|
|
8428
9624
|
}
|
|
9625
|
+
/** Forgets a marker once the item loading it leaves the inbox. */
|
|
9626
|
+
handleInboxRemoved(remote, syncOps) {
|
|
9627
|
+
const received = this.receivedMarkers.get(remote.meta.name);
|
|
9628
|
+
if (received === void 0 || received.size === 0) return;
|
|
9629
|
+
const name = remote.meta.name;
|
|
9630
|
+
for (const syncOp of syncOps) for (const id of markerIdsOf(syncOp)) {
|
|
9631
|
+
if (received.get(id) !== syncOp) continue;
|
|
9632
|
+
received.delete(id);
|
|
9633
|
+
this.writeMarker(name, id, () => this.markerStorage.remove(name, id));
|
|
9634
|
+
}
|
|
9635
|
+
}
|
|
9636
|
+
markerWritesOf(remoteNames) {
|
|
9637
|
+
if (remoteNames === void 0) return [...this.markerWrites.values()];
|
|
9638
|
+
const prefixes = [...remoteNames].map((name) => `${name}\u0000`);
|
|
9639
|
+
return [...this.markerWrites].filter(([key]) => prefixes.some((prefix) => key.startsWith(prefix))).map(([, write]) => write);
|
|
9640
|
+
}
|
|
9641
|
+
/** Kept until its outcome, so a restart does not ack past it unapplied. */
|
|
9642
|
+
storeReceivedMarker(remote, syncOp) {
|
|
9643
|
+
if (this.restoredMarkers.has(syncOp)) return;
|
|
9644
|
+
if (this.purgedDocumentIds.has(syncOp.documentId)) return;
|
|
9645
|
+
const name = remote.meta.name;
|
|
9646
|
+
for (const operation of syncOp.operations) {
|
|
9647
|
+
if (!isPurgeMarker$1(operation)) continue;
|
|
9648
|
+
const markerId = operation.operation.id;
|
|
9649
|
+
this.writeMarker(name, markerId, () => this.markerStorage.upsert({
|
|
9650
|
+
remoteName: name,
|
|
9651
|
+
markerId,
|
|
9652
|
+
documentId: syncOp.documentId,
|
|
9653
|
+
branch: syncOp.branch,
|
|
9654
|
+
operation,
|
|
9655
|
+
receivedAtUtcMs: Date.now()
|
|
9656
|
+
}));
|
|
9657
|
+
}
|
|
9658
|
+
}
|
|
9659
|
+
writeMarker(remoteName, markerId, write) {
|
|
9660
|
+
const key = `${remoteName}\u0000${markerId}`;
|
|
9661
|
+
const next = (this.markerWrites.get(key) ?? Promise.resolve()).catch(() => void 0).then(write);
|
|
9662
|
+
this.markerWrites.set(key, next);
|
|
9663
|
+
const settle = () => {
|
|
9664
|
+
if (this.markerWrites.get(key) === next) this.markerWrites.delete(key);
|
|
9665
|
+
};
|
|
9666
|
+
next.then(settle, (error) => {
|
|
9667
|
+
this.logger.error("Failed to store received marker (@remote, @markerId, @error)", remoteName, markerId, error instanceof Error ? error.message : String(error));
|
|
9668
|
+
settle();
|
|
9669
|
+
});
|
|
9670
|
+
}
|
|
9671
|
+
/** Queued before init resets latestOrdinal, so a puller is re-served above it. */
|
|
9672
|
+
async restoreReceivedMarkers(remote) {
|
|
9673
|
+
const name = remote.meta.name;
|
|
9674
|
+
let records;
|
|
9675
|
+
try {
|
|
9676
|
+
records = await this.markerStorage.list(name);
|
|
9677
|
+
} catch (error) {
|
|
9678
|
+
this.logger.error("Failed to load received markers for remote (@name, @error)", name, error instanceof Error ? error.message : String(error));
|
|
9679
|
+
return;
|
|
9680
|
+
}
|
|
9681
|
+
if (records.length === 0) return;
|
|
9682
|
+
const syncOps = records.map((record) => {
|
|
9683
|
+
const syncOp = new SyncOperation(crypto.randomUUID(), "", [], name, record.documentId, [record.operation.context.scope], record.branch, [record.operation]);
|
|
9684
|
+
syncOp.transported();
|
|
9685
|
+
this.restoredMarkers.add(syncOp);
|
|
9686
|
+
return syncOp;
|
|
9687
|
+
});
|
|
9688
|
+
remote.channel.inbox.add(...syncOps);
|
|
9689
|
+
}
|
|
9690
|
+
receivedMarkersOf(name) {
|
|
9691
|
+
let received = this.receivedMarkers.get(name);
|
|
9692
|
+
if (received === void 0) {
|
|
9693
|
+
received = /* @__PURE__ */ new Map();
|
|
9694
|
+
this.receivedMarkers.set(name, received);
|
|
9695
|
+
}
|
|
9696
|
+
return received;
|
|
9697
|
+
}
|
|
8429
9698
|
processInboxChunks(chunks) {
|
|
8430
9699
|
const next = this.inboxChunkChain.then(async () => {
|
|
8431
9700
|
for (const chunk of chunks) {
|
|
@@ -8448,12 +9717,17 @@ var SyncManager = class {
|
|
|
8448
9717
|
if (this.isShutdown) return;
|
|
8449
9718
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
8450
9719
|
this.logger.error("Failed to load operations from inbox (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, err.message);
|
|
9720
|
+
if (carriesMarker(syncOp)) {
|
|
9721
|
+
this.retryMarker(remote, syncOp, err.message);
|
|
9722
|
+
return;
|
|
9723
|
+
}
|
|
8451
9724
|
const channelError = new ChannelError("inbox", err);
|
|
8452
9725
|
syncOp.failed(channelError);
|
|
8453
9726
|
remote.channel.deadLetter.add(syncOp);
|
|
8454
9727
|
remote.channel.inbox.remove(syncOp);
|
|
8455
9728
|
return;
|
|
8456
9729
|
}
|
|
9730
|
+
if (syncOp.jobId) this.recordPlanKeyMapping(syncOp.jobId, jobInfo.id);
|
|
8457
9731
|
let completedJobInfo;
|
|
8458
9732
|
try {
|
|
8459
9733
|
completedJobInfo = await this.awaiter.waitForJob(jobInfo.id, this.abortController.signal);
|
|
@@ -8461,6 +9735,10 @@ var SyncManager = class {
|
|
|
8461
9735
|
if (this.isShutdown) return;
|
|
8462
9736
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
8463
9737
|
this.logger.error("Failed to wait for job completion (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, jobInfo.id, err.message);
|
|
9738
|
+
if (carriesMarker(syncOp)) {
|
|
9739
|
+
this.retryMarker(remote, syncOp, err.message);
|
|
9740
|
+
return;
|
|
9741
|
+
}
|
|
8464
9742
|
const channelError = new ChannelError("inbox", err);
|
|
8465
9743
|
syncOp.failed(channelError);
|
|
8466
9744
|
remote.channel.deadLetter.add(syncOp);
|
|
@@ -8468,14 +9746,57 @@ var SyncManager = class {
|
|
|
8468
9746
|
return;
|
|
8469
9747
|
}
|
|
8470
9748
|
if (this.isShutdown) return;
|
|
8471
|
-
if (completedJobInfo.status
|
|
9749
|
+
if (completedJobInfo.status !== "FAILED") {
|
|
9750
|
+
syncOp.executed();
|
|
9751
|
+
if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
|
|
9752
|
+
} else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
|
|
9753
|
+
this.tombstone(syncOp.documentId);
|
|
9754
|
+
syncOp.executed();
|
|
9755
|
+
} else {
|
|
8472
9756
|
const errorMessage = completedJobInfo.error?.message || "Unknown error";
|
|
8473
9757
|
this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
|
|
8474
|
-
syncOp
|
|
9758
|
+
if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
|
|
9759
|
+
this.retryMarker(remote, syncOp, errorMessage);
|
|
9760
|
+
return;
|
|
9761
|
+
}
|
|
9762
|
+
syncOp.failed(this.inboxFailure(syncOp, completedJobInfo.error));
|
|
8475
9763
|
remote.channel.deadLetter.add(syncOp);
|
|
8476
|
-
}
|
|
9764
|
+
}
|
|
9765
|
+
this.markerRetries.delete(syncOp.id);
|
|
8477
9766
|
remote.channel.inbox.remove(syncOp);
|
|
8478
9767
|
}
|
|
9768
|
+
/** Reloads a marker with backoff; it stays in the inbox, not dead-lettered. */
|
|
9769
|
+
retryMarker(remote, syncOp, reason) {
|
|
9770
|
+
if (this.isShutdown) return;
|
|
9771
|
+
const attempt = (this.markerRetries.get(syncOp.id)?.attempt ?? 0) + 1;
|
|
9772
|
+
const delay = calculateBackoffDelay(attempt, this.config.markerRetryBaseDelayMs, this.config.markerRetryMaxDelayMs, Math.random());
|
|
9773
|
+
this.logger.warn("Purge marker load failed; retrying (@remote, @documentId, @attempt, @delayMs, @error)", remote.meta.name, syncOp.documentId, attempt, Math.round(delay), reason);
|
|
9774
|
+
const timer = setTimeout(() => {
|
|
9775
|
+
this.redeliverMarker(remote, syncOp).catch((error) => {
|
|
9776
|
+
this.logger.error("Purge marker retry failed (@remote, @documentId, @error)", remote.meta.name, syncOp.documentId, error instanceof Error ? error.message : String(error));
|
|
9777
|
+
});
|
|
9778
|
+
}, delay);
|
|
9779
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
9780
|
+
this.markerRetries.set(syncOp.id, {
|
|
9781
|
+
remoteName: remote.meta.name,
|
|
9782
|
+
attempt,
|
|
9783
|
+
timer
|
|
9784
|
+
});
|
|
9785
|
+
}
|
|
9786
|
+
async redeliverMarker(remote, syncOp) {
|
|
9787
|
+
const name = remote.meta.name;
|
|
9788
|
+
if (this.isShutdown || this.remotes.get(name) !== remote || this.removing.has(name) || remote.channel.inbox.get(syncOp.id) !== syncOp) {
|
|
9789
|
+
this.markerRetries.delete(syncOp.id);
|
|
9790
|
+
return;
|
|
9791
|
+
}
|
|
9792
|
+
if (this.purgedDocumentIds.has(syncOp.documentId)) {
|
|
9793
|
+
this.markerRetries.delete(syncOp.id);
|
|
9794
|
+
syncOp.executed();
|
|
9795
|
+
remote.channel.inbox.remove(syncOp);
|
|
9796
|
+
return;
|
|
9797
|
+
}
|
|
9798
|
+
await this.applyInboxJob(remote, syncOp);
|
|
9799
|
+
}
|
|
8479
9800
|
async applyInboxBatch(received) {
|
|
8480
9801
|
const refused = /* @__PURE__ */ new Set();
|
|
8481
9802
|
for (const { remote, syncOp } of received) {
|
|
@@ -8518,6 +9839,10 @@ var SyncManager = class {
|
|
|
8518
9839
|
if (this.isShutdown) return;
|
|
8519
9840
|
for (const { remote, syncOp } of items) {
|
|
8520
9841
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
9842
|
+
if (carriesMarker(syncOp)) {
|
|
9843
|
+
this.retryMarker(remote, syncOp, err.message);
|
|
9844
|
+
continue;
|
|
9845
|
+
}
|
|
8521
9846
|
syncOp.failed(new ChannelError("inbox", err));
|
|
8522
9847
|
remote.channel.deadLetter.add(syncOp);
|
|
8523
9848
|
remote.channel.inbox.remove(syncOp);
|
|
@@ -8548,16 +9873,29 @@ var SyncManager = class {
|
|
|
8548
9873
|
} catch (error) {
|
|
8549
9874
|
if (this.isShutdown) continue;
|
|
8550
9875
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
9876
|
+
if (carriesMarker(syncOp)) {
|
|
9877
|
+
this.retryMarker(remote, syncOp, err.message);
|
|
9878
|
+
continue;
|
|
9879
|
+
}
|
|
8551
9880
|
syncOp.failed(new ChannelError("inbox", err));
|
|
8552
9881
|
remote.channel.deadLetter.add(syncOp);
|
|
8553
9882
|
remote.channel.inbox.remove(syncOp);
|
|
8554
9883
|
continue;
|
|
8555
9884
|
}
|
|
8556
9885
|
if (this.isShutdown) return;
|
|
8557
|
-
if (completedJobInfo.status
|
|
8558
|
-
syncOp.
|
|
9886
|
+
if (completedJobInfo.status !== "FAILED") {
|
|
9887
|
+
syncOp.executed();
|
|
9888
|
+
if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
|
|
9889
|
+
} else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
|
|
9890
|
+
this.tombstone(syncOp.documentId);
|
|
9891
|
+
syncOp.executed();
|
|
9892
|
+
} else if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
|
|
9893
|
+
this.retryMarker(remote, syncOp, completedJobInfo.error?.message || "Unknown error");
|
|
9894
|
+
continue;
|
|
9895
|
+
} else {
|
|
9896
|
+
syncOp.failed(this.inboxFailure(syncOp, completedJobInfo.error));
|
|
8559
9897
|
remote.channel.deadLetter.add(syncOp);
|
|
8560
|
-
}
|
|
9898
|
+
}
|
|
8561
9899
|
remote.channel.inbox.remove(syncOp);
|
|
8562
9900
|
}
|
|
8563
9901
|
}
|
|
@@ -8571,9 +9909,9 @@ var SyncManager = class {
|
|
|
8571
9909
|
* keep syncing, because reconciling the two policies needs the traffic a
|
|
8572
9910
|
* quarantine would stop.
|
|
8573
9911
|
*/
|
|
8574
|
-
inboxFailure(error) {
|
|
9912
|
+
inboxFailure(syncOp, error) {
|
|
8575
9913
|
const message = error?.message || "Unknown error";
|
|
8576
|
-
return new ChannelError("inbox", /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), classifyJobFailure(error?.name ?? "Error"));
|
|
9914
|
+
return new ChannelError("inbox", /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), carriesMarker(syncOp) ? "MARKER_REFUSED" : classifyJobFailure(error?.name ?? "Error"));
|
|
8577
9915
|
}
|
|
8578
9916
|
/**
|
|
8579
9917
|
* Where the next outbox derivation starts: at the evicted floor when there is
|
|
@@ -8705,6 +10043,7 @@ var SyncManager = class {
|
|
|
8705
10043
|
}
|
|
8706
10044
|
}
|
|
8707
10045
|
emitBatches(remote, operations, mode, chain) {
|
|
10046
|
+
operations = operations.filter((op) => isPurgeMarker$1(op) || !this.purgedDocumentIds.has(op.context.documentId));
|
|
8708
10047
|
if (operations.length === 0) return;
|
|
8709
10048
|
const batches = batchOperationsByDocument(operations);
|
|
8710
10049
|
const syncOps = [];
|
|
@@ -8725,13 +10064,14 @@ var SyncManager = class {
|
|
|
8725
10064
|
async deriveOutbox(remote, ackOrdinal, mode, signal) {
|
|
8726
10065
|
const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
|
|
8727
10066
|
const startOrdinal = this.refillOrdinal(remote, ackOrdinal);
|
|
10067
|
+
const throughOrdinal = this.watermark.settledThrough;
|
|
8728
10068
|
let maxOrdinal = startOrdinal;
|
|
8729
10069
|
const chain = { lastJobByDoc: /* @__PURE__ */ new Map() };
|
|
8730
10070
|
const sinceTimestamp = remote.meta.options.sinceTimestampUtcMs;
|
|
8731
10071
|
const emitBatches = (operations) => this.emitBatches(remote, operations, mode, chain);
|
|
8732
10072
|
let page = await this.operationIndex.find(remote.meta.collectionId.key, startOrdinal, {
|
|
8733
10073
|
excludeSourceRemote: remote.meta.name,
|
|
8734
|
-
throughOrdinal
|
|
10074
|
+
throughOrdinal
|
|
8735
10075
|
}, void 0, composedSignal);
|
|
8736
10076
|
let carry = [];
|
|
8737
10077
|
let hasMore;
|
|
@@ -8743,9 +10083,11 @@ var SyncManager = class {
|
|
|
8743
10083
|
operations = [...carry, ...operations];
|
|
8744
10084
|
carry = [];
|
|
8745
10085
|
}
|
|
8746
|
-
|
|
8747
|
-
operations =
|
|
8748
|
-
operations = operations.filter((op) =>
|
|
10086
|
+
for (const op of operations) if (isPurgeMarker$1(op)) this.tombstone(op.context.documentId);
|
|
10087
|
+
operations = operations.filter((op) => isPurgeMarker$1(op) || !this.purgedDocumentIds.has(op.context.documentId));
|
|
10088
|
+
if (sinceTimestamp && sinceTimestamp !== "0") operations = operations.filter((op) => isPurgeMarker$1(op) || op.operation.timestampUtcMs >= sinceTimestamp);
|
|
10089
|
+
operations = filterForRemote(operations, remote.meta.filter);
|
|
10090
|
+
operations = operations.filter((op) => isPurgeMarker$1(op) || !this.quarantinedDocumentIds.has(op.context.documentId));
|
|
8749
10091
|
if (this.gates(remote)) operations = await this.gateOutbound(remote, operations);
|
|
8750
10092
|
hasMore = !!page.next;
|
|
8751
10093
|
if (operations.length > 0) {
|
|
@@ -8772,6 +10114,9 @@ var SyncManager = class {
|
|
|
8772
10114
|
} while (hasMore);
|
|
8773
10115
|
if (carry.length > 0) emitBatches(carry);
|
|
8774
10116
|
remote.channel.outbox.advanceOrdinal(maxOrdinal);
|
|
10117
|
+
const name = remote.meta.name;
|
|
10118
|
+
if (this.evictedOutboxFloors.has(name)) this.derivedThrough.delete(name);
|
|
10119
|
+
else this.derivedThrough.set(name, Math.max(this.derivedThrough.get(name) ?? -1, throughOrdinal));
|
|
8775
10120
|
}
|
|
8776
10121
|
};
|
|
8777
10122
|
//#endregion
|
|
@@ -8782,6 +10127,8 @@ var SyncBuilder = class {
|
|
|
8782
10127
|
cursorStorage;
|
|
8783
10128
|
deadLetterStorage;
|
|
8784
10129
|
holdStorage;
|
|
10130
|
+
receivedMarkerStorage;
|
|
10131
|
+
purgeRefusalStorage;
|
|
8785
10132
|
config = {};
|
|
8786
10133
|
withChannelFactory(factory) {
|
|
8787
10134
|
this.channelFactory = factory;
|
|
@@ -8803,6 +10150,14 @@ var SyncBuilder = class {
|
|
|
8803
10150
|
this.holdStorage = storage;
|
|
8804
10151
|
return this;
|
|
8805
10152
|
}
|
|
10153
|
+
withReceivedMarkerStorage(storage) {
|
|
10154
|
+
this.receivedMarkerStorage = storage;
|
|
10155
|
+
return this;
|
|
10156
|
+
}
|
|
10157
|
+
withPurgeRefusalStorage(storage) {
|
|
10158
|
+
this.purgeRefusalStorage = storage;
|
|
10159
|
+
return this;
|
|
10160
|
+
}
|
|
8806
10161
|
withMaxDeadLettersPerRemote(limit) {
|
|
8807
10162
|
this.config.maxDeadLettersPerRemote = limit;
|
|
8808
10163
|
return this;
|
|
@@ -8828,7 +10183,9 @@ var SyncBuilder = class {
|
|
|
8828
10183
|
const cursorStorage = this.cursorStorage ?? new KyselySyncCursorStorage(db);
|
|
8829
10184
|
const deadLetterStorage = this.deadLetterStorage ?? new KyselySyncDeadLetterStorage(db);
|
|
8830
10185
|
const holdStorage = this.holdStorage ?? new KyselySyncHoldStorage(db);
|
|
8831
|
-
const
|
|
10186
|
+
const receivedMarkerStorage = this.receivedMarkerStorage ?? new KyselySyncReceivedMarkerStorage(db);
|
|
10187
|
+
const purgeRefusalStorage = this.purgeRefusalStorage ?? new KyselySyncPurgeRefusalStorage(db);
|
|
10188
|
+
const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, watermark, this.config, localPeer, holdStorage, { listPurged: () => listPurged(db) }, receivedMarkerStorage, { at: (documentId, ordinal) => deliveryAt(db, documentId, ordinal) }, purgeRefusalStorage);
|
|
8832
10189
|
return {
|
|
8833
10190
|
remoteStorage,
|
|
8834
10191
|
cursorStorage,
|
|
@@ -8893,6 +10250,8 @@ var AbortError = class extends Error {
|
|
|
8893
10250
|
* for the new Reactor architecture.
|
|
8894
10251
|
*/
|
|
8895
10252
|
var Reactor = class {
|
|
10253
|
+
disposers;
|
|
10254
|
+
closers;
|
|
8896
10255
|
logger;
|
|
8897
10256
|
documentModelRegistry;
|
|
8898
10257
|
shutdownStatus;
|
|
@@ -8908,7 +10267,9 @@ var Reactor = class {
|
|
|
8908
10267
|
eventBus;
|
|
8909
10268
|
executorManager;
|
|
8910
10269
|
catchUp;
|
|
8911
|
-
constructor(logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp) {
|
|
10270
|
+
constructor(logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp, disposers = [], closers = []) {
|
|
10271
|
+
this.disposers = disposers;
|
|
10272
|
+
this.closers = closers;
|
|
8912
10273
|
this.catchUp = catchUp;
|
|
8913
10274
|
this.logger = logger;
|
|
8914
10275
|
this.documentModelRegistry = documentModelRegistry;
|
|
@@ -8936,9 +10297,15 @@ var Reactor = class {
|
|
|
8936
10297
|
this.setShutdown(true);
|
|
8937
10298
|
const shutdownAsync = async () => {
|
|
8938
10299
|
await this.executorManager.stop(true);
|
|
10300
|
+
for (const dispose of this.disposers) dispose();
|
|
8939
10301
|
await this.catchUp?.stop();
|
|
8940
10302
|
this.readModelCoordinator.stop();
|
|
8941
10303
|
this.jobTracker.shutdown();
|
|
10304
|
+
for (const close of this.closers) try {
|
|
10305
|
+
await close();
|
|
10306
|
+
} catch (error) {
|
|
10307
|
+
this.logger.error("Shutdown step failed: @Error", error);
|
|
10308
|
+
}
|
|
8942
10309
|
};
|
|
8943
10310
|
this.setCompleted(shutdownAsync());
|
|
8944
10311
|
return this.shutdownStatus;
|
|
@@ -9823,6 +11190,13 @@ var ReactorBuilder = class {
|
|
|
9823
11190
|
});
|
|
9824
11191
|
const documentMetaCache = new DocumentMetaCache(operationStore, { maxDocuments: 1e3 });
|
|
9825
11192
|
await documentMetaCache.startup();
|
|
11193
|
+
const unsubscribeMarkerEviction = eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, (_type, event) => {
|
|
11194
|
+
for (const item of event.operations) {
|
|
11195
|
+
if (!isPurgeMarker$1(item)) continue;
|
|
11196
|
+
writeCache.invalidate(item.context.documentId);
|
|
11197
|
+
documentMetaCache.invalidate(item.context.documentId);
|
|
11198
|
+
}
|
|
11199
|
+
});
|
|
9826
11200
|
const collectionMembershipCache = new CollectionMembershipCache(operationIndex);
|
|
9827
11201
|
const executionScope = new KyselyExecutionScope(database, operationStore, operationIndex, keyframeStore, writeCache, documentMetaCache, collectionMembershipCache);
|
|
9828
11202
|
let executorManager = this.executorManager;
|
|
@@ -9918,7 +11292,7 @@ var ReactorBuilder = class {
|
|
|
9918
11292
|
const indexedReadModels = readModelCoordinator.indexedReadModels?.bind(readModelCoordinator);
|
|
9919
11293
|
if (indexedReadModels) catchUp.addSource(() => indexedReadModels().filter((model) => isCatchUpConsumer(model)), "host");
|
|
9920
11294
|
else this.logger.warn("The read model coordinator does not report indexedReadModels; the host runs no read-side catch-up sweep");
|
|
9921
|
-
const reactor = new Reactor(this.logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, this.features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp);
|
|
11295
|
+
const reactor = new Reactor(this.logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, this.features, documentView, documentIndexer, operationStore, eventBus, executorManager, catchUp, [unsubscribeMarkerEviction], [() => processorManager.shutdown()]);
|
|
9922
11296
|
const localPeer = {
|
|
9923
11297
|
capabilities: this.getPeerCapabilities(),
|
|
9924
11298
|
flags: featureFlags,
|
|
@@ -9929,6 +11303,9 @@ var ReactorBuilder = class {
|
|
|
9929
11303
|
} catch {
|
|
9930
11304
|
return;
|
|
9931
11305
|
}
|
|
11306
|
+
},
|
|
11307
|
+
forgetDocument: (documentId) => {
|
|
11308
|
+
documentMetaCache.invalidate(documentId);
|
|
9932
11309
|
}
|
|
9933
11310
|
};
|
|
9934
11311
|
let syncModule = void 0;
|
|
@@ -9973,7 +11350,8 @@ var ReactorBuilder = class {
|
|
|
9973
11350
|
pools: this.instrumentedPools,
|
|
9974
11351
|
degradedComponents,
|
|
9975
11352
|
catchUp,
|
|
9976
|
-
settledWatermark
|
|
11353
|
+
settledWatermark,
|
|
11354
|
+
documentPurgeService: new DocumentPurgeService(queue, jobTracker, eventBus)
|
|
9977
11355
|
};
|
|
9978
11356
|
catchUp.start();
|
|
9979
11357
|
if (degradedComponents.length > 0) this.logger.warn("Reactor started with @count degraded component(s): @components", degradedComponents.length, degradedComponents.map(({ component }) => component).join(", "));
|
|
@@ -10040,7 +11418,7 @@ var ReactorBuilder = class {
|
|
|
10040
11418
|
poolInstrumentations.push(forwarder);
|
|
10041
11419
|
this.instrumentedPools.push(forwarder);
|
|
10042
11420
|
}
|
|
10043
|
-
const { ProjectionShardManager } = await import("./projection-shard-manager-
|
|
11421
|
+
const { ProjectionShardManager } = await import("./projection-shard-manager-BSK-GEyP.js");
|
|
10044
11422
|
const manager = new ProjectionShardManager({
|
|
10045
11423
|
shardCount: config.shardCount,
|
|
10046
11424
|
db,
|
|
@@ -10085,7 +11463,7 @@ var ReactorBuilder = class {
|
|
|
10085
11463
|
*/
|
|
10086
11464
|
async createDefaultWorkerFactory(numWorkers, db) {
|
|
10087
11465
|
const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
|
|
10088
|
-
import("./worker-handle-
|
|
11466
|
+
import("./worker-handle-weNGCZme.js"),
|
|
10089
11467
|
import("./transport-CZDHzfvy.js"),
|
|
10090
11468
|
Promise.resolve().then(() => worker_exports)
|
|
10091
11469
|
]);
|
|
@@ -10834,6 +12212,6 @@ var DocumentIntegrityService = class {
|
|
|
10834
12212
|
}
|
|
10835
12213
|
};
|
|
10836
12214
|
//#endregion
|
|
10837
|
-
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
|
|
12215
|
+
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isPurgeMarker, isRecoverableGraphQLError, listPurged, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
|
|
10838
12216
|
|
|
10839
12217
|
//# sourceMappingURL=index.js.map
|