@powerhousedao/reactor 6.2.2-dev.9 → 6.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{build-worker-executor--nhFRF47.js → build-worker-executor-DnU3ZU4H.js} +2 -2
- package/dist/{build-worker-executor--nhFRF47.js.map → build-worker-executor-DnU3ZU4H.js.map} +1 -1
- package/dist/{document-indexer-FGJmRAdX.js → document-indexer-C5Gsa1B3.js} +69 -28
- package/dist/document-indexer-C5Gsa1B3.js.map +1 -0
- package/dist/drive-container-types-yZrksiJR.js +5310 -0
- package/dist/drive-container-types-yZrksiJR.js.map +1 -0
- package/dist/entry.js +3 -2
- package/dist/entry.js.map +1 -1
- package/dist/index.d.ts +3176 -1661
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1534 -296
- package/dist/index.js.map +1 -1
- package/dist/projection-entry.js +4 -4
- package/dist/projection-entry.js.map +1 -1
- package/dist/{projection-shard-manager-BnQKFqQd.js → projection-shard-manager-D6KYcEZi.js} +89 -4
- package/dist/projection-shard-manager-D6KYcEZi.js.map +1 -0
- package/dist/{worker-DBJOv8Gp.js → worker-HPysBfhx.js} +2 -2
- package/dist/{worker-DBJOv8Gp.js.map → worker-HPysBfhx.js.map} +1 -1
- package/dist/{worker-handle-B1w03nRA.js → worker-handle-CrERzl8s.js} +3 -2
- package/dist/worker-handle-CrERzl8s.js.map +1 -0
- package/package.json +22 -5
- package/dist/document-indexer-FGJmRAdX.js.map +0 -1
- package/dist/drive-container-types-DpJp2AmE.js +0 -2984
- package/dist/drive-container-types-DpJp2AmE.js.map +0 -1
- package/dist/projection-shard-manager-BnQKFqQd.js.map +0 -1
- package/dist/worker-handle-B1w03nRA.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as selectDecisionModel, B as RevisionMismatchError, C as InvalidModuleError, D as createEmptyConsistencyToken, E as createConsistencyToken, F as AppendConditionFailedError, G as DocumentNotFoundError, H as AuthTimestampNotMonotonicError, I as DocumentAlreadyExistsError, J as UpgradePreconditionFailedError, K as ExcessiveReshuffleError, L as DocumentExistence, M as authDecisionModel, N as buildDecisionModel, O as targetDocumentId, P as APPEND_CONDITION_FAILED_PREFIX, R as DuplicateOperationError, S as DuplicateModuleError, T as GATED_DOCUMENT_ACTIONS, U as AuthorizationDeniedError, V as AuthEnforcementDisabledError, W as DocumentDeletedError, X as parsePagingOptions, Y as matchesScope, Z as throwIfAborted, _ as DocumentMetaCache, a as createForwardingPoolInstrumentation, b as JobExecutorEventTypes, c as KyselyKeyframeStore, d as DriveCollectionId, f as KyselyExecutionScope, g as KyselyOperationIndex, h as KyselyWriteCache, i as runMigrations, j as documentDecisionModel, k as decideAtHead, l as DocumentModelRegistry, m as EventBus, n as REACTOR_SCHEMA, o as instrumentPgPool, p as resolveFeatureFlags, q as InvalidOperationTimestampError, r as getMigrationStatus, s as KyselyOperationStore, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as SimpleJobExecutor, v as CollectionMembershipCache, w as ModuleNotFoundError, x as DuplicateManifestError, y as DEFAULT_DEFERRED_JOB_TTL_MS, z as OptimisticLockError } from "./drive-container-types-yZrksiJR.js";
|
|
2
2
|
import { n as ReactorEventTypes, t as EventBusAggregateError } from "./types-DMKLa0Ok.js";
|
|
3
3
|
import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-D3S6Eysd.js";
|
|
4
|
-
import { a as
|
|
4
|
+
import { a as DOCUMENT_INDEXER_READ_MODEL, c as BaseReadModel, i as KyselyDocumentView, n as ConsistencyTracker, o as DOCUMENT_VIEW_READ_MODEL, r as makeConsistencyKey, s as ReadModelCoordinator, t as KyselyDocumentIndexer } from "./document-indexer-C5Gsa1B3.js";
|
|
5
5
|
import { n as errorToInfo, r as sanitizeArg, t as createForwardingLogger } from "./forwarding-logger-BBkMSxuJ.js";
|
|
6
|
-
import { t as workerEntryPath } from "./worker-
|
|
7
|
-
import { actions, actions as documentActions, createPresignedHeader, generateId, hashDocumentStateForScope,
|
|
6
|
+
import { t as workerEntryPath } from "./worker-HPysBfhx.js";
|
|
7
|
+
import { DowngradeNotSupportedError, MAX_AUTH_GRANTS, UnsupportedDocumentModelVersionError, actions, actions as documentActions, createPresignedHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, normalizeDocumentModelVersion, sortOperations, toTransportAction } from "@powerhousedao/shared/document-model";
|
|
8
8
|
import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
|
|
9
9
|
import { v4 } from "uuid";
|
|
10
10
|
import { ConsoleLogger, childLogger } from "document-model";
|
|
@@ -190,10 +190,12 @@ function topologicalSort(jobs) {
|
|
|
190
190
|
*/
|
|
191
191
|
function toErrorInfo$1(error) {
|
|
192
192
|
if (error instanceof Error) return {
|
|
193
|
+
name: error.name,
|
|
193
194
|
message: error.message,
|
|
194
195
|
stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
|
|
195
196
|
};
|
|
196
197
|
return {
|
|
198
|
+
name: "Error",
|
|
197
199
|
message: error,
|
|
198
200
|
stack: (/* @__PURE__ */ new Error()).stack || ""
|
|
199
201
|
};
|
|
@@ -363,7 +365,7 @@ var DriveClient = class {
|
|
|
363
365
|
documentId: document.header.id,
|
|
364
366
|
model: document.header.documentType,
|
|
365
367
|
fromVersion: 0,
|
|
366
|
-
toVersion:
|
|
368
|
+
toVersion: normalizeDocumentModelVersion(document.state.document?.version),
|
|
367
369
|
initialState: document.state
|
|
368
370
|
}),
|
|
369
371
|
addRelationshipAction(driveIdentifier, documentId, "child")
|
|
@@ -374,24 +376,23 @@ var DriveClient = class {
|
|
|
374
376
|
documentType: document.header.documentType,
|
|
375
377
|
parentFolder
|
|
376
378
|
})], this.signer, signal);
|
|
377
|
-
|
|
379
|
+
await this.runJobs([{
|
|
378
380
|
key: "document",
|
|
379
381
|
documentId,
|
|
380
382
|
scope: getSharedActionScope(documentActions),
|
|
381
383
|
branch: "main",
|
|
382
384
|
actions: documentActions,
|
|
383
385
|
dependsOn: []
|
|
384
|
-
},
|
|
386
|
+
}], signal);
|
|
387
|
+
await this.runJobs([{
|
|
385
388
|
key: "drive",
|
|
386
389
|
documentId: driveIdentifier,
|
|
387
390
|
scope: getSharedActionScope(driveActions),
|
|
388
391
|
branch: "main",
|
|
389
392
|
actions: driveActions,
|
|
390
|
-
dependsOn: [
|
|
391
|
-
}]
|
|
392
|
-
|
|
393
|
-
for (const job of completedJobs) if (job.status === JobStatus.FAILED) throw new Error(job.error?.message);
|
|
394
|
-
return this.reactor.get(documentId);
|
|
393
|
+
dependsOn: []
|
|
394
|
+
}], signal);
|
|
395
|
+
return this.client.get(documentId, void 0, signal);
|
|
395
396
|
}
|
|
396
397
|
async addFolder(driveIdentifier, name, parentFolder, signal) {
|
|
397
398
|
this.logger.verbose("drives.addFolder(@driveIdentifier, @name, @parentFolder)", driveIdentifier, name, parentFolder);
|
|
@@ -400,7 +401,7 @@ var DriveClient = class {
|
|
|
400
401
|
id: folderId,
|
|
401
402
|
name,
|
|
402
403
|
parentFolder
|
|
403
|
-
})], signal)).state.global
|
|
404
|
+
})], signal)).state.global?.nodes.find((n) => n.id === folderId);
|
|
404
405
|
if (!node || !isFolderNode(node)) throw new Error("Folder creation failed");
|
|
405
406
|
return node;
|
|
406
407
|
}
|
|
@@ -427,7 +428,7 @@ var DriveClient = class {
|
|
|
427
428
|
const node = (await this.client.execute(driveIdentifier, "main", [updateNode({
|
|
428
429
|
id: nodeId,
|
|
429
430
|
name
|
|
430
|
-
})], signal)).state.global
|
|
431
|
+
})], signal)).state.global?.nodes.find((n) => n.id === nodeId);
|
|
431
432
|
if (!node) throw new Error("Node missing from drive after rename");
|
|
432
433
|
return node;
|
|
433
434
|
}
|
|
@@ -468,8 +469,12 @@ var DriveClient = class {
|
|
|
468
469
|
const node = drive.state.global.nodes.find((n) => n.id === entry.srcId);
|
|
469
470
|
if (!node || !isFileNode(node)) continue;
|
|
470
471
|
const srcDoc = await this.client.get(entry.srcId, void 0, signal);
|
|
471
|
-
const
|
|
472
|
-
|
|
472
|
+
const duplicated = {
|
|
473
|
+
...srcDoc,
|
|
474
|
+
header: createPresignedHeader(entry.targetId, srcDoc.header.documentType),
|
|
475
|
+
initialState: srcDoc.state,
|
|
476
|
+
operations: {}
|
|
477
|
+
};
|
|
473
478
|
const resolvedName = resolvedNamesByTargetId.get(entry.targetId);
|
|
474
479
|
if (resolvedName) duplicated.header.name = resolvedName;
|
|
475
480
|
await this.addFile(driveIdentifier, duplicated, entry.targetParentFolder ?? void 0, signal);
|
|
@@ -499,6 +504,22 @@ var DriveClient = class {
|
|
|
499
504
|
totalCount: filtered.length
|
|
500
505
|
};
|
|
501
506
|
}
|
|
507
|
+
/**
|
|
508
|
+
* Runs one batch to completion, throwing the first failure. The thrown error
|
|
509
|
+
* carries the job error's name, which is what callers classify on: an id
|
|
510
|
+
* collision has to be told apart from a transient failure.
|
|
511
|
+
*/
|
|
512
|
+
async runJobs(jobs, signal) {
|
|
513
|
+
const batchResult = await this.reactor.executeBatch({ jobs }, signal);
|
|
514
|
+
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.client.waitForJob(job, signal)));
|
|
515
|
+
for (const job of completedJobs) if (job.status === JobStatus.FAILED) throw this.toError(job);
|
|
516
|
+
return completedJobs;
|
|
517
|
+
}
|
|
518
|
+
toError(job) {
|
|
519
|
+
const error = new Error(job.error?.message ?? "Job failed");
|
|
520
|
+
if (job.error?.name) error.name = job.error.name;
|
|
521
|
+
return error;
|
|
522
|
+
}
|
|
502
523
|
async documentExists(documentId, signal) {
|
|
503
524
|
try {
|
|
504
525
|
await this.client.get(documentId, void 0, signal);
|
|
@@ -680,8 +701,335 @@ let DocumentChangeType = /* @__PURE__ */ function(DocumentChangeType) {
|
|
|
680
701
|
return DocumentChangeType;
|
|
681
702
|
}({});
|
|
682
703
|
//#endregion
|
|
704
|
+
//#region src/decision/read-gate.ts
|
|
705
|
+
/**
|
|
706
|
+
* Scopes every holder of a document may read, whatever the grants say. Denying
|
|
707
|
+
* the policy itself would let a replica sync a document without it, read the
|
|
708
|
+
* auth scope as uninitialized, and allow every operation it holds, so replicas
|
|
709
|
+
* would diverge permanently. The document scope carries the metadata the same
|
|
710
|
+
* argument covers. Grants gate domain-scope reads only.
|
|
711
|
+
*/
|
|
712
|
+
const ALWAYS_READABLE_SCOPES = new Set(["auth", "document"]);
|
|
713
|
+
/** What a gate does with an uninitialized policy when told nothing. */
|
|
714
|
+
const OPEN_BY_DEFAULT = { withholdUninitialized: false };
|
|
715
|
+
/**
|
|
716
|
+
* The branch a group is read on. A group's member list lives on its main
|
|
717
|
+
* branch whatever branch the document naming it is on, and the reference
|
|
718
|
+
* relation records no branch of its own.
|
|
719
|
+
*/
|
|
720
|
+
const GROUP_BRANCH = "main";
|
|
721
|
+
/**
|
|
722
|
+
* The scope of a group document that holds its member list. It is the only
|
|
723
|
+
* scope group serving reaches, because it is the only one the audience must
|
|
724
|
+
* fold in order to evaluate auth with the group.
|
|
725
|
+
*/
|
|
726
|
+
const GROUP_MEMBERSHIP_SCOPE = "global";
|
|
727
|
+
/**
|
|
728
|
+
* How many of a group's referencing documents are examined before a read of
|
|
729
|
+
* that group gives up and withholds. Each one costs a model build, and the
|
|
730
|
+
* relation is append-only, so an old and briefly-held reference still counts.
|
|
731
|
+
* Truncating denies, which is the direction that withholds. It is the cap a
|
|
732
|
+
* version-1 policy carries on its own grant list, taken from there rather than
|
|
733
|
+
* restated: raising that cap raises the read fan-out here.
|
|
734
|
+
*/
|
|
735
|
+
const MAX_EXAMINED_REFERENCERS = MAX_AUTH_GRANTS;
|
|
736
|
+
/**
|
|
737
|
+
* How many referencers are probed at once. The reactor's Postgres pool size is
|
|
738
|
+
* optional and defaults to ten, and a listing resolves one gate per result, so
|
|
739
|
+
* per-gate fan-out multiplies: a handful keeps one group read from claiming the
|
|
740
|
+
* whole pool while still cutting the walk several times over. On PGlite the
|
|
741
|
+
* queries serialize regardless, so a larger number buys nothing in a browser.
|
|
742
|
+
*/
|
|
743
|
+
const REFERENCER_PROBE_CONCURRENCY = 4;
|
|
744
|
+
/**
|
|
745
|
+
* The policy carried on a document, if it carries one. A document handed to the
|
|
746
|
+
* gate with no state at all is not policied, so it is not gated.
|
|
747
|
+
*/
|
|
748
|
+
function authOf(document) {
|
|
749
|
+
return document.state?.auth;
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* The model reads enforce. Below `authEnforcement` there is no model to
|
|
753
|
+
* enforce: the document-only model ignores the auth scope entirely, so reading
|
|
754
|
+
* through it would serve every domain scope of a policied document to anyone.
|
|
755
|
+
* Undefined therefore means "evaluate the policy alone", which is what the read
|
|
756
|
+
* surface did before the model existed.
|
|
757
|
+
*/
|
|
758
|
+
function readDecisionModel(flags, registry) {
|
|
759
|
+
return flags.authEnforcement ? selectDecisionModel(flags, registry) : void 0;
|
|
760
|
+
}
|
|
761
|
+
/**
|
|
762
|
+
* Evaluates the policy on its own, with no groups map and no condition context.
|
|
763
|
+
* A `{ group }` or conditional grant therefore never applies: an allow that
|
|
764
|
+
* does not apply withholds access, so this cannot widen a policy, but a policy
|
|
765
|
+
* relying on a conditional deny is weaker here than it is written.
|
|
766
|
+
*/
|
|
767
|
+
var BareReadGate = class {
|
|
768
|
+
scopePredicate(document, subject) {
|
|
769
|
+
const auth = authOf(document);
|
|
770
|
+
return Promise.resolve((scope) => ALWAYS_READABLE_SCOPES.has(scope) || decide(auth, subject, {
|
|
771
|
+
verb: "read",
|
|
772
|
+
scope
|
|
773
|
+
}) === "allow");
|
|
774
|
+
}
|
|
775
|
+
};
|
|
776
|
+
/**
|
|
777
|
+
* Answers a stream read from the document already fetched, and anything else
|
|
778
|
+
* through the read side.
|
|
779
|
+
*
|
|
780
|
+
* The seed is why routing reads through a decision model costs no extra I/O for
|
|
781
|
+
* the document being read: its `document` and `auth` scopes are the two static
|
|
782
|
+
* projections, and the caller has both in hand. Only a group stream the grant
|
|
783
|
+
* list names is fetched.
|
|
784
|
+
*/
|
|
785
|
+
var SeededStateReader = class {
|
|
786
|
+
constructor(documentView, seed, branch) {
|
|
787
|
+
this.documentView = documentView;
|
|
788
|
+
this.seed = seed;
|
|
789
|
+
this.branch = branch;
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* A stream this replica does not hold has to reach buildDecisionModel as the
|
|
793
|
+
* absence it recognises, or the whole read fails instead of leaving the group
|
|
794
|
+
* out of the model, where its principal does not match and the policy fails
|
|
795
|
+
* closed. The read side reports absence as a plain Error, so the absence is
|
|
796
|
+
* confirmed rather than inferred from the message: a transient failure must
|
|
797
|
+
* surface, not silently deny.
|
|
798
|
+
*/
|
|
799
|
+
async getState(documentId, scope, branch, targetRevision, signal) {
|
|
800
|
+
if (targetRevision === void 0 && documentId === this.seed.header.id && branch === this.branch && scope in this.seed.state) return this.seed;
|
|
801
|
+
try {
|
|
802
|
+
return await this.documentView.get(documentId, {
|
|
803
|
+
branch,
|
|
804
|
+
scopes: [scope]
|
|
805
|
+
}, void 0, signal);
|
|
806
|
+
} catch (error) {
|
|
807
|
+
await this.assertAbsent(documentId, error, signal);
|
|
808
|
+
throw new DocumentNotFoundError(documentId);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
async assertAbsent(documentId, error, signal) {
|
|
812
|
+
if (DocumentNotFoundError.isError(error)) return;
|
|
813
|
+
let exists;
|
|
814
|
+
try {
|
|
815
|
+
exists = await this.documentView.exists([documentId], DocumentExistence.LiveOnly, void 0, signal);
|
|
816
|
+
} catch {
|
|
817
|
+
throw error;
|
|
818
|
+
}
|
|
819
|
+
if (exists[0]) throw error;
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
/**
|
|
823
|
+
* Evaluates a read against the registered decision model, built at the stream
|
|
824
|
+
* heads. This is what makes `{ group }` principals and conditional grants apply
|
|
825
|
+
* to a read: the model supplies the groups map and the scope's own state, the
|
|
826
|
+
* same two things admission supplies.
|
|
827
|
+
*
|
|
828
|
+
* A read has no action, so a condition on `action.input.*` never holds for one.
|
|
829
|
+
*
|
|
830
|
+
* State is read through the read side rather than the write cache. The write
|
|
831
|
+
* cache is invalidated by whichever process runs the executor, so a reactor
|
|
832
|
+
* running its executors in worker processes would answer reads in the parent
|
|
833
|
+
* from state no commit ever invalidates.
|
|
834
|
+
*/
|
|
835
|
+
var ModelReadGate = class {
|
|
836
|
+
constructor(model, documentView, servesGroups, operationIndex, logger, options = OPEN_BY_DEFAULT) {
|
|
837
|
+
this.model = model;
|
|
838
|
+
this.documentView = documentView;
|
|
839
|
+
this.servesGroups = servesGroups;
|
|
840
|
+
this.operationIndex = operationIndex;
|
|
841
|
+
this.logger = logger;
|
|
842
|
+
this.options = options;
|
|
843
|
+
}
|
|
844
|
+
/**
|
|
845
|
+
* A served group yields its member list and nothing else. What the audience
|
|
846
|
+
* is owed is the state it must fold to evaluate auth with the group; a
|
|
847
|
+
* group's other scopes are its own business and stay behind its own grants.
|
|
848
|
+
*/
|
|
849
|
+
async scopePredicate(document, subject, branch, signal) {
|
|
850
|
+
const own = await this.ownPolicyPredicate(document, subject, branch, signal);
|
|
851
|
+
if (!this.servesGroup(document) || own(GROUP_MEMBERSHIP_SCOPE)) return own;
|
|
852
|
+
const audience = await this.servesGroupTo(document.header.id, subject, signal);
|
|
853
|
+
return (scope) => own(scope) || audience && scope === GROUP_MEMBERSHIP_SCOPE;
|
|
854
|
+
}
|
|
855
|
+
/**
|
|
856
|
+
* Whether the subject is served this group because a policy names it.
|
|
857
|
+
*
|
|
858
|
+
* A replica must fold a group's membership to evaluate auth with it, so a
|
|
859
|
+
* group a grant names is served to the audience of the document that names
|
|
860
|
+
* it, whatever the group's own read grants say. Naming a group in a policy
|
|
861
|
+
* publishes its roster to that policy's audience; a group whose membership
|
|
862
|
+
* must stay confidential does not belong in a grant.
|
|
863
|
+
*
|
|
864
|
+
* The referencing document's own domain scopes are the test. Its `auth` and
|
|
865
|
+
* `document` scopes are readable by every holder, so testing those would
|
|
866
|
+
* serve every referenced group to everybody.
|
|
867
|
+
*
|
|
868
|
+
* One level only. A referencer that is itself a group is skipped, and a
|
|
869
|
+
* referencer's own readability is decided from its policy alone, so a
|
|
870
|
+
* reference cycle terminates. Cycles are reachable: the reference relation
|
|
871
|
+
* is recorded from an operation's input, including one later stored denied,
|
|
872
|
+
* so a refused grant naming a group from inside another group leaves a row
|
|
873
|
+
* behind that validation never saw.
|
|
874
|
+
*
|
|
875
|
+
* The referencers are probed a few at a time and the walk stops at the first
|
|
876
|
+
* that serves, because a subject outside the audience is the case that runs to
|
|
877
|
+
* the bound, and it is the common one. A probe that failed decides only when
|
|
878
|
+
* nothing served: serving rests on a real allow, so this cannot widen, and it
|
|
879
|
+
* stops one unreachable referencer from turning an allow already in hand into
|
|
880
|
+
* a denial. A read records no operation, so replicas differing over a
|
|
881
|
+
* transient failure has no consensus consequence.
|
|
882
|
+
*
|
|
883
|
+
* The probes are awaited together rather than raced, so none is ever left
|
|
884
|
+
* running with nobody awaiting it, which is where unhandled rejections come
|
|
885
|
+
* from.
|
|
886
|
+
*/
|
|
887
|
+
async servesGroupTo(groupId, subject, signal) {
|
|
888
|
+
if (!this.operationIndex) return false;
|
|
889
|
+
const referencers = await this.operationIndex.getGroupReferencers(groupId, signal);
|
|
890
|
+
const examined = referencers.slice(0, MAX_EXAMINED_REFERENCERS);
|
|
891
|
+
if (examined.length < referencers.length) this.logger?.warn(`Group ${groupId} is referenced by ${referencers.length} documents; only the first ${MAX_EXAMINED_REFERENCERS} decide whether it is served`);
|
|
892
|
+
const walk = {
|
|
893
|
+
next: 0,
|
|
894
|
+
served: false,
|
|
895
|
+
failure: void 0
|
|
896
|
+
};
|
|
897
|
+
const probe = async () => {
|
|
898
|
+
while (!walk.served && walk.next < examined.length) {
|
|
899
|
+
const referencerId = examined[walk.next++];
|
|
900
|
+
let serves = false;
|
|
901
|
+
try {
|
|
902
|
+
serves = await this.servesThrough(referencerId, subject, signal);
|
|
903
|
+
} catch (error) {
|
|
904
|
+
walk.failure ??= Error.isError(error) ? error : /* @__PURE__ */ new Error(`Probing referencer ${referencerId} failed`);
|
|
905
|
+
}
|
|
906
|
+
if (serves) walk.served = true;
|
|
907
|
+
}
|
|
908
|
+
};
|
|
909
|
+
await Promise.all(Array.from({ length: Math.min(REFERENCER_PROBE_CONCURRENCY, examined.length) }, probe));
|
|
910
|
+
if (!walk.served && walk.failure) throw walk.failure;
|
|
911
|
+
return walk.served;
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* Whether one referencing document serves the subject any domain scope. A
|
|
915
|
+
* referencer this replica does not hold serves nothing, which fails closed
|
|
916
|
+
* the same way a group it does not hold does.
|
|
917
|
+
*/
|
|
918
|
+
async servesThrough(referencerId, subject, signal) {
|
|
919
|
+
let referencer;
|
|
920
|
+
try {
|
|
921
|
+
referencer = await this.documentView.get(referencerId, { branch: GROUP_BRANCH }, void 0, signal);
|
|
922
|
+
} catch {
|
|
923
|
+
return false;
|
|
924
|
+
}
|
|
925
|
+
if (this.servesGroup(referencer)) return false;
|
|
926
|
+
const readable = await this.ownPolicyPredicate(referencer, subject, GROUP_BRANCH, signal);
|
|
927
|
+
return Object.keys(referencer.state ?? {}).filter((scope) => !ALWAYS_READABLE_SCOPES.has(scope)).some((scope) => readable(scope));
|
|
928
|
+
}
|
|
929
|
+
servesGroup(document) {
|
|
930
|
+
return this.servesGroups && this.operationIndex !== void 0 && document.header.documentType === groupDocumentType;
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* What this document's own policy says, with no group serving applied.
|
|
934
|
+
*
|
|
935
|
+
* An unpoliced document is readable in full unless the host closes by default,
|
|
936
|
+
* and either way it is the common case and the one worth answering without
|
|
937
|
+
* building anything. The test is the one `evaluate` makes: a legacy `{}` auth
|
|
938
|
+
* scope and version 0 both mean uninitialized, and "no grants" does not,
|
|
939
|
+
* because a policy with a version and an empty grant list denies everything.
|
|
940
|
+
*
|
|
941
|
+
* Closing here rather than around the gate is what keeps a policy able to
|
|
942
|
+
* publish an unpoliced group it names: the referencer walk asks this question
|
|
943
|
+
* of the referencing document, whose real policy answers it.
|
|
944
|
+
*/
|
|
945
|
+
async ownPolicyPredicate(document, subject, branch, signal) {
|
|
946
|
+
const auth = authOf(document);
|
|
947
|
+
if (!auth || !auth.version) return this.options.withholdUninitialized ? (scope) => ALWAYS_READABLE_SCOPES.has(scope) : () => true;
|
|
948
|
+
const target = {
|
|
949
|
+
documentId: document.header.id,
|
|
950
|
+
branch
|
|
951
|
+
};
|
|
952
|
+
const built = await buildDecisionModel(new SeededStateReader(this.documentView, document, branch), this.model, target, signal);
|
|
953
|
+
const definition = this.model(target);
|
|
954
|
+
const scopeStates = document.state ?? {};
|
|
955
|
+
return (scope) => ALWAYS_READABLE_SCOPES.has(scope) || definition.decide(built.model, subject, {
|
|
956
|
+
verb: "read",
|
|
957
|
+
scope
|
|
958
|
+
}, {
|
|
959
|
+
scopeState: scopeStates[scope],
|
|
960
|
+
actionInput: void 0
|
|
961
|
+
}).decision === "allow";
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
//#endregion
|
|
965
|
+
//#region src/client/util.ts
|
|
966
|
+
function authSubjectFromSigner(signer) {
|
|
967
|
+
return {
|
|
968
|
+
address: signer.user?.address,
|
|
969
|
+
key: signer.app?.key
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
function withAuthScope(view) {
|
|
973
|
+
if (view?.scopes && view.scopes.length > 0) return {
|
|
974
|
+
...view,
|
|
975
|
+
scopes: [...new Set([...view.scopes, "auth"])]
|
|
976
|
+
};
|
|
977
|
+
return view;
|
|
978
|
+
}
|
|
979
|
+
/**
|
|
980
|
+
* Drops the scopes the predicate refuses. The predicate is resolved by the read
|
|
981
|
+
* gate, which decides against the whole policy once per document; this only
|
|
982
|
+
* applies the answer, so it stays synchronous.
|
|
983
|
+
*/
|
|
984
|
+
function filterReadableScopes(document, readable) {
|
|
985
|
+
const state = document.state;
|
|
986
|
+
if (!state) return document;
|
|
987
|
+
return {
|
|
988
|
+
...document,
|
|
989
|
+
state: keepReadableScopes(state, readable),
|
|
990
|
+
initialState: keepReadableScopes(document.initialState, readable)
|
|
991
|
+
};
|
|
992
|
+
}
|
|
993
|
+
function keepReadableScopes(scopes, readable) {
|
|
994
|
+
if (!scopes) return scopes;
|
|
995
|
+
const kept = {};
|
|
996
|
+
for (const scope of Object.keys(scopes)) if (readable(scope)) kept[scope] = scopes[scope];
|
|
997
|
+
return kept;
|
|
998
|
+
}
|
|
999
|
+
//#endregion
|
|
683
1000
|
//#region src/client/reactor-client.ts
|
|
684
1001
|
/**
|
|
1002
|
+
* The document a candidate is decided against. Routed on the action type alone,
|
|
1003
|
+
* which is what the executor routes on: every action the reactor reduces onto
|
|
1004
|
+
* the document scope carries that scope already, and the gate follows the
|
|
1005
|
+
* action's own target rather than the document the request named.
|
|
1006
|
+
*/
|
|
1007
|
+
function evaluationTargetId(candidate, fallback) {
|
|
1008
|
+
return GATED_DOCUMENT_ACTIONS.has(candidate.type) ? targetDocumentId({
|
|
1009
|
+
type: candidate.type,
|
|
1010
|
+
input: candidate.input
|
|
1011
|
+
}, fallback) : fallback;
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* One candidate's verdict, decided exactly as admission decides it: the same
|
|
1015
|
+
* request, and a condition context populated only while authConditions is on,
|
|
1016
|
+
* so a conditional grant applies here precisely when it would apply there.
|
|
1017
|
+
* Groups need nothing added, because the selected model carries them.
|
|
1018
|
+
*/
|
|
1019
|
+
function decideCandidate(config, target, subject, candidate) {
|
|
1020
|
+
return target.definition.decide(target.model, subject, {
|
|
1021
|
+
verb: "execute",
|
|
1022
|
+
scope: candidate.scope,
|
|
1023
|
+
operation: candidate.type
|
|
1024
|
+
}, config.flags.authConditions ? {
|
|
1025
|
+
scopeState: target.scopeStates[candidate.scope],
|
|
1026
|
+
actionInput: candidate.input
|
|
1027
|
+
} : {
|
|
1028
|
+
scopeState: void 0,
|
|
1029
|
+
actionInput: void 0
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
/**
|
|
685
1033
|
* ReactorClient implementation that wraps lower-level APIs to provide
|
|
686
1034
|
* a simpler interface for document operations.
|
|
687
1035
|
*
|
|
@@ -699,8 +1047,10 @@ var ReactorClient = class {
|
|
|
699
1047
|
jobAwaiter;
|
|
700
1048
|
documentIndexer;
|
|
701
1049
|
documentView;
|
|
1050
|
+
readGate;
|
|
1051
|
+
actionEvaluation;
|
|
702
1052
|
drives;
|
|
703
|
-
constructor(logger, reactor, signer, subscriptionManager, jobAwaiter, documentIndexer, documentView) {
|
|
1053
|
+
constructor(logger, reactor, signer, subscriptionManager, jobAwaiter, documentIndexer, documentView, readGate = new BareReadGate(), actionEvaluation) {
|
|
704
1054
|
this.logger = logger;
|
|
705
1055
|
this.reactor = reactor;
|
|
706
1056
|
this.signer = signer;
|
|
@@ -708,9 +1058,33 @@ var ReactorClient = class {
|
|
|
708
1058
|
this.jobAwaiter = jobAwaiter;
|
|
709
1059
|
this.documentIndexer = documentIndexer;
|
|
710
1060
|
this.documentView = documentView;
|
|
1061
|
+
this.readGate = readGate;
|
|
1062
|
+
this.actionEvaluation = actionEvaluation;
|
|
711
1063
|
this.drives = new DriveClient(this, logger, reactor, signer);
|
|
712
1064
|
this.logger.verbose("ReactorClient initialized");
|
|
713
1065
|
}
|
|
1066
|
+
readSubject(subject) {
|
|
1067
|
+
return subject ?? authSubjectFromSigner(this.signer);
|
|
1068
|
+
}
|
|
1069
|
+
/**
|
|
1070
|
+
* Which scopes of one document the subject may read. Resolved once per
|
|
1071
|
+
* document, so the gate builds its model once however many scopes are then
|
|
1072
|
+
* tested, and the filtering itself stays synchronous.
|
|
1073
|
+
*/
|
|
1074
|
+
readableScopes(document, view, signal) {
|
|
1075
|
+
return this.readGate.scopePredicate(document, this.readSubject(view?.subject), view?.branch ?? "main", signal);
|
|
1076
|
+
}
|
|
1077
|
+
/**
|
|
1078
|
+
* One document, filtered to the scopes the subject may read. Every method
|
|
1079
|
+
* that hands a document back goes through here, including the ones that
|
|
1080
|
+
* follow a write: a document returned from a mutation is a read like any
|
|
1081
|
+
* other, and returning it whole served scopes the same subject would be
|
|
1082
|
+
* refused by `get`. Its author still sees what it wrote, because an allow on
|
|
1083
|
+
* execute confers read of that scope.
|
|
1084
|
+
*/
|
|
1085
|
+
async gateDocument(document, view, signal) {
|
|
1086
|
+
return filterReadableScopes(document, await this.readableScopes(document, view, signal));
|
|
1087
|
+
}
|
|
714
1088
|
/**
|
|
715
1089
|
* Retrieves a list of document model modules.
|
|
716
1090
|
*/
|
|
@@ -725,24 +1099,58 @@ var ReactorClient = class {
|
|
|
725
1099
|
* @returns The document model module
|
|
726
1100
|
*/
|
|
727
1101
|
async getDocumentModelModule(documentType) {
|
|
728
|
-
const
|
|
729
|
-
|
|
730
|
-
|
|
1102
|
+
const modules = await this.reactor.getDocumentModels();
|
|
1103
|
+
let latestModule;
|
|
1104
|
+
let latestVersion = -1;
|
|
1105
|
+
for (const module of modules.results) {
|
|
1106
|
+
if (module.documentModel.global.id !== documentType) continue;
|
|
1107
|
+
const version = module.version ?? 1;
|
|
1108
|
+
if (version > latestVersion) {
|
|
1109
|
+
latestVersion = version;
|
|
1110
|
+
latestModule = module;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
if (!latestModule) throw new Error(`Document model module not found for type: ${documentType}`);
|
|
1114
|
+
return latestModule;
|
|
1115
|
+
}
|
|
1116
|
+
/**
|
|
1117
|
+
* Retrieves the document model module matching the version the document is
|
|
1118
|
+
* stamped with, so not-yet-upgraded documents get the reducer their
|
|
1119
|
+
* history was written with rather than the latest.
|
|
1120
|
+
*/
|
|
1121
|
+
async getDocumentModelModuleForDocument(document) {
|
|
1122
|
+
const documentType = document.header.documentType;
|
|
1123
|
+
const version = normalizeDocumentModelVersion(document.state.document?.version);
|
|
1124
|
+
const modules = await this.reactor.getDocumentModels();
|
|
1125
|
+
const availableVersions = [];
|
|
1126
|
+
for (const module of modules.results) {
|
|
1127
|
+
if (module.documentModel.global.id !== documentType) continue;
|
|
1128
|
+
const moduleVersion = normalizeDocumentModelVersion(module.version);
|
|
1129
|
+
if (moduleVersion === version) return module;
|
|
1130
|
+
availableVersions.push(moduleVersion);
|
|
1131
|
+
}
|
|
1132
|
+
throw new UnsupportedDocumentModelVersionError(documentType, version, availableVersions.sort((a, b) => a - b));
|
|
731
1133
|
}
|
|
732
1134
|
/**
|
|
733
1135
|
* Retrieves a specific PHDocument
|
|
734
1136
|
*/
|
|
735
1137
|
async get(identifier, view, signal) {
|
|
736
1138
|
this.logger.verbose("get(@identifier, @view)", identifier, view);
|
|
737
|
-
|
|
1139
|
+
const document = await this.reactor.getByIdOrSlug(identifier, withAuthScope(view), void 0, signal);
|
|
1140
|
+
return this.gateDocument(document, view, signal);
|
|
738
1141
|
}
|
|
739
1142
|
/**
|
|
740
1143
|
* Resolves an identifier (id or slug) to the canonical document id, using the
|
|
741
1144
|
* same lookup as the data path. Resolves against the "main" branch. Throws if
|
|
742
1145
|
* the identifier cannot be resolved or is ambiguous.
|
|
743
1146
|
*/
|
|
744
|
-
async resolveIdOrSlug(identifier, signal) {
|
|
745
|
-
return this.documentView.resolveIdOrSlug(identifier,
|
|
1147
|
+
async resolveIdOrSlug(identifier, view, signal) {
|
|
1148
|
+
return this.documentView.resolveIdOrSlug(identifier, view, void 0, signal);
|
|
1149
|
+
}
|
|
1150
|
+
/** True when the id is taken; a soft-deleted document's id is still taken. */
|
|
1151
|
+
async isDocumentIdTaken(documentId, signal) {
|
|
1152
|
+
this.logger.verbose("isDocumentIdTaken(@documentId)", documentId);
|
|
1153
|
+
return (await this.documentView.exists([documentId], DocumentExistence.IncludingDeleted, void 0, signal))[0];
|
|
746
1154
|
}
|
|
747
1155
|
/**
|
|
748
1156
|
* Retrieves operations for a document
|
|
@@ -750,8 +1158,11 @@ var ReactorClient = class {
|
|
|
750
1158
|
async getOperations(documentIdentifier, view, filter, paging, signal) {
|
|
751
1159
|
this.logger.verbose("getOperations(@documentIdentifier, @view, @filter, @paging)", documentIdentifier, view, filter, paging);
|
|
752
1160
|
const documentId = await this.documentView.resolveIdOrSlug(documentIdentifier, view, void 0, signal);
|
|
753
|
-
|
|
1161
|
+
const gated = await this.reactor.getByIdOrSlug(documentId, withAuthScope(view), void 0, signal);
|
|
1162
|
+
const canRead = await this.readableScopes(gated, view, signal);
|
|
1163
|
+
if (paging?.cursor && isCompositeCursor(paging.cursor)) return this.getOperationsWithCompositeCursor(documentId, view, filter, paging, signal, canRead);
|
|
754
1164
|
const operationsByScope = await this.reactor.getOperations(documentId, view, filter, paging, void 0, signal);
|
|
1165
|
+
for (const scope of Object.keys(operationsByScope)) if (!canRead(scope)) delete operationsByScope[scope];
|
|
755
1166
|
const scopeEntries = Object.entries(operationsByScope);
|
|
756
1167
|
const effectivePaging = paging || {
|
|
757
1168
|
cursor: "0",
|
|
@@ -779,11 +1190,12 @@ var ReactorClient = class {
|
|
|
779
1190
|
nextCursor: Object.keys(activeCursors).length > 0 ? encodeCompositeCursor(activeCursors) : void 0
|
|
780
1191
|
};
|
|
781
1192
|
}
|
|
782
|
-
async getOperationsWithCompositeCursor(documentId, view, filter, paging, signal) {
|
|
1193
|
+
async getOperationsWithCompositeCursor(documentId, view, filter, paging, signal, canRead) {
|
|
783
1194
|
const scopeCursors = decodeCompositeCursor(paging.cursor);
|
|
784
1195
|
const allOperations = [];
|
|
785
1196
|
const activeCursors = {};
|
|
786
1197
|
for (const [scopeName, cursor] of Object.entries(scopeCursors)) {
|
|
1198
|
+
if (!canRead(scopeName)) continue;
|
|
787
1199
|
const scopeView = {
|
|
788
1200
|
...view,
|
|
789
1201
|
scopes: [scopeName]
|
|
@@ -817,7 +1229,7 @@ var ReactorClient = class {
|
|
|
817
1229
|
limit: 0
|
|
818
1230
|
}
|
|
819
1231
|
};
|
|
820
|
-
return this.
|
|
1232
|
+
return this.find({ ids: targetIds }, view, paging, signal);
|
|
821
1233
|
}
|
|
822
1234
|
/**
|
|
823
1235
|
* Retrieves incoming relationships of a given type to a target document.
|
|
@@ -833,14 +1245,90 @@ var ReactorClient = class {
|
|
|
833
1245
|
limit: 0
|
|
834
1246
|
}
|
|
835
1247
|
};
|
|
836
|
-
return this.
|
|
1248
|
+
return this.find({ ids: sourceIds }, view, paging, signal);
|
|
837
1249
|
}
|
|
838
1250
|
/**
|
|
839
1251
|
* Filters documents by criteria and returns a list of them
|
|
840
1252
|
*/
|
|
841
1253
|
async find(search, view, paging, signal) {
|
|
842
1254
|
this.logger.verbose("find(@search, @view, @paging)", search, view, paging);
|
|
843
|
-
|
|
1255
|
+
const results = await this.reactor.find(search, withAuthScope(view), paging, void 0, signal);
|
|
1256
|
+
return {
|
|
1257
|
+
...results,
|
|
1258
|
+
results: await Promise.all(results.results.map(async (doc) => filterReadableScopes(doc, await this.readableScopes(doc, view, signal))))
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Predicts the admission verdict for each candidate. See
|
|
1263
|
+
* {@link IReactorClient.evaluateActions} for the contract and its caveats.
|
|
1264
|
+
*
|
|
1265
|
+
* Read-only throughout, and never through the write cache: that cache is
|
|
1266
|
+
* invalidated by whichever process runs the executor, so a reactor running
|
|
1267
|
+
* its executors in worker processes would answer here from state no commit
|
|
1268
|
+
* ever invalidates.
|
|
1269
|
+
*/
|
|
1270
|
+
async evaluateActions(documentIdentifier, branch, candidates, subject, signal) {
|
|
1271
|
+
this.logger.verbose("evaluateActions(@documentIdentifier, @branch, @count candidates)", documentIdentifier, branch, candidates.length);
|
|
1272
|
+
const config = this.actionEvaluation;
|
|
1273
|
+
if (config === void 0) throw new AuthEnforcementDisabledError();
|
|
1274
|
+
const decidedSubject = this.readSubject(subject);
|
|
1275
|
+
const resolvedId = await this.documentView.resolveIdOrSlug(documentIdentifier, { branch }, void 0, signal);
|
|
1276
|
+
const targets = /* @__PURE__ */ new Map();
|
|
1277
|
+
const evaluations = [];
|
|
1278
|
+
for (const candidate of candidates) {
|
|
1279
|
+
if (candidate.type === "CREATE_DOCUMENT") {
|
|
1280
|
+
evaluations.push({ decision: "allow" });
|
|
1281
|
+
continue;
|
|
1282
|
+
}
|
|
1283
|
+
const targetId = evaluationTargetId(candidate, resolvedId);
|
|
1284
|
+
let target = targets.get(targetId);
|
|
1285
|
+
if (target === void 0) {
|
|
1286
|
+
target = await this.buildEvaluationTarget(config, targetId, branch, signal);
|
|
1287
|
+
targets.set(targetId, target);
|
|
1288
|
+
}
|
|
1289
|
+
evaluations.push(decideCandidate(config, target, decidedSubject, candidate));
|
|
1290
|
+
}
|
|
1291
|
+
const allowed = evaluations.filter((evaluation) => evaluation.decision === "allow").length;
|
|
1292
|
+
return {
|
|
1293
|
+
evaluations,
|
|
1294
|
+
allAllowed: candidates.length > 0 && allowed === candidates.length,
|
|
1295
|
+
anyAllowed: allowed > 0,
|
|
1296
|
+
allDenied: candidates.length > 0 && allowed === 0,
|
|
1297
|
+
anyDenied: allowed < candidates.length
|
|
1298
|
+
};
|
|
1299
|
+
}
|
|
1300
|
+
/**
|
|
1301
|
+
* The decision model for one target document, built at its stream heads.
|
|
1302
|
+
*
|
|
1303
|
+
* The document is fetched unfiltered, because the policy is what decides:
|
|
1304
|
+
* reading it through the read gate would withhold the very scopes the
|
|
1305
|
+
* decision is about. A deleted document is served at its deletion boundary,
|
|
1306
|
+
* which is what lets the model refuse an execute against it -- authEnforcement
|
|
1307
|
+
* requires documentDecisions, so that read is available whenever this runs.
|
|
1308
|
+
*
|
|
1309
|
+
* Reading past the gate discloses nothing a submit does not. The `auth` and
|
|
1310
|
+
* `document` scopes are readable by every holder, so a verdict resting on the
|
|
1311
|
+
* policy alone is one the caller could compute unaided; and a verdict resting
|
|
1312
|
+
* on a conditional grant reads the executing scope's state exactly as
|
|
1313
|
+
* admission reads it, so the answer here is what submitting and being refused
|
|
1314
|
+
* would have revealed anyway.
|
|
1315
|
+
*
|
|
1316
|
+
* The append condition the build records is dropped. It guards a write, and
|
|
1317
|
+
* this makes none; reproducing it is also what the preflight cannot do, which
|
|
1318
|
+
* is why the answer is a prediction.
|
|
1319
|
+
*/
|
|
1320
|
+
async buildEvaluationTarget(config, documentId, branch, signal) {
|
|
1321
|
+
const document = await this.documentView.get(documentId, { branch }, void 0, signal);
|
|
1322
|
+
const target = {
|
|
1323
|
+
documentId,
|
|
1324
|
+
branch
|
|
1325
|
+
};
|
|
1326
|
+
const built = await buildDecisionModel(new SeededStateReader(this.documentView, document, branch), config.model, target, signal);
|
|
1327
|
+
return {
|
|
1328
|
+
definition: config.model(target),
|
|
1329
|
+
model: built.model,
|
|
1330
|
+
scopeStates: document.state ?? {}
|
|
1331
|
+
};
|
|
844
1332
|
}
|
|
845
1333
|
/**
|
|
846
1334
|
* Creates a document and waits for completion
|
|
@@ -868,7 +1356,7 @@ var ReactorClient = class {
|
|
|
868
1356
|
documentId,
|
|
869
1357
|
model: document.header.documentType,
|
|
870
1358
|
fromVersion: 0,
|
|
871
|
-
toVersion: document.state.document
|
|
1359
|
+
toVersion: normalizeDocumentModelVersion(document.state.document?.version),
|
|
872
1360
|
initialState: document.state
|
|
873
1361
|
})], this.signer, signal);
|
|
874
1362
|
const jobs = [{
|
|
@@ -893,7 +1381,8 @@ var ReactorClient = class {
|
|
|
893
1381
|
const batchResult = await this.reactor.executeBatch({ jobs }, signal);
|
|
894
1382
|
const completedJobs = await Promise.all(Object.values(batchResult.jobs).map((job) => this.waitForJob(job, signal)));
|
|
895
1383
|
for (const job of completedJobs) if (job.status === JobStatus.FAILED) throw new Error(job.error?.message);
|
|
896
|
-
|
|
1384
|
+
const created = await this.reactor.get(documentId);
|
|
1385
|
+
return this.gateDocument(created, void 0, signal);
|
|
897
1386
|
}
|
|
898
1387
|
/**
|
|
899
1388
|
* Creates an empty document and waits for completion
|
|
@@ -903,20 +1392,64 @@ var ReactorClient = class {
|
|
|
903
1392
|
const matchingModules = (await this.reactor.getDocumentModels(void 0, void 0, signal)).results.filter((m) => m.documentModel.global.id === documentModelType);
|
|
904
1393
|
let module;
|
|
905
1394
|
if (options?.documentModelVersion !== void 0) {
|
|
906
|
-
|
|
1395
|
+
const requestedVersion = normalizeDocumentModelVersion(options.documentModelVersion);
|
|
1396
|
+
module = matchingModules.find((m) => normalizeDocumentModelVersion(m.version) === requestedVersion);
|
|
907
1397
|
if (!module) throw new Error(`Document model not found for type: ${documentModelType} with version: ${options.documentModelVersion}`);
|
|
908
1398
|
} else {
|
|
909
1399
|
module = matchingModules.reduce((latest, current) => {
|
|
910
1400
|
if (latest === void 0) return current;
|
|
911
|
-
return (current.version
|
|
1401
|
+
return normalizeDocumentModelVersion(current.version) > normalizeDocumentModelVersion(latest.version) ? current : latest;
|
|
912
1402
|
}, void 0);
|
|
913
1403
|
if (!module) throw new Error(`Document model not found for type: ${documentModelType}`);
|
|
914
1404
|
}
|
|
915
1405
|
const document = module.utils.createDocument();
|
|
916
|
-
document.state.document.version = module.version
|
|
1406
|
+
document.state.document.version = normalizeDocumentModelVersion(module.version);
|
|
917
1407
|
return this.create(document, options?.parentIdentifier, signal);
|
|
918
1408
|
}
|
|
919
1409
|
/**
|
|
1410
|
+
* Upgrades a document to a newer document model version by dispatching an
|
|
1411
|
+
* UPGRADE_DOCUMENT action. When toVersion is omitted, upgrades to the
|
|
1412
|
+
* latest registered module version for the document's type. Returns the
|
|
1413
|
+
* document unchanged when it is already at the target version.
|
|
1414
|
+
*
|
|
1415
|
+
* The executor validates the action's version and revision snapshot against
|
|
1416
|
+
* the state the migration actually runs on. When a concurrent edit
|
|
1417
|
+
* invalidates the snapshot, the upgrade is rebuilt from a fresh read and
|
|
1418
|
+
* retried up to maxConflictRetries times before the conflict is surfaced.
|
|
1419
|
+
*/
|
|
1420
|
+
async upgradeDocument(documentIdentifier, toVersion, options, signal) {
|
|
1421
|
+
this.logger.verbose("upgradeDocument(@documentIdentifier, @toVersion)", documentIdentifier, toVersion);
|
|
1422
|
+
const maxConflictRetries = options?.maxConflictRetries ?? 3;
|
|
1423
|
+
let lastConflictMessage = "";
|
|
1424
|
+
for (let attempt = 0; attempt <= maxConflictRetries; attempt++) {
|
|
1425
|
+
const document = await this.reactor.getByIdOrSlug(documentIdentifier, void 0, void 0, signal);
|
|
1426
|
+
const documentId = document.header.id;
|
|
1427
|
+
const documentType = document.header.documentType;
|
|
1428
|
+
const branch = document.header.branch || "main";
|
|
1429
|
+
const fromVersion = normalizeDocumentModelVersion(document.state.document?.version);
|
|
1430
|
+
let targetVersion = toVersion;
|
|
1431
|
+
if (targetVersion === void 0) targetVersion = normalizeDocumentModelVersion((await this.getDocumentModelModule(documentType)).version);
|
|
1432
|
+
if (targetVersion === fromVersion) return this.gateDocument(document, { branch }, signal);
|
|
1433
|
+
if (targetVersion < fromVersion) throw new DowngradeNotSupportedError(documentType, fromVersion, targetVersion);
|
|
1434
|
+
const signedActions = await signActions([upgradeDocumentAction({
|
|
1435
|
+
documentId,
|
|
1436
|
+
model: documentType,
|
|
1437
|
+
fromVersion,
|
|
1438
|
+
toVersion: targetVersion,
|
|
1439
|
+
revision: { ...document.header.revision }
|
|
1440
|
+
})], this.signer, signal);
|
|
1441
|
+
const jobInfo = await this.reactor.execute(documentId, branch, signedActions, signal);
|
|
1442
|
+
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
1443
|
+
if (completedJob.status !== JobStatus.FAILED) {
|
|
1444
|
+
const upgraded = await this.reactor.getByIdOrSlug(documentId, { branch }, completedJob.consistencyToken, signal);
|
|
1445
|
+
return this.gateDocument(upgraded, { branch }, signal);
|
|
1446
|
+
}
|
|
1447
|
+
if (completedJob.error?.name !== "UpgradePreconditionFailedError") throw new Error(completedJob.error?.message);
|
|
1448
|
+
lastConflictMessage = completedJob.error.message;
|
|
1449
|
+
}
|
|
1450
|
+
throw new Error(`Upgrade of document ${documentIdentifier} conflicted with concurrent edits after ${maxConflictRetries + 1} attempts: ${lastConflictMessage}`);
|
|
1451
|
+
}
|
|
1452
|
+
/**
|
|
920
1453
|
* Creates an empty document in a drive as a single batched operation.
|
|
921
1454
|
* Delegates to {@link IDriveClient.addFile}.
|
|
922
1455
|
*
|
|
@@ -927,16 +1460,23 @@ var ReactorClient = class {
|
|
|
927
1460
|
return this.drives.addFile(driveId, document, parentFolder, signal);
|
|
928
1461
|
}
|
|
929
1462
|
/**
|
|
930
|
-
* Applies a list of actions to a document and waits for completion
|
|
1463
|
+
* Applies a list of actions to a document and waits for completion.
|
|
1464
|
+
*
|
|
1465
|
+
* `subject` names who the returned document is read as; see
|
|
1466
|
+
* {@link IReactorClient.execute}.
|
|
931
1467
|
*/
|
|
932
|
-
async execute(documentIdentifier, branch, actions, signal) {
|
|
1468
|
+
async execute(documentIdentifier, branch, actions, signal, subject) {
|
|
933
1469
|
this.logger.verbose("execute(@documentIdentifier, @branch, @count actions)", documentIdentifier, branch, actions.length);
|
|
934
1470
|
const signedActions = await signActions(actions, this.signer, signal);
|
|
935
1471
|
const jobInfo = await this.reactor.execute(documentIdentifier, branch, signedActions, signal);
|
|
936
1472
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
937
1473
|
if (completedJob.status === JobStatus.FAILED) throw new Error(completedJob.error?.message);
|
|
938
|
-
const view = {
|
|
939
|
-
|
|
1474
|
+
const view = {
|
|
1475
|
+
branch,
|
|
1476
|
+
subject
|
|
1477
|
+
};
|
|
1478
|
+
const result = await this.reactor.getByIdOrSlug(documentIdentifier, view, completedJob.consistencyToken, signal);
|
|
1479
|
+
return this.gateDocument(result, view, signal);
|
|
940
1480
|
}
|
|
941
1481
|
/**
|
|
942
1482
|
* Submits a list of actions to a document
|
|
@@ -980,7 +1520,8 @@ var ReactorClient = class {
|
|
|
980
1520
|
const jobInfo = await this.reactor.addRelationship(sourceIdentifier, targetIdentifier, relationshipType, branch, this.signer, signal);
|
|
981
1521
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
982
1522
|
if (completedJob.status === JobStatus.FAILED) throw new Error(completedJob.error?.message);
|
|
983
|
-
|
|
1523
|
+
const result = await this.reactor.getByIdOrSlug(sourceIdentifier, { branch }, completedJob.consistencyToken, signal);
|
|
1524
|
+
return this.gateDocument(result, { branch }, signal);
|
|
984
1525
|
}
|
|
985
1526
|
/**
|
|
986
1527
|
* Removes a relationship between two documents and waits for completion.
|
|
@@ -990,7 +1531,8 @@ var ReactorClient = class {
|
|
|
990
1531
|
const jobInfo = await this.reactor.removeRelationship(sourceIdentifier, targetIdentifier, relationshipType, branch, this.signer, signal);
|
|
991
1532
|
const completedJob = await this.waitForJob(jobInfo, signal);
|
|
992
1533
|
if (completedJob.status === JobStatus.FAILED) throw new Error(completedJob.error?.message);
|
|
993
|
-
|
|
1534
|
+
const result = await this.reactor.getByIdOrSlug(sourceIdentifier, { branch }, completedJob.consistencyToken, signal);
|
|
1535
|
+
return this.gateDocument(result, { branch }, signal);
|
|
994
1536
|
}
|
|
995
1537
|
/**
|
|
996
1538
|
* Moves a relationship from one source document to another and waits for completion.
|
|
@@ -1003,9 +1545,11 @@ var ReactorClient = class {
|
|
|
1003
1545
|
const addJobInfo = await this.reactor.addRelationship(targetParentIdentifier, targetIdentifier, relationshipType, branch, this.signer, signal);
|
|
1004
1546
|
const addCompletedJob = await this.waitForJob(addJobInfo, signal);
|
|
1005
1547
|
if (addCompletedJob.status === JobStatus.FAILED) throw new Error(addCompletedJob.error?.message);
|
|
1548
|
+
const sourceResult = await this.reactor.getByIdOrSlug(sourceParentIdentifier, { branch }, removeCompletedJob.consistencyToken, signal);
|
|
1549
|
+
const targetResult = await this.reactor.getByIdOrSlug(targetParentIdentifier, { branch }, addCompletedJob.consistencyToken, signal);
|
|
1006
1550
|
return {
|
|
1007
|
-
source: await this.
|
|
1008
|
-
target: await this.
|
|
1551
|
+
source: await this.gateDocument(sourceResult, { branch }, signal),
|
|
1552
|
+
target: await this.gateDocument(targetResult, { branch }, signal)
|
|
1009
1553
|
};
|
|
1010
1554
|
}
|
|
1011
1555
|
async loadBatch(request, signal) {
|
|
@@ -1078,32 +1622,56 @@ var ReactorClient = class {
|
|
|
1078
1622
|
*/
|
|
1079
1623
|
subscribe(search, callback, view) {
|
|
1080
1624
|
this.logger.verbose("subscribe(@search, @view)", search, view);
|
|
1625
|
+
const readable = async (document) => filterReadableScopes(document, await this.readableScopes(document, view));
|
|
1626
|
+
let disposed = false;
|
|
1627
|
+
let delivering = Promise.resolve();
|
|
1628
|
+
/**
|
|
1629
|
+
* Queues one event behind those already queued. Gating an event resolves
|
|
1630
|
+
* asynchronously, so an event needing a group fetch would otherwise be
|
|
1631
|
+
* overtaken by the one behind it, and one still in flight would reach a
|
|
1632
|
+
* subscriber that has already unsubscribed. Only delivery is ordered, not
|
|
1633
|
+
* the gating: a single subscription can cover every document in the
|
|
1634
|
+
* reactor, so serializing that work would make its delivery rate the sum
|
|
1635
|
+
* of every gate build rather than the slowest. An event that cannot be
|
|
1636
|
+
* gated is withheld, because serving it unfiltered would leak the scopes
|
|
1637
|
+
* the gate did not clear -- but it is logged rather than swallowed, since
|
|
1638
|
+
* the gate rethrows a transient failure precisely so it is not read as a
|
|
1639
|
+
* denial.
|
|
1640
|
+
*/
|
|
1641
|
+
const deliver = (event) => {
|
|
1642
|
+
Promise.resolve(event).catch(() => void 0);
|
|
1643
|
+
delivering = delivering.then(async () => {
|
|
1644
|
+
const built = await event;
|
|
1645
|
+
if (disposed) return;
|
|
1646
|
+
callback(built);
|
|
1647
|
+
}).catch((error) => {
|
|
1648
|
+
this.logger.error("Subscription delivery failed for @search: @Error", { search }, error);
|
|
1649
|
+
});
|
|
1650
|
+
};
|
|
1081
1651
|
const unsubscribeCreated = this.subscriptionManager.onDocumentCreated((result) => {
|
|
1082
|
-
(async () => {
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
} catch {}
|
|
1090
|
-
})();
|
|
1652
|
+
deliver((async () => {
|
|
1653
|
+
const documents = await Promise.all(result.results.map((id) => this.reactor.get(id, withAuthScope(view), void 0, void 0)));
|
|
1654
|
+
return {
|
|
1655
|
+
type: DocumentChangeType.Created,
|
|
1656
|
+
documents: await Promise.all(documents.map(readable))
|
|
1657
|
+
};
|
|
1658
|
+
})());
|
|
1091
1659
|
}, search);
|
|
1092
1660
|
const unsubscribeDeleted = this.subscriptionManager.onDocumentDeleted((documentIds) => {
|
|
1093
|
-
|
|
1661
|
+
deliver({
|
|
1094
1662
|
type: DocumentChangeType.Deleted,
|
|
1095
1663
|
documents: [],
|
|
1096
1664
|
context: { childId: documentIds[0] }
|
|
1097
1665
|
});
|
|
1098
1666
|
}, search);
|
|
1099
1667
|
const unsubscribeUpdated = this.subscriptionManager.onDocumentStateUpdated((result) => {
|
|
1100
|
-
|
|
1668
|
+
deliver((async () => ({
|
|
1101
1669
|
type: DocumentChangeType.Updated,
|
|
1102
|
-
documents: result.results
|
|
1103
|
-
});
|
|
1670
|
+
documents: await Promise.all(result.results.map(readable))
|
|
1671
|
+
}))());
|
|
1104
1672
|
}, search, view);
|
|
1105
1673
|
const unsubscribeRelationship = this.subscriptionManager.onRelationshipChanged((parentId, childId, changeType) => {
|
|
1106
|
-
|
|
1674
|
+
deliver({
|
|
1107
1675
|
type: changeType === RelationshipChangeType.Added ? DocumentChangeType.ChildAdded : DocumentChangeType.ChildRemoved,
|
|
1108
1676
|
documents: [],
|
|
1109
1677
|
context: {
|
|
@@ -1113,6 +1681,7 @@ var ReactorClient = class {
|
|
|
1113
1681
|
});
|
|
1114
1682
|
}, search);
|
|
1115
1683
|
return () => {
|
|
1684
|
+
disposed = true;
|
|
1116
1685
|
unsubscribeCreated();
|
|
1117
1686
|
unsubscribeDeleted();
|
|
1118
1687
|
unsubscribeUpdated();
|
|
@@ -1144,17 +1713,33 @@ let JobQueueState = /* @__PURE__ */ function(JobQueueState) {
|
|
|
1144
1713
|
return JobQueueState;
|
|
1145
1714
|
}({});
|
|
1146
1715
|
/**
|
|
1716
|
+
* How a retry is accounted against the job's retry limit.
|
|
1717
|
+
* - `CountAgainstLimit` (default): a fault; the job eventually exhausts its
|
|
1718
|
+
* retries and fails terminally.
|
|
1719
|
+
* - `ExemptFromLimit`: not a fault, so the attempt is not charged to the job.
|
|
1720
|
+
* Used for concurrency conflicts, where the retry does new work.
|
|
1721
|
+
*/
|
|
1722
|
+
let RetryAccounting = /* @__PURE__ */ function(RetryAccounting) {
|
|
1723
|
+
RetryAccounting["CountAgainstLimit"] = "count-against-limit";
|
|
1724
|
+
RetryAccounting["ExemptFromLimit"] = "exempt-from-limit";
|
|
1725
|
+
return RetryAccounting;
|
|
1726
|
+
}({});
|
|
1727
|
+
/**
|
|
1147
1728
|
* Event types for the queue system
|
|
1148
1729
|
*/
|
|
1149
1730
|
const QueueEventTypes = { JOB_AVAILABLE: 1e4 };
|
|
1150
1731
|
//#endregion
|
|
1151
1732
|
//#region src/executor/job-result-handler.ts
|
|
1733
|
+
/** Conflict retries a job may take without charging its retry limit. */
|
|
1734
|
+
const MAX_EXEMPT_CONFLICT_RETRIES = 20;
|
|
1152
1735
|
function toErrorInfo(error) {
|
|
1153
1736
|
if (error instanceof Error) return {
|
|
1737
|
+
name: error.name,
|
|
1154
1738
|
message: error.message,
|
|
1155
1739
|
stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
|
|
1156
1740
|
};
|
|
1157
1741
|
return {
|
|
1742
|
+
name: "Error",
|
|
1158
1743
|
message: error,
|
|
1159
1744
|
stack: (/* @__PURE__ */ new Error()).stack || ""
|
|
1160
1745
|
};
|
|
@@ -1187,12 +1772,19 @@ var JobResultHandler = class {
|
|
|
1187
1772
|
} catch {}
|
|
1188
1773
|
}
|
|
1189
1774
|
}
|
|
1190
|
-
if (result.error &&
|
|
1775
|
+
if (result.error && AppendConditionFailedError.isError(result.error) && this.countConflicts(handle.job) < MAX_EXEMPT_CONFLICT_RETRIES) {
|
|
1776
|
+
const errorInfo = toErrorInfo(result.error);
|
|
1777
|
+
try {
|
|
1778
|
+
await this.queue.retryJob(handle.job.id, errorInfo, RetryAccounting.ExemptFromLimit);
|
|
1779
|
+
return;
|
|
1780
|
+
} catch {}
|
|
1781
|
+
}
|
|
1782
|
+
if (result.error && DocumentNotFoundError.isError(result.error) && handle.job.kind === "load") {
|
|
1191
1783
|
handle.defer();
|
|
1192
|
-
callbacks.deferJob(
|
|
1784
|
+
callbacks.deferJob(result.error.documentId, handle.job);
|
|
1193
1785
|
return;
|
|
1194
1786
|
}
|
|
1195
|
-
if (result.error && DocumentDeletedError.isError(result.error)) {
|
|
1787
|
+
if (result.error && (DocumentDeletedError.isError(result.error) || DocumentAlreadyExistsError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error))) {
|
|
1196
1788
|
const errorInfo = toErrorInfo(result.error);
|
|
1197
1789
|
this.jobTracker.markFailed(handle.job.id, errorInfo, handle.job);
|
|
1198
1790
|
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
@@ -1230,6 +1822,12 @@ var JobResultHandler = class {
|
|
|
1230
1822
|
handle.fail(fullErrorInfo);
|
|
1231
1823
|
}
|
|
1232
1824
|
}
|
|
1825
|
+
/** How many times this job has already lost an append-condition race. */
|
|
1826
|
+
countConflicts(job) {
|
|
1827
|
+
let conflicts = 0;
|
|
1828
|
+
for (const error of job.errorHistory) if (AppendConditionFailedError.isFailureMessage(error.message)) conflicts++;
|
|
1829
|
+
return conflicts;
|
|
1830
|
+
}
|
|
1233
1831
|
hasCreateDocumentAction(job) {
|
|
1234
1832
|
for (const action of job.actions) if (action.type === "CREATE_DOCUMENT") return true;
|
|
1235
1833
|
for (const operation of job.operations) if (operation.action.type === "CREATE_DOCUMENT") return true;
|
|
@@ -1245,22 +1843,134 @@ var JobResultHandler = class {
|
|
|
1245
1843
|
stackLines.push(`[Attempt ${index + 1}] Stack trace:\n${error.stack}`);
|
|
1246
1844
|
});
|
|
1247
1845
|
return {
|
|
1846
|
+
name: currentError.name,
|
|
1248
1847
|
message: messageLines.join("\n"),
|
|
1249
1848
|
stack: stackLines.join("\n\n")
|
|
1250
1849
|
};
|
|
1251
1850
|
}
|
|
1252
1851
|
};
|
|
1253
1852
|
//#endregion
|
|
1254
|
-
//#region src/executor/
|
|
1853
|
+
//#region src/executor/deferred-jobs.ts
|
|
1255
1854
|
/**
|
|
1256
|
-
*
|
|
1855
|
+
* The jobs held back because the document they write to was not there yet.
|
|
1856
|
+
*
|
|
1857
|
+
* A job whose document is missing is deferred rather than failed, because the
|
|
1858
|
+
* operations that create that document may still be on their way - out of order
|
|
1859
|
+
* from a peer, or a sibling job in the same batch. Deferring emits no event and
|
|
1860
|
+
* moves no job status, which is what makes it cheap: a job that gets flushed
|
|
1861
|
+
* runs as if it had merely waited its turn.
|
|
1862
|
+
*
|
|
1863
|
+
* That silence is also why the wait has to be bounded. A deferred job is only
|
|
1864
|
+
* released when a job carrying CREATE_DOCUMENT for that exact document id
|
|
1865
|
+
* completes, so a job deferred against an id nothing will ever create is
|
|
1866
|
+
* released by nothing. Its status stays RUNNING, which is not terminal, so a
|
|
1867
|
+
* caller awaiting it waits forever - there is no timeout on that side. The TTL
|
|
1868
|
+
* converts that silence into the failure the caller can act on.
|
|
1869
|
+
*
|
|
1870
|
+
* Failing through the queue rather than only the tracker matters: the queue
|
|
1871
|
+
* records the job as resolved, which is what unblocks anything that declared a
|
|
1872
|
+
* dependency on it. Deferring alone does not, so a dependent of a deferred job
|
|
1873
|
+
* is stranded at PENDING for as long as the deferral lasts.
|
|
1257
1874
|
*/
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1875
|
+
var DeferredJobs = class {
|
|
1876
|
+
#byDocumentId = /* @__PURE__ */ new Map();
|
|
1877
|
+
#timers = /* @__PURE__ */ new Map();
|
|
1878
|
+
constructor(queue, jobTracker, eventBus, logger, ttlMs = DEFAULT_DEFERRED_JOB_TTL_MS, onReleased = () => Promise.resolve()) {
|
|
1879
|
+
this.queue = queue;
|
|
1880
|
+
this.jobTracker = jobTracker;
|
|
1881
|
+
this.eventBus = eventBus;
|
|
1882
|
+
this.logger = logger;
|
|
1883
|
+
this.ttlMs = ttlMs;
|
|
1884
|
+
this.onReleased = onReleased;
|
|
1885
|
+
}
|
|
1886
|
+
/**
|
|
1887
|
+
* Holds a job until its document arrives, or until the TTL elapses.
|
|
1888
|
+
*
|
|
1889
|
+
* A job deferred again after a flush replaces its own timer rather than
|
|
1890
|
+
* accumulating one, so the wait is measured from the most recent attempt.
|
|
1891
|
+
*/
|
|
1892
|
+
add(documentId, job) {
|
|
1893
|
+
const existing = this.#byDocumentId.get(documentId) ?? [];
|
|
1894
|
+
existing.push(job);
|
|
1895
|
+
this.#byDocumentId.set(documentId, existing);
|
|
1896
|
+
this.#clearTimer(job.id);
|
|
1897
|
+
const timer = setTimeout(() => {
|
|
1898
|
+
this.#expire(documentId, job);
|
|
1899
|
+
}, this.ttlMs);
|
|
1900
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
1901
|
+
this.#timers.set(job.id, timer);
|
|
1902
|
+
}
|
|
1903
|
+
/** Re-enqueues everything waiting on a document that has now arrived. */
|
|
1904
|
+
async flush(documentId) {
|
|
1905
|
+
const jobs = this.#byDocumentId.get(documentId);
|
|
1906
|
+
if (!jobs || jobs.length === 0) return;
|
|
1907
|
+
this.#byDocumentId.delete(documentId);
|
|
1908
|
+
for (const job of jobs) {
|
|
1909
|
+
this.#clearTimer(job.id);
|
|
1910
|
+
try {
|
|
1911
|
+
await this.queue.enqueue(job);
|
|
1912
|
+
} catch (error) {
|
|
1913
|
+
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
/**
|
|
1918
|
+
* Fails everything still deferred, for a reactor that is shutting down.
|
|
1919
|
+
*
|
|
1920
|
+
* The queue is deliberately not told: it is going away too, and nothing is
|
|
1921
|
+
* left to unblock.
|
|
1922
|
+
*/
|
|
1923
|
+
failAll() {
|
|
1924
|
+
for (const [, jobs] of this.#byDocumentId) for (const job of jobs) {
|
|
1925
|
+
this.#clearTimer(job.id);
|
|
1926
|
+
this.#markFailed(job);
|
|
1927
|
+
}
|
|
1928
|
+
this.#byDocumentId.clear();
|
|
1929
|
+
}
|
|
1930
|
+
/** The jobs still waiting on a document. Exposed for assertions. */
|
|
1931
|
+
waitingOn(documentId) {
|
|
1932
|
+
return this.#byDocumentId.get(documentId) ?? [];
|
|
1933
|
+
}
|
|
1934
|
+
async #expire(documentId, job) {
|
|
1935
|
+
const jobs = this.#byDocumentId.get(documentId);
|
|
1936
|
+
if (!jobs?.includes(job)) return;
|
|
1937
|
+
const remaining = jobs.filter((held) => held !== job);
|
|
1938
|
+
if (remaining.length === 0) this.#byDocumentId.delete(documentId);
|
|
1939
|
+
else this.#byDocumentId.set(documentId, remaining);
|
|
1940
|
+
this.#timers.delete(job.id);
|
|
1941
|
+
this.logger.error("Deferred job @jobId gave up waiting for document @documentId after @ttlMs ms", job.id, documentId, this.ttlMs);
|
|
1942
|
+
const errorInfo = this.#markFailed(job);
|
|
1943
|
+
try {
|
|
1944
|
+
await this.queue.failJob(job.id, errorInfo);
|
|
1945
|
+
} catch (error) {
|
|
1946
|
+
this.logger.error("Error failing an expired deferred job: @Error", error);
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1950
|
+
try {
|
|
1951
|
+
await this.onReleased();
|
|
1952
|
+
} catch (error) {
|
|
1953
|
+
this.logger.error("Error dispatching after a deferred job expired: @Error", error);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
#markFailed(job) {
|
|
1957
|
+
const error = new DocumentNotFoundError(job.documentId);
|
|
1958
|
+
const errorInfo = toErrorInfo(error);
|
|
1959
|
+
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1960
|
+
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
1961
|
+
jobId: job.id,
|
|
1962
|
+
error,
|
|
1963
|
+
job
|
|
1964
|
+
}).catch(() => {});
|
|
1965
|
+
return errorInfo;
|
|
1966
|
+
}
|
|
1967
|
+
#clearTimer(jobId) {
|
|
1968
|
+
const timer = this.#timers.get(jobId);
|
|
1969
|
+
if (timer !== void 0) {
|
|
1970
|
+
clearTimeout(timer);
|
|
1971
|
+
this.#timers.delete(jobId);
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1264
1974
|
};
|
|
1265
1975
|
//#endregion
|
|
1266
1976
|
//#region src/executor/worker-pool-router.ts
|
|
@@ -1297,18 +2007,6 @@ function bucketFor(documentId, numWorkers) {
|
|
|
1297
2007
|
//#endregion
|
|
1298
2008
|
//#region src/executor/worker-pool-job-executor-manager.ts
|
|
1299
2009
|
/**
|
|
1300
|
-
* Action types whose application invalidates the parent's collection
|
|
1301
|
-
* membership cache. Mirrors the in-process invalidation pattern in
|
|
1302
|
-
* `document-action-handler.ts` (the worker pool relocates that work to
|
|
1303
|
-
* the parent because the cache lives there).
|
|
1304
|
-
*/
|
|
1305
|
-
const MEMBERSHIP_INVALIDATING_ACTIONS = new Set([
|
|
1306
|
-
"ADD_RELATIONSHIP",
|
|
1307
|
-
"REMOVE_RELATIONSHIP",
|
|
1308
|
-
"UPDATE_RELATIONSHIP",
|
|
1309
|
-
"DELETE_DOCUMENT"
|
|
1310
|
-
]);
|
|
1311
|
-
/**
|
|
1312
2010
|
* Manages a pool of executor workers and dispatches jobs across them with
|
|
1313
2011
|
* sticky-by-documentId routing. Replaces `SimpleJobExecutorManager` when
|
|
1314
2012
|
* the worker pool is enabled.
|
|
@@ -1318,10 +2016,19 @@ const MEMBERSHIP_INVALIDATING_ACTIONS = new Set([
|
|
|
1318
2016
|
* - Emitting `JOB_RUNNING` and `JOB_WRITE_READY` events; the worker's
|
|
1319
2017
|
* local event bus is a no-op stub.
|
|
1320
2018
|
* - Maintaining the deferred-jobs map for `DocumentNotFoundError`.
|
|
1321
|
-
* -
|
|
1322
|
-
*
|
|
1323
|
-
*
|
|
1324
|
-
*
|
|
2019
|
+
* - Enriching the outgoing `JOB_WRITE_READY` with
|
|
2020
|
+
* `collectionMemberships`, read from the operation index after the
|
|
2021
|
+
* worker has committed. It takes the whole index rather than a
|
|
2022
|
+
* narrower read interface so that handing it a cache does not
|
|
2023
|
+
* compile: which documents a commit moves between collections is not
|
|
2024
|
+
* derivable from action shape, because joining a collection also
|
|
2025
|
+
* joins every group the joining document has referenced and that set
|
|
2026
|
+
* comes from selects run inside the commit. A parent-side cache
|
|
2027
|
+
* cannot learn it went stale, and a stale entry silently drops the
|
|
2028
|
+
* document's operations from the outbox of every remote subscribed to
|
|
2029
|
+
* the omitted collection. Costs one primary-key-prefix lookup on
|
|
2030
|
+
* `document_collections` per job that produced operations, on a
|
|
2031
|
+
* fire-and-forget path.
|
|
1325
2032
|
*
|
|
1326
2033
|
* @see Executor Worker Pool Design wiki page
|
|
1327
2034
|
* (Powerhouse board wiki id: d400d711-f07e-4389-a226-4e9fdd4fa8ba)
|
|
@@ -1332,18 +2039,19 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1332
2039
|
activeJobs = 0;
|
|
1333
2040
|
totalJobsProcessed = 0;
|
|
1334
2041
|
unsubscribe;
|
|
1335
|
-
deferredJobs
|
|
2042
|
+
deferredJobs;
|
|
1336
2043
|
resultHandler;
|
|
1337
2044
|
jobTimeoutMs;
|
|
1338
|
-
constructor(workerFactory, eventBus, queue, jobTracker, logger, resolver,
|
|
2045
|
+
constructor(workerFactory, eventBus, queue, jobTracker, logger, resolver, operationIndex, jobTimeoutMs = 3e4, deferredJobTtlMs = DEFAULT_DEFERRED_JOB_TTL_MS) {
|
|
1339
2046
|
this.workerFactory = workerFactory;
|
|
1340
2047
|
this.eventBus = eventBus;
|
|
1341
2048
|
this.queue = queue;
|
|
1342
2049
|
this.jobTracker = jobTracker;
|
|
1343
2050
|
this.logger = logger;
|
|
1344
2051
|
this.resolver = resolver;
|
|
1345
|
-
this.
|
|
2052
|
+
this.operationIndex = operationIndex;
|
|
1346
2053
|
this.jobTimeoutMs = jobTimeoutMs;
|
|
2054
|
+
this.deferredJobs = new DeferredJobs(queue, jobTracker, eventBus, logger, deferredJobTtlMs, () => this.tryDispatchAll());
|
|
1347
2055
|
this.resultHandler = new JobResultHandler(queue, jobTracker, eventBus, resolver, logger);
|
|
1348
2056
|
}
|
|
1349
2057
|
async start(numWorkers) {
|
|
@@ -1364,16 +2072,7 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1364
2072
|
this.unsubscribe = void 0;
|
|
1365
2073
|
}
|
|
1366
2074
|
if (graceful) while (this.activeJobs > 0) await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1367
|
-
|
|
1368
|
-
const errorInfo = toErrorInfo(new DocumentNotFoundError(job.documentId));
|
|
1369
|
-
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1370
|
-
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
1371
|
-
jobId: job.id,
|
|
1372
|
-
error: new DocumentNotFoundError(job.documentId),
|
|
1373
|
-
job
|
|
1374
|
-
}).catch(() => {});
|
|
1375
|
-
}
|
|
1376
|
-
this.deferredJobs.clear();
|
|
2075
|
+
this.deferredJobs.failAll();
|
|
1377
2076
|
await Promise.all(this.workers.map((w) => w.shutdown(graceful).catch((err) => {
|
|
1378
2077
|
this.logger.warn("worker shutdown failed: @Error", err);
|
|
1379
2078
|
})));
|
|
@@ -1495,22 +2194,17 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1495
2194
|
this.logger.error("emitWriteReady failed for job @jobId: @Error", { jobId: handle.job.id }, error);
|
|
1496
2195
|
});
|
|
1497
2196
|
await this.resultHandler.handleResult(handle, outcome.result, {
|
|
1498
|
-
deferJob: (documentId, job) =>
|
|
1499
|
-
|
|
1500
|
-
existing.push(job);
|
|
1501
|
-
this.deferredJobs.set(documentId, existing);
|
|
1502
|
-
},
|
|
1503
|
-
flushDeferredFor: (documentId) => this.flushDeferredJobs(documentId)
|
|
2197
|
+
deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
|
|
2198
|
+
flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
|
|
1504
2199
|
});
|
|
1505
2200
|
this.activeJobs--;
|
|
1506
2201
|
await this.tryDispatchFor(worker);
|
|
1507
2202
|
}
|
|
1508
2203
|
async emitWriteReady(job, payload) {
|
|
1509
|
-
this.invalidateMembershipsFor(payload.operations);
|
|
1510
2204
|
const documentIds = [...new Set(payload.operations.map((op) => op.context.documentId))];
|
|
1511
2205
|
let collectionMemberships = {};
|
|
1512
|
-
try {
|
|
1513
|
-
collectionMemberships = await this.
|
|
2206
|
+
if (documentIds.length > 0) try {
|
|
2207
|
+
collectionMemberships = fillMissingMemberships(documentIds, await this.operationIndex.getCollectionsForDocuments(documentIds));
|
|
1514
2208
|
} catch (error) {
|
|
1515
2209
|
this.logger.error("Failed to load collection memberships for JOB_WRITE_READY: @Error", error);
|
|
1516
2210
|
}
|
|
@@ -1526,14 +2220,6 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1526
2220
|
this.logger.error("Failed to emit JOB_WRITE_READY event: @Error", error);
|
|
1527
2221
|
}
|
|
1528
2222
|
}
|
|
1529
|
-
invalidateMembershipsFor(operations) {
|
|
1530
|
-
for (const op of operations) {
|
|
1531
|
-
const actionType = op.operation.action.type;
|
|
1532
|
-
if (!MEMBERSHIP_INVALIDATING_ACTIONS.has(actionType)) continue;
|
|
1533
|
-
const target = extractMembershipTarget(op);
|
|
1534
|
-
if (target) this.collectionMembershipCache.invalidate(target);
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
2223
|
/**
|
|
1538
2224
|
* Handle a worker-transport failure (worker exited / init failed / abort
|
|
1539
2225
|
* timed out) detected while `worker.execute` was in flight. Re-enqueues
|
|
@@ -1582,16 +2268,6 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1582
2268
|
this.workers[deadIndex] = fresh;
|
|
1583
2269
|
await this.tryDispatchFor(fresh);
|
|
1584
2270
|
}
|
|
1585
|
-
async flushDeferredJobs(documentId) {
|
|
1586
|
-
const jobs = this.deferredJobs.get(documentId);
|
|
1587
|
-
if (!jobs || jobs.length === 0) return;
|
|
1588
|
-
this.deferredJobs.delete(documentId);
|
|
1589
|
-
for (const job of jobs) try {
|
|
1590
|
-
await this.queue.enqueue(job);
|
|
1591
|
-
} catch (error) {
|
|
1592
|
-
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
2271
|
};
|
|
1596
2272
|
function isWorkerTransportError(error) {
|
|
1597
2273
|
return error instanceof WorkerExitedError || error instanceof WorkerInitFailedError || error instanceof WorkerAbortTimeoutError;
|
|
@@ -1602,11 +2278,16 @@ function isDuplicateModuleFailure(reason) {
|
|
|
1602
2278
|
const cause = reason.cause;
|
|
1603
2279
|
return cause instanceof Error && cause.name === "DuplicateModuleError";
|
|
1604
2280
|
}
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
2281
|
+
/**
|
|
2282
|
+
* Gives every requested document a key, empty when it belongs to no
|
|
2283
|
+
* collection. The operation index omits documents with no rows while the
|
|
2284
|
+
* membership cache defaulted them to `[]`, so this keeps the emitted
|
|
2285
|
+
* `JobWriteReadyEvent.collectionMemberships` shape unchanged.
|
|
2286
|
+
*/
|
|
2287
|
+
function fillMissingMemberships(documentIds, found) {
|
|
2288
|
+
const result = {};
|
|
2289
|
+
for (const documentId of documentIds) result[documentId] = found[documentId] ?? [];
|
|
2290
|
+
return result;
|
|
1610
2291
|
}
|
|
1611
2292
|
//#endregion
|
|
1612
2293
|
//#region src/executor/simple-job-executor-manager.ts
|
|
@@ -1620,10 +2301,10 @@ var SimpleJobExecutorManager = class {
|
|
|
1620
2301
|
activeJobs = 0;
|
|
1621
2302
|
totalJobsProcessed = 0;
|
|
1622
2303
|
unsubscribe;
|
|
1623
|
-
deferredJobs
|
|
2304
|
+
deferredJobs;
|
|
1624
2305
|
resultHandler;
|
|
1625
2306
|
jobTimeoutMs;
|
|
1626
|
-
constructor(executorFactory, eventBus, queue, jobTracker, logger, resolver, jobTimeoutMs = 3e4) {
|
|
2307
|
+
constructor(executorFactory, eventBus, queue, jobTracker, logger, resolver, jobTimeoutMs = 3e4, deferredJobTtlMs = DEFAULT_DEFERRED_JOB_TTL_MS) {
|
|
1627
2308
|
this.executorFactory = executorFactory;
|
|
1628
2309
|
this.eventBus = eventBus;
|
|
1629
2310
|
this.queue = queue;
|
|
@@ -1631,6 +2312,7 @@ var SimpleJobExecutorManager = class {
|
|
|
1631
2312
|
this.logger = logger;
|
|
1632
2313
|
this.resolver = resolver;
|
|
1633
2314
|
this.jobTimeoutMs = jobTimeoutMs;
|
|
2315
|
+
this.deferredJobs = new DeferredJobs(queue, jobTracker, eventBus, logger, deferredJobTtlMs, () => this.checkForMoreJobs());
|
|
1634
2316
|
this.resultHandler = new JobResultHandler(queue, jobTracker, eventBus, resolver, logger);
|
|
1635
2317
|
}
|
|
1636
2318
|
async start(numExecutors) {
|
|
@@ -1651,16 +2333,7 @@ var SimpleJobExecutorManager = class {
|
|
|
1651
2333
|
this.unsubscribe = void 0;
|
|
1652
2334
|
}
|
|
1653
2335
|
if (graceful) while (this.activeJobs > 0) await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1654
|
-
|
|
1655
|
-
const errorInfo = toErrorInfo(new DocumentNotFoundError(job.documentId));
|
|
1656
|
-
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1657
|
-
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
1658
|
-
jobId: job.id,
|
|
1659
|
-
error: new DocumentNotFoundError(job.documentId),
|
|
1660
|
-
job
|
|
1661
|
-
}).catch(() => {});
|
|
1662
|
-
}
|
|
1663
|
-
this.deferredJobs.clear();
|
|
2336
|
+
this.deferredJobs.failAll();
|
|
1664
2337
|
this.executors = [];
|
|
1665
2338
|
this.isRunning = false;
|
|
1666
2339
|
}
|
|
@@ -1753,12 +2426,8 @@ var SimpleJobExecutorManager = class {
|
|
|
1753
2426
|
this.eventBus.emit(JobExecutorEventTypes.JOB_FAILED, failedEvent).catch(() => {});
|
|
1754
2427
|
}
|
|
1755
2428
|
await this.resultHandler.handleResult(handle, result, {
|
|
1756
|
-
deferJob: (documentId, job) =>
|
|
1757
|
-
|
|
1758
|
-
existing.push(job);
|
|
1759
|
-
this.deferredJobs.set(documentId, existing);
|
|
1760
|
-
},
|
|
1761
|
-
flushDeferredFor: (documentId) => this.flushDeferredJobs(documentId)
|
|
2429
|
+
deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
|
|
2430
|
+
flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
|
|
1762
2431
|
});
|
|
1763
2432
|
this.activeJobs--;
|
|
1764
2433
|
await this.checkForMoreJobs();
|
|
@@ -1792,16 +2461,6 @@ var SimpleJobExecutorManager = class {
|
|
|
1792
2461
|
}
|
|
1793
2462
|
}
|
|
1794
2463
|
}
|
|
1795
|
-
async flushDeferredJobs(documentId) {
|
|
1796
|
-
const jobs = this.deferredJobs.get(documentId);
|
|
1797
|
-
if (!jobs || jobs.length === 0) return;
|
|
1798
|
-
this.deferredJobs.delete(documentId);
|
|
1799
|
-
for (const job of jobs) try {
|
|
1800
|
-
await this.queue.enqueue(job);
|
|
1801
|
-
} catch (error) {
|
|
1802
|
-
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
2464
|
};
|
|
1806
2465
|
//#endregion
|
|
1807
2466
|
//#region src/job-tracker/in-memory-job-tracker.ts
|
|
@@ -1850,6 +2509,7 @@ var InMemoryJobTracker = class {
|
|
|
1850
2509
|
}
|
|
1851
2510
|
handleJobFailed(event) {
|
|
1852
2511
|
this.markFailed(event.jobId, {
|
|
2512
|
+
name: event.error.name,
|
|
1853
2513
|
message: event.error.message,
|
|
1854
2514
|
stack: event.error.stack || ""
|
|
1855
2515
|
}, event.job);
|
|
@@ -1958,6 +2618,26 @@ function matchesFilter(op, filter) {
|
|
|
1958
2618
|
}
|
|
1959
2619
|
return true;
|
|
1960
2620
|
}
|
|
2621
|
+
function nonEmpty(value) {
|
|
2622
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
2623
|
+
}
|
|
2624
|
+
function namespaceOf(processor) {
|
|
2625
|
+
return nonEmpty(processor.namespace);
|
|
2626
|
+
}
|
|
2627
|
+
function classNameOf(processor) {
|
|
2628
|
+
const name = processor.constructor.name;
|
|
2629
|
+
return name && name !== "Object" ? name : void 0;
|
|
2630
|
+
}
|
|
2631
|
+
function resolveProcessorSlots(records, legacy) {
|
|
2632
|
+
if (legacy) return records.map((_, i) => String(i));
|
|
2633
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2634
|
+
return records.map((record, i) => {
|
|
2635
|
+
const base = nonEmpty(record.id) ?? namespaceOf(record.processor) ?? classNameOf(record.processor) ?? String(i);
|
|
2636
|
+
const count = seen.get(base) ?? 0;
|
|
2637
|
+
seen.set(base, count + 1);
|
|
2638
|
+
return count === 0 ? base : `${base}#${count}`;
|
|
2639
|
+
});
|
|
2640
|
+
}
|
|
1961
2641
|
//#endregion
|
|
1962
2642
|
//#region src/processors/processor-manager.ts
|
|
1963
2643
|
/**
|
|
@@ -1979,13 +2659,15 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
1979
2659
|
cursorCache = /* @__PURE__ */ new Map();
|
|
1980
2660
|
logger;
|
|
1981
2661
|
driveContainerTypes;
|
|
1982
|
-
|
|
2662
|
+
legacyProcessorIds;
|
|
2663
|
+
constructor(db, operationIndex, writeCache, consistencyTracker, logger, driveContainerTypes, options = {}) {
|
|
1983
2664
|
super(db, operationIndex, writeCache, consistencyTracker, {
|
|
1984
2665
|
readModelId: "processor-manager",
|
|
1985
2666
|
rebuildStateOnInit: true
|
|
1986
2667
|
});
|
|
1987
2668
|
this.logger = logger;
|
|
1988
2669
|
this.driveContainerTypes = driveContainerTypes;
|
|
2670
|
+
this.legacyProcessorIds = options.legacyProcessorIds ?? true;
|
|
1989
2671
|
}
|
|
1990
2672
|
async init() {
|
|
1991
2673
|
await super.init();
|
|
@@ -2072,9 +2754,10 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
2072
2754
|
}
|
|
2073
2755
|
if (records.length === 0) return;
|
|
2074
2756
|
const trackedList = [];
|
|
2757
|
+
const slots = resolveProcessorSlots(records, this.legacyProcessorIds);
|
|
2075
2758
|
for (let i = 0; i < records.length; i++) {
|
|
2076
2759
|
const record = records[i];
|
|
2077
|
-
const processorId = `${identifier}:${driveId}:${i}`;
|
|
2760
|
+
const processorId = `${identifier}:${driveId}:${slots[i]}`;
|
|
2078
2761
|
const cached = this.cursorCache.get(processorId);
|
|
2079
2762
|
let lastOrdinal;
|
|
2080
2763
|
let status;
|
|
@@ -2106,8 +2789,9 @@ var ProcessorManager = class extends BaseReadModel {
|
|
|
2106
2789
|
trackedList.push(tracked);
|
|
2107
2790
|
await this.saveProcessorCursor(tracked);
|
|
2108
2791
|
}
|
|
2109
|
-
|
|
2110
|
-
|
|
2792
|
+
const liveIds = new Set(trackedList.map((t) => t.processorId));
|
|
2793
|
+
await this.db.deleteFrom("ProcessorCursor").where("factoryId", "=", identifier).where("driveId", "=", driveId).where("processorId", "not in", [...liveIds]).execute();
|
|
2794
|
+
for (const [id, row] of this.cursorCache) if (row.factoryId === identifier && row.driveId === driveId && !liveIds.has(id)) this.cursorCache.delete(id);
|
|
2111
2795
|
const factoryProcessors = this.factoryToProcessors.get(identifier);
|
|
2112
2796
|
if (factoryProcessors) factoryProcessors.set(driveId, trackedList);
|
|
2113
2797
|
const existingDriveProcessors = this.processorsByDrive.get(driveId) ?? [];
|
|
@@ -2311,10 +2995,12 @@ var InMemoryQueue = class {
|
|
|
2311
2995
|
}
|
|
2312
2996
|
toErrorInfo(error) {
|
|
2313
2997
|
if (error instanceof Error) return {
|
|
2998
|
+
name: error.name,
|
|
2314
2999
|
message: error.message,
|
|
2315
3000
|
stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
|
|
2316
3001
|
};
|
|
2317
3002
|
return {
|
|
3003
|
+
name: "Error",
|
|
2318
3004
|
message: error,
|
|
2319
3005
|
stack: (/* @__PURE__ */ new Error()).stack || ""
|
|
2320
3006
|
};
|
|
@@ -2402,6 +3088,7 @@ var InMemoryQueue = class {
|
|
|
2402
3088
|
await this.resolver.ensureModelLoaded(documentType);
|
|
2403
3089
|
} catch {
|
|
2404
3090
|
await this.failJob(job.id, {
|
|
3091
|
+
name: "Error",
|
|
2405
3092
|
message: `Failed to load document model for type: ${documentType}`,
|
|
2406
3093
|
stack: (/* @__PURE__ */ new Error()).stack || ""
|
|
2407
3094
|
});
|
|
@@ -2575,9 +3262,11 @@ var InMemoryQueue = class {
|
|
|
2575
3262
|
this.jobIndex.delete(jobId);
|
|
2576
3263
|
this.completedJobs.add(jobId);
|
|
2577
3264
|
await this.remove(jobId);
|
|
3265
|
+
const emittedError = new Error(error?.message ?? "Job failed");
|
|
3266
|
+
if (error?.name) emittedError.name = error.name;
|
|
2578
3267
|
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
2579
3268
|
jobId,
|
|
2580
|
-
error:
|
|
3269
|
+
error: emittedError,
|
|
2581
3270
|
job
|
|
2582
3271
|
}).catch(() => {});
|
|
2583
3272
|
this.checkDrained();
|
|
@@ -2587,7 +3276,7 @@ var InMemoryQueue = class {
|
|
|
2587
3276
|
if (documentId) this.markJobComplete(jobId, documentId);
|
|
2588
3277
|
this.jobIndex.delete(jobId);
|
|
2589
3278
|
}
|
|
2590
|
-
async retryJob(jobId, error) {
|
|
3279
|
+
async retryJob(jobId, error, accounting = RetryAccounting.CountAgainstLimit) {
|
|
2591
3280
|
const job = this.jobIndex.get(jobId);
|
|
2592
3281
|
if (!job) return;
|
|
2593
3282
|
job.lastError = error;
|
|
@@ -2596,9 +3285,10 @@ var InMemoryQueue = class {
|
|
|
2596
3285
|
this.jobIndex.delete(jobId);
|
|
2597
3286
|
this.jobIdToQueueKey.delete(jobId);
|
|
2598
3287
|
if (error) job.errorHistory.push(error);
|
|
3288
|
+
const retryCount = job.retryCount || 0;
|
|
2599
3289
|
const updatedJob = {
|
|
2600
3290
|
...job,
|
|
2601
|
-
retryCount:
|
|
3291
|
+
retryCount: accounting === RetryAccounting.CountAgainstLimit ? retryCount + 1 : retryCount,
|
|
2602
3292
|
lastError: error
|
|
2603
3293
|
};
|
|
2604
3294
|
await this.enqueue(updatedJob);
|
|
@@ -2686,6 +3376,19 @@ var InMemoryQueue = class {
|
|
|
2686
3376
|
}
|
|
2687
3377
|
};
|
|
2688
3378
|
//#endregion
|
|
3379
|
+
//#region src/projection/read-model-kinds.ts
|
|
3380
|
+
/**
|
|
3381
|
+
* Every {@link BuiltInReadModelKind}, at runtime. Derived from a total
|
|
3382
|
+
* record so adding a kind to the union without adding it here is a compile
|
|
3383
|
+
* error: under projection sharding a kind missing from both `preReadyKinds`
|
|
3384
|
+
* and `postReadyKinds` is indexed by nobody — the host's own copies never
|
|
3385
|
+
* index an operation — so a gap here reads as silently stale data.
|
|
3386
|
+
*/
|
|
3387
|
+
const BUILT_IN_READ_MODEL_KINDS = Object.keys({
|
|
3388
|
+
"document-view": true,
|
|
3389
|
+
"document-indexer": true
|
|
3390
|
+
});
|
|
3391
|
+
//#endregion
|
|
2689
3392
|
//#region src/core/model-sources.ts
|
|
2690
3393
|
function modelSourceKey(module) {
|
|
2691
3394
|
return `${module.documentModel.global.id}@${module.version ?? 1}`;
|
|
@@ -3116,6 +3819,83 @@ var SubscriptionNotificationReadModel = class {
|
|
|
3116
3819
|
}
|
|
3117
3820
|
};
|
|
3118
3821
|
//#endregion
|
|
3822
|
+
//#region src/core/group-reevaluation-trigger.ts
|
|
3823
|
+
/**
|
|
3824
|
+
* Watches committed writes for group membership changes and enqueues a
|
|
3825
|
+
* re-evaluation job for every document whose auth history references the
|
|
3826
|
+
* changed group, found through the reverse direction of the group-reference
|
|
3827
|
+
* relation. Each affected document is re-judged in its own job, so the work
|
|
3828
|
+
* runs under that document's execution slot rather than the group's.
|
|
3829
|
+
*
|
|
3830
|
+
* The job carries the earliest changed membership timestamp; the executor
|
|
3831
|
+
* skips the pass when everything the document holds sorts before it, which
|
|
3832
|
+
* keeps the common case (a membership write later than all history) free.
|
|
3833
|
+
*/
|
|
3834
|
+
var GroupReevaluationTrigger = class {
|
|
3835
|
+
unsubscribe;
|
|
3836
|
+
constructor(logger, eventBus, queue, operationIndex) {
|
|
3837
|
+
this.logger = logger;
|
|
3838
|
+
this.eventBus = eventBus;
|
|
3839
|
+
this.queue = queue;
|
|
3840
|
+
this.operationIndex = operationIndex;
|
|
3841
|
+
}
|
|
3842
|
+
startup() {
|
|
3843
|
+
this.unsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, async (_type, event) => this.onWriteReady(event));
|
|
3844
|
+
}
|
|
3845
|
+
shutdown() {
|
|
3846
|
+
this.unsubscribe?.();
|
|
3847
|
+
this.unsubscribe = void 0;
|
|
3848
|
+
}
|
|
3849
|
+
async onWriteReady(event) {
|
|
3850
|
+
const changed = /* @__PURE__ */ new Map();
|
|
3851
|
+
for (const { operation, context } of event.operations) {
|
|
3852
|
+
if (context.documentType !== groupDocumentType || context.scope !== "global" || !groupMembershipActionTypes.includes(operation.action.type)) continue;
|
|
3853
|
+
const existing = changed.get(context.documentId);
|
|
3854
|
+
if (existing === void 0 || Date.parse(operation.timestampUtcMs) < Date.parse(existing)) changed.set(context.documentId, operation.timestampUtcMs);
|
|
3855
|
+
}
|
|
3856
|
+
if (changed.size === 0) return;
|
|
3857
|
+
const affected = /* @__PURE__ */ new Map();
|
|
3858
|
+
for (const [groupId, timestamp] of changed) {
|
|
3859
|
+
let referencers;
|
|
3860
|
+
try {
|
|
3861
|
+
referencers = await this.operationIndex.getGroupReferencers(groupId);
|
|
3862
|
+
} catch (error) {
|
|
3863
|
+
this.logger.error("Failed to resolve referencers of group @groupId: @error", groupId, error);
|
|
3864
|
+
continue;
|
|
3865
|
+
}
|
|
3866
|
+
for (const documentId of referencers) {
|
|
3867
|
+
const existing = affected.get(documentId);
|
|
3868
|
+
if (existing === void 0 || Date.parse(timestamp) < Date.parse(existing)) affected.set(documentId, timestamp);
|
|
3869
|
+
}
|
|
3870
|
+
}
|
|
3871
|
+
for (const [documentId, timestamp] of affected) {
|
|
3872
|
+
const jobId = v4();
|
|
3873
|
+
const job = {
|
|
3874
|
+
id: jobId,
|
|
3875
|
+
kind: "reevaluation",
|
|
3876
|
+
documentId,
|
|
3877
|
+
scope: "global",
|
|
3878
|
+
branch: "main",
|
|
3879
|
+
actions: [],
|
|
3880
|
+
operations: [],
|
|
3881
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3882
|
+
queueHint: [],
|
|
3883
|
+
maxRetries: 3,
|
|
3884
|
+
errorHistory: [],
|
|
3885
|
+
meta: {
|
|
3886
|
+
...buildSingleJobMeta(jobId),
|
|
3887
|
+
triggerTimestampUtcMs: timestamp
|
|
3888
|
+
}
|
|
3889
|
+
};
|
|
3890
|
+
try {
|
|
3891
|
+
await this.queue.enqueue(job);
|
|
3892
|
+
} catch (error) {
|
|
3893
|
+
this.logger.error("Failed to enqueue re-evaluation of @documentId: @error", documentId, error);
|
|
3894
|
+
}
|
|
3895
|
+
}
|
|
3896
|
+
}
|
|
3897
|
+
};
|
|
3898
|
+
//#endregion
|
|
3119
3899
|
//#region src/sync/types.ts
|
|
3120
3900
|
let ChannelScheme = /* @__PURE__ */ function(ChannelScheme) {
|
|
3121
3901
|
ChannelScheme["CONNECT"] = "connect";
|
|
@@ -3407,13 +4187,42 @@ var BufferedMailbox = class {
|
|
|
3407
4187
|
var GraphQLRequestError = class extends Error {
|
|
3408
4188
|
statusCode;
|
|
3409
4189
|
category;
|
|
3410
|
-
|
|
4190
|
+
/**
|
|
4191
|
+
* One entry per error the response carried, in order, holding its
|
|
4192
|
+
* `extensions.code` - undefined where it declared none. Kept per error rather
|
|
4193
|
+
* than as a set, because a response that mixes a classified error with an
|
|
4194
|
+
* unclassified one must not be read as if only the classified one arrived.
|
|
4195
|
+
*/
|
|
4196
|
+
codes;
|
|
4197
|
+
constructor(message, category, statusCode, codes = []) {
|
|
3411
4198
|
super(message);
|
|
3412
4199
|
this.name = "GraphQLRequestError";
|
|
3413
4200
|
this.category = category;
|
|
3414
4201
|
this.statusCode = statusCode;
|
|
4202
|
+
this.codes = codes;
|
|
3415
4203
|
}
|
|
3416
4204
|
};
|
|
4205
|
+
/**
|
|
4206
|
+
* Extension codes a remote uses to say a failure is worth polling through.
|
|
4207
|
+
*
|
|
4208
|
+
* Shared with reactor-api so the server throws what this check reads and the two
|
|
4209
|
+
* cannot drift. A `graphql` category error is otherwise permanent: it stops the
|
|
4210
|
+
* poll timer, and nothing restarts it, so a code that lands here is the
|
|
4211
|
+
* difference between a channel that recovers and one that is dead for the
|
|
4212
|
+
* process lifetime.
|
|
4213
|
+
*/
|
|
4214
|
+
const RECOVERABLE_GRAPHQL_ERROR_CODES = { malformedStoredOperation: "MALFORMED_STORED_OPERATION" };
|
|
4215
|
+
const RECOVERABLE_CODES = new Set(Object.values(RECOVERABLE_GRAPHQL_ERROR_CODES));
|
|
4216
|
+
/**
|
|
4217
|
+
* True when every error the response carried named a recoverable code.
|
|
4218
|
+
*
|
|
4219
|
+
* Unanimity is the requirement: one unclassified error alongside a recoverable
|
|
4220
|
+
* one means something else also went wrong, and polling through that would be
|
|
4221
|
+
* guessing.
|
|
4222
|
+
*/
|
|
4223
|
+
function isRecoverableGraphQLError(error) {
|
|
4224
|
+
return error.codes.length > 0 && error.codes.every((code) => code !== void 0 && RECOVERABLE_CODES.has(code));
|
|
4225
|
+
}
|
|
3417
4226
|
/** Auth-rejection message fragments the switchboard emits. Shared with
|
|
3418
4227
|
* reactor-api so server throws and this client check can't drift. */
|
|
3419
4228
|
const DRIVE_AUTH_ERROR_MESSAGES = {
|
|
@@ -3437,11 +4246,18 @@ var PollingChannelError = class extends Error {
|
|
|
3437
4246
|
var ChannelError = class extends Error {
|
|
3438
4247
|
source;
|
|
3439
4248
|
error;
|
|
3440
|
-
|
|
4249
|
+
/**
|
|
4250
|
+
* The classification when something other than the error carries it. Absent
|
|
4251
|
+
* means derive it from `error.name`; a dead letter mirrored from a peer sets it,
|
|
4252
|
+
* because only the message crosses the wire.
|
|
4253
|
+
*/
|
|
4254
|
+
errorType;
|
|
4255
|
+
constructor(source, error, errorType) {
|
|
3441
4256
|
super(`ChannelError[${source}]: ${error.message}`);
|
|
3442
4257
|
this.name = "ChannelError";
|
|
3443
4258
|
this.source = source;
|
|
3444
4259
|
this.error = error;
|
|
4260
|
+
this.errorType = errorType;
|
|
3445
4261
|
}
|
|
3446
4262
|
};
|
|
3447
4263
|
//#endregion
|
|
@@ -3646,6 +4462,7 @@ function toOperationWithContext(entry) {
|
|
|
3646
4462
|
skip: entry.skip,
|
|
3647
4463
|
hash: entry.hash,
|
|
3648
4464
|
timestampUtcMs: entry.timestampUtcMs,
|
|
4465
|
+
deniedReason: entry.deniedReason,
|
|
3649
4466
|
action: entry.action
|
|
3650
4467
|
},
|
|
3651
4468
|
context: {
|
|
@@ -3662,29 +4479,35 @@ function toOperationWithContext(entry) {
|
|
|
3662
4479
|
* a single SyncOperation per group. Within each group, operations are sorted
|
|
3663
4480
|
* by context.ordinal. The merged SyncOperation keeps the first group member's
|
|
3664
4481
|
* jobId; all other jobIds are remapped so external dependencies still resolve.
|
|
4482
|
+
*
|
|
4483
|
+
* Only CONTIGUOUS runs merge. A document whose scopes interleave arrives as an
|
|
4484
|
+
* alternating chain (document -> auth -> document -> auth) where each entry
|
|
4485
|
+
* depends on the one before it. Merging every occurrence of a
|
|
4486
|
+
* (documentId, scope, branch) collapses that chain into two nodes that each
|
|
4487
|
+
* depend on the other, which validateBatchStructure rejects as a dependency
|
|
4488
|
+
* cycle. Merging only adjacent entries keeps the chain linear.
|
|
3665
4489
|
*/
|
|
3666
4490
|
function consolidateSyncOperations(syncOps) {
|
|
3667
4491
|
if (syncOps.length <= 1) return syncOps;
|
|
3668
|
-
const groups =
|
|
4492
|
+
const groups = [];
|
|
3669
4493
|
const jobIdRemap = /* @__PURE__ */ new Map();
|
|
3670
|
-
|
|
4494
|
+
let prevKey = null;
|
|
3671
4495
|
for (const syncOp of syncOps) {
|
|
3672
4496
|
const key = `${syncOp.documentId}|${syncOp.scopes.slice().sort().join(",")}|${syncOp.branch}`;
|
|
3673
|
-
const
|
|
3674
|
-
if (
|
|
3675
|
-
|
|
3676
|
-
if (syncOp.jobId && syncOp.jobId !==
|
|
4497
|
+
const current = key === prevKey ? groups[groups.length - 1] : void 0;
|
|
4498
|
+
if (current) {
|
|
4499
|
+
current.ops.push(syncOp);
|
|
4500
|
+
if (syncOp.jobId && syncOp.jobId !== current.canonicalJobId) jobIdRemap.set(syncOp.jobId, current.canonicalJobId);
|
|
3677
4501
|
} else {
|
|
3678
|
-
groups.
|
|
4502
|
+
groups.push({
|
|
3679
4503
|
ops: [syncOp],
|
|
3680
4504
|
canonicalJobId: syncOp.jobId
|
|
3681
4505
|
});
|
|
3682
|
-
|
|
4506
|
+
prevKey = key;
|
|
3683
4507
|
}
|
|
3684
4508
|
}
|
|
3685
4509
|
const result = [];
|
|
3686
|
-
for (const
|
|
3687
|
-
const group = groups.get(key);
|
|
4510
|
+
for (const group of groups) {
|
|
3688
4511
|
const allOperations = group.ops.flatMap((op) => op.operations).sort((a, b) => a.context.ordinal - b.context.ordinal);
|
|
3689
4512
|
const allDeps = /* @__PURE__ */ new Set();
|
|
3690
4513
|
for (const op of group.ops) for (const dep of op.jobDependencies) allDeps.add(dep);
|
|
@@ -3811,6 +4634,33 @@ function splitComponent(items, maxSize) {
|
|
|
3811
4634
|
for (let i = 0; i < sorted.length; i += maxSize) chunks.push(sorted.slice(i, i + maxSize));
|
|
3812
4635
|
return chunks;
|
|
3813
4636
|
}
|
|
4637
|
+
/**
|
|
4638
|
+
* Classifies a failure by error name rather than `instanceof`, because a failure
|
|
4639
|
+
* that crossed the pooled-worker boundary arrives as plain data.
|
|
4640
|
+
*/
|
|
4641
|
+
function classifyJobFailure(errorName) {
|
|
4642
|
+
switch (errorName) {
|
|
4643
|
+
case "AuthTimestampNotMonotonicError": return "AUTH_TIMESTAMP_NOT_MONOTONIC";
|
|
4644
|
+
case "InvalidOperationTimestampError": return "INVALID_TIMESTAMP";
|
|
4645
|
+
case "ExcessiveReshuffleError": return "EXCESSIVE_SHUFFLE";
|
|
4646
|
+
case "InvalidSignatureError": return "SIGNATURE_INVALID";
|
|
4647
|
+
case "HashMismatchError": return "HASH_MISMATCH";
|
|
4648
|
+
default: return "UNCLASSIFIED";
|
|
4649
|
+
}
|
|
4650
|
+
}
|
|
4651
|
+
/** The explicit type when something else carried it, else derived by name. */
|
|
4652
|
+
function syncOperationErrorType(error) {
|
|
4653
|
+
return error?.errorType ?? classifyJobFailure(error?.error?.name ?? "Error");
|
|
4654
|
+
}
|
|
4655
|
+
/** Dead-letter types that must not stop the document syncing. */
|
|
4656
|
+
const NON_QUARANTINING_ERROR_TYPES = new Set(["AUTH_TIMESTAMP_NOT_MONOTONIC"]);
|
|
4657
|
+
/**
|
|
4658
|
+
* A held auth operation must not quarantine: reconciling the two policies needs
|
|
4659
|
+
* the traffic a quarantine would stop.
|
|
4660
|
+
*/
|
|
4661
|
+
function quarantinesDocument(errorType) {
|
|
4662
|
+
return !NON_QUARANTINING_ERROR_TYPES.has(errorType);
|
|
4663
|
+
}
|
|
3814
4664
|
//#endregion
|
|
3815
4665
|
//#region src/sync/channels/interval-poll-timer.ts
|
|
3816
4666
|
const DEFAULT_CONFIG = {
|
|
@@ -3924,36 +4774,15 @@ var IntervalPollTimer = class {
|
|
|
3924
4774
|
//#region src/sync/channels/utils.ts
|
|
3925
4775
|
let syncOpCounter = 0;
|
|
3926
4776
|
/**
|
|
3927
|
-
* Serializes an action for GraphQL transport
|
|
4777
|
+
* Serializes an action for GraphQL transport.
|
|
3928
4778
|
*
|
|
3929
|
-
*
|
|
3930
|
-
*
|
|
3931
|
-
*
|
|
3932
|
-
*
|
|
4779
|
+
* Projects onto exactly the fields `ActionInput` declares, which is what keeps a
|
|
4780
|
+
* stale runtime-only field - a legacy `attachments` array, say - from riding
|
|
4781
|
+
* along into the mutation variables, where the schema would reject the whole
|
|
4782
|
+
* request rather than the field.
|
|
3933
4783
|
*/
|
|
3934
4784
|
function serializeAction(action) {
|
|
3935
|
-
|
|
3936
|
-
id: action.id,
|
|
3937
|
-
type: action.type,
|
|
3938
|
-
timestampUtcMs: action.timestampUtcMs,
|
|
3939
|
-
input: action.input,
|
|
3940
|
-
scope: action.scope
|
|
3941
|
-
};
|
|
3942
|
-
const signer = action.context?.signer;
|
|
3943
|
-
if (!signer?.signatures) return action.context ? {
|
|
3944
|
-
...base,
|
|
3945
|
-
context: action.context
|
|
3946
|
-
} : base;
|
|
3947
|
-
return {
|
|
3948
|
-
...base,
|
|
3949
|
-
context: {
|
|
3950
|
-
...action.context,
|
|
3951
|
-
signer: {
|
|
3952
|
-
...signer,
|
|
3953
|
-
signatures: signer.signatures.map((sig) => Array.isArray(sig) ? sig.join(", ") : sig)
|
|
3954
|
-
}
|
|
3955
|
-
}
|
|
3956
|
-
};
|
|
4785
|
+
return toTransportAction(action);
|
|
3957
4786
|
}
|
|
3958
4787
|
/**
|
|
3959
4788
|
* Serializes a SyncEnvelope for GraphQL transport.
|
|
@@ -3973,6 +4802,7 @@ function serializeEnvelope(envelope) {
|
|
|
3973
4802
|
hash: opWithContext.operation.hash,
|
|
3974
4803
|
skip: opWithContext.operation.skip,
|
|
3975
4804
|
error: opWithContext.operation.error,
|
|
4805
|
+
...opWithContext.operation.deniedReason !== void 0 ? { deniedReason: opWithContext.operation.deniedReason } : {},
|
|
3976
4806
|
id: opWithContext.operation.id,
|
|
3977
4807
|
action: serializeAction(opWithContext.operation.action)
|
|
3978
4808
|
},
|
|
@@ -3990,31 +4820,33 @@ function serializeEnvelope(envelope) {
|
|
|
3990
4820
|
};
|
|
3991
4821
|
}
|
|
3992
4822
|
/**
|
|
3993
|
-
*
|
|
3994
|
-
*
|
|
3995
|
-
*
|
|
3996
|
-
* This function converts them back to the expected [string, string, string, string, string] format.
|
|
4823
|
+
* `isDenied` tests strictly against undefined, so a null left in place would mark
|
|
4824
|
+
* every synced operation denied. The key is removed rather than set to undefined,
|
|
4825
|
+
* so an operation that arrived without it stays identical to the one sent.
|
|
3997
4826
|
*/
|
|
3998
|
-
function
|
|
3999
|
-
|
|
4000
|
-
|
|
4827
|
+
function normalizeAbsentFields(operation) {
|
|
4828
|
+
const wire = operation;
|
|
4829
|
+
if (wire.error !== null && wire.deniedReason !== null) return operation;
|
|
4830
|
+
const normalized = { ...operation };
|
|
4831
|
+
if (wire.error === null) delete normalized.error;
|
|
4832
|
+
if (wire.deniedReason === null) delete normalized.deniedReason;
|
|
4833
|
+
return normalized;
|
|
4001
4834
|
}
|
|
4002
|
-
/**
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
if (!signer?.signatures || signer.signatures.length === 0) return opWithContext;
|
|
4835
|
+
/** Restores signature tuples and null-valued optional fields to undefined. */
|
|
4836
|
+
function deserializeOperation(opWithContext) {
|
|
4837
|
+
const operation = normalizeAbsentFields(opWithContext.operation);
|
|
4838
|
+
const signer = operation.action.context?.signer;
|
|
4839
|
+
if (!signer?.signatures || signer.signatures.length === 0) return {
|
|
4840
|
+
...opWithContext,
|
|
4841
|
+
operation
|
|
4842
|
+
};
|
|
4011
4843
|
const deserializedSignatures = signer.signatures.map(deserializeSignature);
|
|
4012
4844
|
const deserializedOperation = {
|
|
4013
|
-
...
|
|
4845
|
+
...operation,
|
|
4014
4846
|
action: {
|
|
4015
|
-
...
|
|
4847
|
+
...operation.action,
|
|
4016
4848
|
context: {
|
|
4017
|
-
...
|
|
4849
|
+
...operation.action.context,
|
|
4018
4850
|
signer: {
|
|
4019
4851
|
...signer,
|
|
4020
4852
|
signatures: deserializedSignatures
|
|
@@ -4041,7 +4873,7 @@ function deserializeOperationSignatures(opWithContext) {
|
|
|
4041
4873
|
*/
|
|
4042
4874
|
function envelopesToSyncOperations(envelope, remoteName) {
|
|
4043
4875
|
if (!envelope.operations || envelope.operations.length === 0) return [];
|
|
4044
|
-
return batchOperationsByDocument(envelope.operations.map(
|
|
4876
|
+
return batchOperationsByDocument(envelope.operations.map(deserializeOperation)).map((batch) => {
|
|
4045
4877
|
return new SyncOperation(`syncop-${envelope.channelMeta.id}-${Date.now()}-${syncOpCounter++}`, envelope.key ?? "", (envelope.dependsOn ?? []).filter(Boolean), remoteName, batch.documentId, [batch.scope], batch.branch, batch.operations);
|
|
4046
4878
|
});
|
|
4047
4879
|
}
|
|
@@ -4053,6 +4885,12 @@ const getLatestAppliedOrdinal = (syncOps) => {
|
|
|
4053
4885
|
//#endregion
|
|
4054
4886
|
//#region src/sync/channels/gql-req-channel.ts
|
|
4055
4887
|
/**
|
|
4888
|
+
* Fields the auth projection added to the sync schema. A remote that predates
|
|
4889
|
+
* them rejects the whole query for naming one, so they are selected only while
|
|
4890
|
+
* the remote is known to serve them.
|
|
4891
|
+
*/
|
|
4892
|
+
const DECISION_FIELDS = ["deniedReason", "errorType"];
|
|
4893
|
+
/**
|
|
4056
4894
|
* GraphQL-based synchronization channel for network communication between reactors.
|
|
4057
4895
|
*/
|
|
4058
4896
|
var GqlRequestChannel = class {
|
|
@@ -4079,6 +4917,8 @@ var GqlRequestChannel = class {
|
|
|
4079
4917
|
isPushing = false;
|
|
4080
4918
|
pendingDrain = false;
|
|
4081
4919
|
receivingPages = false;
|
|
4920
|
+
/** Cleared for good the first time the remote rejects {@link DECISION_FIELDS}. */
|
|
4921
|
+
peerServesDecisionFields = true;
|
|
4082
4922
|
isRecovering = false;
|
|
4083
4923
|
connectionState = "connecting";
|
|
4084
4924
|
/** Latest unrecoverable error was an auth rejection; cleared on connect. */
|
|
@@ -4257,7 +5097,7 @@ var GqlRequestChannel = class {
|
|
|
4257
5097
|
const syncOps = [];
|
|
4258
5098
|
for (const dl of deadLetters) {
|
|
4259
5099
|
const syncOp = new SyncOperation(crypto.randomUUID(), dl.jobId, [], this.remoteName, dl.documentId, dl.scopes, dl.branch, []);
|
|
4260
|
-
syncOp.failed(new ChannelError(ChannelErrorSource.Outbox, new Error(dl.error)));
|
|
5100
|
+
syncOp.failed(new ChannelError(ChannelErrorSource.Outbox, new Error(dl.error), dl.errorType ?? void 0));
|
|
4261
5101
|
syncOps.push(syncOp);
|
|
4262
5102
|
}
|
|
4263
5103
|
this.deadLetter.add(...syncOps);
|
|
@@ -4349,7 +5189,44 @@ var GqlRequestChannel = class {
|
|
|
4349
5189
|
* Queries the remote GraphQL endpoint for sync envelopes.
|
|
4350
5190
|
*/
|
|
4351
5191
|
async pollSyncEnvelopes(ackOrdinal, latestOrdinal) {
|
|
4352
|
-
const
|
|
5192
|
+
const variables = {
|
|
5193
|
+
channelId: this.channelId,
|
|
5194
|
+
outboxAck: ackOrdinal,
|
|
5195
|
+
outboxLatest: latestOrdinal
|
|
5196
|
+
};
|
|
5197
|
+
let response;
|
|
5198
|
+
try {
|
|
5199
|
+
response = await this.executeGraphQL(this.pollQuery(this.peerServesDecisionFields), variables);
|
|
5200
|
+
} catch (error) {
|
|
5201
|
+
if (!this.rejectsDecisionFields(error)) throw error;
|
|
5202
|
+
this.logger.warn("Remote @channelId does not serve deniedReason/errorType; polling without them. The remote is on an older schema, so it has neither to report.", this.channelId);
|
|
5203
|
+
this.peerServesDecisionFields = false;
|
|
5204
|
+
response = await this.executeGraphQL(this.pollQuery(false), variables);
|
|
5205
|
+
}
|
|
5206
|
+
return {
|
|
5207
|
+
envelopes: response.pollSyncEnvelopes.envelopes,
|
|
5208
|
+
ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
|
|
5209
|
+
deadLetters: response.pollSyncEnvelopes.deadLetters ?? [],
|
|
5210
|
+
hasMore: response.pollSyncEnvelopes.hasMore
|
|
5211
|
+
};
|
|
5212
|
+
}
|
|
5213
|
+
/**
|
|
5214
|
+
* True when the remote rejected the query for naming a field it does not
|
|
5215
|
+
* have. Selecting an unknown field fails validation for the whole query, so
|
|
5216
|
+
* an unhandled one takes the channel's polling down until the process
|
|
5217
|
+
* restarts rather than degrading.
|
|
5218
|
+
*/
|
|
5219
|
+
rejectsDecisionFields(error) {
|
|
5220
|
+
if (!this.peerServesDecisionFields) return false;
|
|
5221
|
+
if (!(error instanceof GraphQLRequestError) || error.category !== "graphql") return false;
|
|
5222
|
+
return DECISION_FIELDS.some((field) => error.message.includes(field));
|
|
5223
|
+
}
|
|
5224
|
+
/**
|
|
5225
|
+
* The poll query. `withDecisionFields` selects the two fields added with the
|
|
5226
|
+
* auth projection; a remote on the previous schema is polled without them.
|
|
5227
|
+
*/
|
|
5228
|
+
pollQuery(withDecisionFields) {
|
|
5229
|
+
return `
|
|
4353
5230
|
query PollSyncEnvelopes($channelId: String!, $outboxAck: Int!, $outboxLatest: Int!) {
|
|
4354
5231
|
pollSyncEnvelopes(channelId: $channelId, outboxAck: $outboxAck, outboxLatest: $outboxLatest) {
|
|
4355
5232
|
envelopes {
|
|
@@ -4364,6 +5241,7 @@ var GqlRequestChannel = class {
|
|
|
4364
5241
|
hash
|
|
4365
5242
|
skip
|
|
4366
5243
|
error
|
|
5244
|
+
${withDecisionFields ? "deniedReason" : ""}
|
|
4367
5245
|
id
|
|
4368
5246
|
action {
|
|
4369
5247
|
id
|
|
@@ -4407,6 +5285,7 @@ var GqlRequestChannel = class {
|
|
|
4407
5285
|
deadLetters {
|
|
4408
5286
|
documentId
|
|
4409
5287
|
error
|
|
5288
|
+
${withDecisionFields ? "errorType" : ""}
|
|
4410
5289
|
jobId
|
|
4411
5290
|
branch
|
|
4412
5291
|
scopes
|
|
@@ -4416,18 +5295,6 @@ var GqlRequestChannel = class {
|
|
|
4416
5295
|
}
|
|
4417
5296
|
}
|
|
4418
5297
|
`;
|
|
4419
|
-
const variables = {
|
|
4420
|
-
channelId: this.channelId,
|
|
4421
|
-
outboxAck: ackOrdinal,
|
|
4422
|
-
outboxLatest: latestOrdinal
|
|
4423
|
-
};
|
|
4424
|
-
const response = await this.executeGraphQL(query, variables);
|
|
4425
|
-
return {
|
|
4426
|
-
envelopes: response.pollSyncEnvelopes.envelopes,
|
|
4427
|
-
ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
|
|
4428
|
-
deadLetters: response.pollSyncEnvelopes.deadLetters ?? [],
|
|
4429
|
-
hasMore: response.pollSyncEnvelopes.hasMore
|
|
4430
|
-
};
|
|
4431
5298
|
}
|
|
4432
5299
|
/**
|
|
4433
5300
|
* Registers or updates this channel on the remote server via GraphQL mutation.
|
|
@@ -4546,7 +5413,7 @@ var GqlRequestChannel = class {
|
|
|
4546
5413
|
if (error.statusCode !== void 0 && error.statusCode >= 500) return "recoverable";
|
|
4547
5414
|
return "unrecoverable";
|
|
4548
5415
|
case "parse": return "recoverable";
|
|
4549
|
-
case "graphql": return "unrecoverable";
|
|
5416
|
+
case "graphql": return isRecoverableGraphQLError(error) ? "recoverable" : "unrecoverable";
|
|
4550
5417
|
case "missing-data": return "unrecoverable";
|
|
4551
5418
|
}
|
|
4552
5419
|
}
|
|
@@ -4631,7 +5498,7 @@ var GqlRequestChannel = class {
|
|
|
4631
5498
|
throw new GraphQLRequestError(`Failed to parse GraphQL response: ${error instanceof Error ? error.message : String(error)}`, "parse");
|
|
4632
5499
|
}
|
|
4633
5500
|
this.logger.verbose("GQL response @channelId @operation status=@status data=@data errors=@errors", this.channelId, operationName, response.status, JSON.stringify(result.data), result.errors ? JSON.stringify(result.errors) : "none");
|
|
4634
|
-
if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql");
|
|
5501
|
+
if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql", void 0, result.errors.map((error) => error.extensions?.code));
|
|
4635
5502
|
if (!result.data) throw new GraphQLRequestError("GraphQL response missing data field", "missing-data");
|
|
4636
5503
|
return result.data;
|
|
4637
5504
|
}
|
|
@@ -4723,6 +5590,10 @@ var GqlRequestChannelFactory = class {
|
|
|
4723
5590
|
};
|
|
4724
5591
|
//#endregion
|
|
4725
5592
|
//#region src/sync/channels/gql-res-channel.ts
|
|
5593
|
+
/** Where a sync operation's run of ordinals begins. */
|
|
5594
|
+
function firstOrdinalOf$1(syncOp) {
|
|
5595
|
+
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
5596
|
+
}
|
|
4726
5597
|
/**
|
|
4727
5598
|
* This class is used server-side to accumulate inbox + outbox operations.
|
|
4728
5599
|
*
|
|
@@ -4738,6 +5609,8 @@ var GqlResponseChannel = class {
|
|
|
4738
5609
|
isShutdown;
|
|
4739
5610
|
lastPersistedInboxOrdinal = 0;
|
|
4740
5611
|
lastPersistedOutboxOrdinal = 0;
|
|
5612
|
+
evictedOutboxFloor = Number.POSITIVE_INFINITY;
|
|
5613
|
+
appliedOutboxOrdinal = 0;
|
|
4741
5614
|
connectionState = "connecting";
|
|
4742
5615
|
connectionStateCallbacks = /* @__PURE__ */ new Set();
|
|
4743
5616
|
constructor(logger, channelId, remoteName, cursorStorage) {
|
|
@@ -4750,18 +5623,11 @@ var GqlResponseChannel = class {
|
|
|
4750
5623
|
this.outbox = new Mailbox();
|
|
4751
5624
|
this.deadLetter = new Mailbox();
|
|
4752
5625
|
this.outbox.onRemoved((syncOps) => {
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
cursorType: "outbox",
|
|
4759
|
-
cursorOrdinal: maxOrdinal,
|
|
4760
|
-
lastSyncedAtUtcMs: Date.now()
|
|
4761
|
-
}).catch((error) => {
|
|
4762
|
-
this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
|
|
4763
|
-
});
|
|
4764
|
-
}
|
|
5626
|
+
this.rememberUnserved(syncOps);
|
|
5627
|
+
this.persistOutboxCursor(getLatestAppliedOrdinal(syncOps));
|
|
5628
|
+
});
|
|
5629
|
+
this.outbox.onAdded((syncOps) => {
|
|
5630
|
+
this.forgetEvictedBelow(syncOps);
|
|
4765
5631
|
});
|
|
4766
5632
|
this.inbox.onRemoved((syncOps) => {
|
|
4767
5633
|
const maxOrdinal = getLatestAppliedOrdinal(syncOps);
|
|
@@ -4803,6 +5669,17 @@ var GqlResponseChannel = class {
|
|
|
4803
5669
|
}
|
|
4804
5670
|
/** Response channels are push-driven; resolvers populate mailboxes directly. */
|
|
4805
5671
|
triggerPull() {}
|
|
5672
|
+
async init() {
|
|
5673
|
+
const cursors = await this.cursorStorage.list(this.remoteName);
|
|
5674
|
+
const inboxOrdinal = cursors.find((c) => c.cursorType === "inbox")?.cursorOrdinal ?? 0;
|
|
5675
|
+
const outboxOrdinal = cursors.find((c) => c.cursorType === "outbox")?.cursorOrdinal ?? 0;
|
|
5676
|
+
this.inbox.init(inboxOrdinal);
|
|
5677
|
+
this.outbox.init(outboxOrdinal);
|
|
5678
|
+
this.lastPersistedInboxOrdinal = inboxOrdinal;
|
|
5679
|
+
this.lastPersistedOutboxOrdinal = outboxOrdinal;
|
|
5680
|
+
this.appliedOutboxOrdinal = outboxOrdinal;
|
|
5681
|
+
this.transitionConnectionState("connected");
|
|
5682
|
+
}
|
|
4806
5683
|
transitionConnectionState(next) {
|
|
4807
5684
|
if (this.connectionState === next) return;
|
|
4808
5685
|
this.connectionState = next;
|
|
@@ -4813,15 +5690,68 @@ var GqlResponseChannel = class {
|
|
|
4813
5690
|
this.logger.error("Connection state change callback error: @Error", error);
|
|
4814
5691
|
}
|
|
4815
5692
|
}
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
5693
|
+
/**
|
|
5694
|
+
* Records the ordinals of entries that left the outbox without being served,
|
|
5695
|
+
* so the cursor cannot advance past them.
|
|
5696
|
+
*
|
|
5697
|
+
* An entry can leave unserved because a bound evicted it, and an evicted entry
|
|
5698
|
+
* is exactly one this channel intends to re-derive: it is still owed to the
|
|
5699
|
+
* remote. Remembering the floor across the whole run rather than only while
|
|
5700
|
+
* the entry is present is what makes that true after a later ack would
|
|
5701
|
+
* otherwise have swept the cursor past it.
|
|
5702
|
+
*/
|
|
5703
|
+
rememberUnserved(syncOps) {
|
|
5704
|
+
for (const syncOp of syncOps) {
|
|
5705
|
+
if (syncOp.status === SyncOperationStatus.Applied) continue;
|
|
5706
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5707
|
+
if (first > 0 && first < this.evictedOutboxFloor) this.evictedOutboxFloor = first;
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5710
|
+
/**
|
|
5711
|
+
* Persists the outbox cursor, never past an operation this remote has not
|
|
5712
|
+
* been served.
|
|
5713
|
+
*
|
|
5714
|
+
* The cursor is where a restart resumes deriving the outbox from, so an
|
|
5715
|
+
* ordinal persisted past an unserved entry loses that entry for good: the
|
|
5716
|
+
* rebuild starts beyond it and nothing else remembers it was owed. Acks
|
|
5717
|
+
* arrive out of order with respect to what is withheld -- a later entry can
|
|
5718
|
+
* be acknowledged while an earlier one is still being withheld from this
|
|
5719
|
+
* subject -- so the applied high-water mark alone is not a safe cursor.
|
|
5720
|
+
*/
|
|
5721
|
+
persistOutboxCursor(appliedOrdinal) {
|
|
5722
|
+
this.appliedOutboxOrdinal = Math.max(this.appliedOutboxOrdinal, appliedOrdinal);
|
|
5723
|
+
const ordinal = Math.min(this.appliedOutboxOrdinal, this.unservedFloor() - 1);
|
|
5724
|
+
if (ordinal <= this.lastPersistedOutboxOrdinal) return;
|
|
5725
|
+
this.lastPersistedOutboxOrdinal = ordinal;
|
|
5726
|
+
this.cursorStorage.upsert({
|
|
5727
|
+
remoteName: this.remoteName,
|
|
5728
|
+
cursorType: "outbox",
|
|
5729
|
+
cursorOrdinal: ordinal,
|
|
5730
|
+
lastSyncedAtUtcMs: Date.now()
|
|
5731
|
+
}).catch((error) => {
|
|
5732
|
+
this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
|
|
5733
|
+
});
|
|
5734
|
+
}
|
|
5735
|
+
/** Drops the evicted floor once the entries it stood for are queued again. */
|
|
5736
|
+
forgetEvictedBelow(syncOps) {
|
|
5737
|
+
if (this.evictedOutboxFloor === Number.POSITIVE_INFINITY) return;
|
|
5738
|
+
for (const syncOp of syncOps) {
|
|
5739
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5740
|
+
if (first > 0 && first <= this.evictedOutboxFloor) {
|
|
5741
|
+
this.evictedOutboxFloor = Number.POSITIVE_INFINITY;
|
|
5742
|
+
return;
|
|
5743
|
+
}
|
|
5744
|
+
}
|
|
5745
|
+
}
|
|
5746
|
+
/** The lowest ordinal still owed to this remote, evicted or still queued. */
|
|
5747
|
+
unservedFloor() {
|
|
5748
|
+
let floor = this.evictedOutboxFloor;
|
|
5749
|
+
for (const syncOp of this.outbox.items) {
|
|
5750
|
+
if (syncOp.status === SyncOperationStatus.Applied) continue;
|
|
5751
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5752
|
+
if (first > 0 && first < floor) floor = first;
|
|
5753
|
+
}
|
|
5754
|
+
return floor;
|
|
4825
5755
|
}
|
|
4826
5756
|
};
|
|
4827
5757
|
//#endregion
|
|
@@ -4909,7 +5839,8 @@ function rowToDeadLetterRecord(row) {
|
|
|
4909
5839
|
branch: row.branch,
|
|
4910
5840
|
operations: row.operations,
|
|
4911
5841
|
errorSource: row.error_source,
|
|
4912
|
-
errorMessage: row.error_message
|
|
5842
|
+
errorMessage: row.error_message,
|
|
5843
|
+
errorType: row.error_type ?? "UNCLASSIFIED"
|
|
4913
5844
|
};
|
|
4914
5845
|
}
|
|
4915
5846
|
function deadLetterRecordToRow(record) {
|
|
@@ -4923,7 +5854,8 @@ function deadLetterRecordToRow(record) {
|
|
|
4923
5854
|
branch: record.branch,
|
|
4924
5855
|
operations: JSON.stringify(record.operations),
|
|
4925
5856
|
error_source: record.errorSource,
|
|
4926
|
-
error_message: record.errorMessage
|
|
5857
|
+
error_message: record.errorMessage,
|
|
5858
|
+
error_type: record.errorType
|
|
4927
5859
|
};
|
|
4928
5860
|
}
|
|
4929
5861
|
/**
|
|
@@ -4980,9 +5912,11 @@ var KyselySyncDeadLetterStorage = class {
|
|
|
4980
5912
|
}
|
|
4981
5913
|
async listQuarantinedDocumentIds(signal) {
|
|
4982
5914
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
4983
|
-
const rows = await this.db.selectFrom("sync_dead_letters").select("document_id").distinct().execute();
|
|
5915
|
+
const rows = await this.db.selectFrom("sync_dead_letters").select(["document_id", "error_type"]).distinct().execute();
|
|
4984
5916
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
4985
|
-
|
|
5917
|
+
const quarantined = /* @__PURE__ */ new Set();
|
|
5918
|
+
for (const row of rows) if (quarantinesDocument(row.error_type ?? "UNCLASSIFIED")) quarantined.add(row.document_id);
|
|
5919
|
+
return [...quarantined];
|
|
4986
5920
|
}
|
|
4987
5921
|
};
|
|
4988
5922
|
//#endregion
|
|
@@ -5001,7 +5935,10 @@ function rowToRemoteRecord(row) {
|
|
|
5001
5935
|
scope: row.filter_scopes ?? [],
|
|
5002
5936
|
branch: row.filter_branch
|
|
5003
5937
|
},
|
|
5004
|
-
options: {
|
|
5938
|
+
options: {
|
|
5939
|
+
sinceTimestampUtcMs: "0",
|
|
5940
|
+
boundAddress: row.bound_address ?? void 0
|
|
5941
|
+
},
|
|
5005
5942
|
status: {
|
|
5006
5943
|
push: {
|
|
5007
5944
|
state: row.push_state,
|
|
@@ -5041,7 +5978,8 @@ function remoteRecordToRow(remote) {
|
|
|
5041
5978
|
pull_state: remote.status.pull.state,
|
|
5042
5979
|
pull_last_success_utc_ms: remote.status.pull.lastSuccessUtcMs ? new Date(remote.status.pull.lastSuccessUtcMs).toISOString() : null,
|
|
5043
5980
|
pull_last_failure_utc_ms: remote.status.pull.lastFailureUtcMs ? new Date(remote.status.pull.lastFailureUtcMs).toISOString() : null,
|
|
5044
|
-
pull_failure_count: remote.status.pull.failureCount
|
|
5981
|
+
pull_failure_count: remote.status.pull.failureCount,
|
|
5982
|
+
bound_address: remote.options.boundAddress ?? null
|
|
5045
5983
|
};
|
|
5046
5984
|
}
|
|
5047
5985
|
var KyselySyncRemoteStorage = class {
|
|
@@ -5409,9 +6347,14 @@ var OutboxMode = /* @__PURE__ */ function(OutboxMode) {
|
|
|
5409
6347
|
}(OutboxMode || {});
|
|
5410
6348
|
const defaultSyncManagerConfig = {
|
|
5411
6349
|
maxDeadLettersPerRemote: 100,
|
|
5412
|
-
maxInboxBatchSize: 32
|
|
6350
|
+
maxInboxBatchSize: 32,
|
|
6351
|
+
maxHeldOperationsPerRemote: 1e4
|
|
5413
6352
|
};
|
|
5414
6353
|
const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
|
|
6354
|
+
/** Where a sync operation's run of ordinals begins. */
|
|
6355
|
+
function firstOrdinalOf(syncOp) {
|
|
6356
|
+
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
6357
|
+
}
|
|
5415
6358
|
var SyncManager = class {
|
|
5416
6359
|
logger;
|
|
5417
6360
|
remoteStorage;
|
|
@@ -5435,6 +6378,7 @@ var SyncManager = class {
|
|
|
5435
6378
|
quarantinedDocumentIds = /* @__PURE__ */ new Set();
|
|
5436
6379
|
backfillAbortControllers = /* @__PURE__ */ new Map();
|
|
5437
6380
|
planKeyToJobUuid = /* @__PURE__ */ new Map();
|
|
6381
|
+
evictedOutboxFloors = /* @__PURE__ */ new Map();
|
|
5438
6382
|
lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
|
|
5439
6383
|
inboxChunkChain = Promise.resolve();
|
|
5440
6384
|
constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, config = {}) {
|
|
@@ -5542,6 +6486,28 @@ var SyncManager = class {
|
|
|
5542
6486
|
for (const remote of this.remotes.values()) if (remote.meta.id === id) return remote;
|
|
5543
6487
|
throw new Error(`Remote with id '${id}' does not exist`);
|
|
5544
6488
|
}
|
|
6489
|
+
async bindRemote(id, boundAddress) {
|
|
6490
|
+
const remote = this.getById(id);
|
|
6491
|
+
const bound = remote.meta.options.boundAddress;
|
|
6492
|
+
if (bound === boundAddress) return;
|
|
6493
|
+
if (bound !== void 0) throw new Error(`Remote with id '${id}' is already bound to another address`);
|
|
6494
|
+
remote.meta.options = {
|
|
6495
|
+
...remote.meta.options,
|
|
6496
|
+
boundAddress
|
|
6497
|
+
};
|
|
6498
|
+
await this.remoteStorage.upsert({
|
|
6499
|
+
id: remote.meta.id,
|
|
6500
|
+
name: remote.meta.name,
|
|
6501
|
+
collectionId: remote.meta.collectionId,
|
|
6502
|
+
channelConfig: remote.meta.channelConfig,
|
|
6503
|
+
filter: remote.meta.filter,
|
|
6504
|
+
options: remote.meta.options,
|
|
6505
|
+
status: {
|
|
6506
|
+
push: createIdleHealth(),
|
|
6507
|
+
pull: createIdleHealth()
|
|
6508
|
+
}
|
|
6509
|
+
});
|
|
6510
|
+
}
|
|
5545
6511
|
async add(name, collectionId, channelConfig, filter = {
|
|
5546
6512
|
documentId: [],
|
|
5547
6513
|
scope: [],
|
|
@@ -5655,7 +6621,8 @@ var SyncManager = class {
|
|
|
5655
6621
|
remote.channel.deadLetter.onAdded((syncOps) => {
|
|
5656
6622
|
for (const syncOp of syncOps) {
|
|
5657
6623
|
this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
|
|
5658
|
-
|
|
6624
|
+
const errorType = syncOperationErrorType(syncOp.error);
|
|
6625
|
+
if (quarantinesDocument(errorType)) this.quarantinedDocumentIds.add(syncOp.documentId);
|
|
5659
6626
|
const record = {
|
|
5660
6627
|
id: syncOp.id,
|
|
5661
6628
|
jobId: syncOp.jobId,
|
|
@@ -5666,7 +6633,8 @@ var SyncManager = class {
|
|
|
5666
6633
|
branch: syncOp.branch,
|
|
5667
6634
|
operations: syncOp.operations,
|
|
5668
6635
|
errorSource: syncOp.error?.source ?? ChannelErrorSource.None,
|
|
5669
|
-
errorMessage: syncOp.error?.error.message ?? "unknown"
|
|
6636
|
+
errorMessage: syncOp.error?.error.message ?? "unknown",
|
|
6637
|
+
errorType
|
|
5670
6638
|
};
|
|
5671
6639
|
this.deadLetterStorage.add(record).catch((err) => {
|
|
5672
6640
|
this.logger.error("Failed to persist dead letter (@id, @error)", record.id, err instanceof Error ? err.message : String(err));
|
|
@@ -5676,7 +6644,8 @@ var SyncManager = class {
|
|
|
5676
6644
|
jobId: record.jobId,
|
|
5677
6645
|
remoteName: record.remoteName,
|
|
5678
6646
|
documentId: record.documentId,
|
|
5679
|
-
errorSource: record.errorSource
|
|
6647
|
+
errorSource: record.errorSource,
|
|
6648
|
+
errorType: record.errorType
|
|
5680
6649
|
}).catch(() => {});
|
|
5681
6650
|
}
|
|
5682
6651
|
const items = remote.channel.deadLetter.items;
|
|
@@ -5703,7 +6672,7 @@ var SyncManager = class {
|
|
|
5703
6672
|
const syncOps = [];
|
|
5704
6673
|
for (const record of records) {
|
|
5705
6674
|
const syncOp = new SyncOperation(record.id, record.jobId, record.jobDependencies, record.remoteName, record.documentId, record.scopes, record.branch, record.operations);
|
|
5706
|
-
syncOp.failed(new ChannelError(record.errorSource, new Error(record.errorMessage)));
|
|
6675
|
+
syncOp.failed(new ChannelError(record.errorSource, new Error(record.errorMessage), record.errorType));
|
|
5707
6676
|
syncOps.push(syncOp);
|
|
5708
6677
|
}
|
|
5709
6678
|
remote.channel.deadLetter.add(...syncOps);
|
|
@@ -5784,8 +6753,7 @@ var SyncManager = class {
|
|
|
5784
6753
|
if (completedJobInfo.status === JobStatus.FAILED) {
|
|
5785
6754
|
const errorMessage = completedJobInfo.error?.message || "Unknown error";
|
|
5786
6755
|
this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
|
|
5787
|
-
|
|
5788
|
-
syncOp.failed(error);
|
|
6756
|
+
syncOp.failed(this.inboxFailure(completedJobInfo.error));
|
|
5789
6757
|
remote.channel.deadLetter.add(syncOp);
|
|
5790
6758
|
} else syncOp.executed();
|
|
5791
6759
|
remote.channel.inbox.remove(syncOp);
|
|
@@ -5862,17 +6830,79 @@ var SyncManager = class {
|
|
|
5862
6830
|
}
|
|
5863
6831
|
if (this.isShutdown) return;
|
|
5864
6832
|
if (completedJobInfo.status === JobStatus.FAILED) {
|
|
5865
|
-
|
|
5866
|
-
const channelError = new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${errorMessage}`));
|
|
5867
|
-
syncOp.failed(channelError);
|
|
6833
|
+
syncOp.failed(this.inboxFailure(completedJobInfo.error));
|
|
5868
6834
|
remote.channel.deadLetter.add(syncOp);
|
|
5869
6835
|
} else syncOp.executed();
|
|
5870
6836
|
remote.channel.inbox.remove(syncOp);
|
|
5871
6837
|
}
|
|
5872
6838
|
}
|
|
6839
|
+
/**
|
|
6840
|
+
* The dead letter for a load job the executor failed.
|
|
6841
|
+
*
|
|
6842
|
+
* The classification is passed explicitly because it cannot be recovered
|
|
6843
|
+
* downstream: the wrapper carries the failure's message, not the failure, so
|
|
6844
|
+
* deriving it from the wrapper's own name would classify every one of these
|
|
6845
|
+
* as unclassified and quarantine the document. A held auth operation must
|
|
6846
|
+
* keep syncing, because reconciling the two policies needs the traffic a
|
|
6847
|
+
* quarantine would stop.
|
|
6848
|
+
*/
|
|
6849
|
+
inboxFailure(error) {
|
|
6850
|
+
const message = error?.message || "Unknown error";
|
|
6851
|
+
return new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), classifyJobFailure(error?.name ?? "Error"));
|
|
6852
|
+
}
|
|
6853
|
+
/**
|
|
6854
|
+
* Where the next outbox derivation starts: at the evicted floor when there is
|
|
6855
|
+
* room to take those entries back, and at the caller's cursor otherwise.
|
|
6856
|
+
*
|
|
6857
|
+
* The floor is forgotten as it is folded in. If the refill overflows again the
|
|
6858
|
+
* eviction records a new one, and if it does not, the entries are queued and
|
|
6859
|
+
* the outbox itself holds the cursor down.
|
|
6860
|
+
*/
|
|
6861
|
+
refillOrdinal(remote, ackOrdinal) {
|
|
6862
|
+
const floor = this.evictedOutboxFloors.get(remote.meta.name);
|
|
6863
|
+
if (floor === void 0) return ackOrdinal;
|
|
6864
|
+
if (this.outboxOperationCount(remote) >= this.config.maxHeldOperationsPerRemote) return ackOrdinal;
|
|
6865
|
+
this.evictedOutboxFloors.delete(remote.meta.name);
|
|
6866
|
+
return Math.min(ackOrdinal, floor - 1);
|
|
6867
|
+
}
|
|
6868
|
+
/**
|
|
6869
|
+
* Evicts the newest entries of an outbox that has grown past its bound.
|
|
6870
|
+
*
|
|
6871
|
+
* Only entries that have never been taken up for transport are eligible, which
|
|
6872
|
+
* is what makes this a bound on withheld work rather than a way to drop
|
|
6873
|
+
* something already in flight. One entry is always kept, so an outbox whose
|
|
6874
|
+
* first entry is larger than the whole cap still makes progress.
|
|
6875
|
+
*/
|
|
6876
|
+
evictPastOutboxBound(remote) {
|
|
6877
|
+
const cap = this.config.maxHeldOperationsPerRemote;
|
|
6878
|
+
const queued = remote.channel.outbox.items.filter((syncOp) => syncOp.status === SyncOperationStatus.Unknown).sort((a, b) => firstOrdinalOf(a) - firstOrdinalOf(b));
|
|
6879
|
+
let kept = 0;
|
|
6880
|
+
let cut = queued.length;
|
|
6881
|
+
for (let i = 0; i < queued.length; i++) {
|
|
6882
|
+
const size = queued[i].operations.length;
|
|
6883
|
+
if (i > 0 && kept + size > cap) {
|
|
6884
|
+
cut = i;
|
|
6885
|
+
break;
|
|
6886
|
+
}
|
|
6887
|
+
kept += size;
|
|
6888
|
+
}
|
|
6889
|
+
const evicted = queued.slice(cut);
|
|
6890
|
+
if (evicted.length === 0) return;
|
|
6891
|
+
const floor = firstOrdinalOf(evicted[0]);
|
|
6892
|
+
const known = this.evictedOutboxFloors.get(remote.meta.name);
|
|
6893
|
+
this.evictedOutboxFloors.set(remote.meta.name, known === void 0 ? floor : Math.min(known, floor));
|
|
6894
|
+
this.logger.warn("Outbox for @RemoteName is past its bound of @Cap operations; evicting @Count entries from ordinal @Floor, to be derived again once it drains", remote.meta.name, cap, evicted.length, floor);
|
|
6895
|
+
remote.channel.outbox.remove(...evicted);
|
|
6896
|
+
}
|
|
6897
|
+
outboxOperationCount(remote) {
|
|
6898
|
+
let count = 0;
|
|
6899
|
+
for (const syncOp of remote.channel.outbox.items) count += syncOp.operations.length;
|
|
6900
|
+
return count;
|
|
6901
|
+
}
|
|
5873
6902
|
async updateOutbox(remote, ackOrdinal, mode = OutboxMode.Backfill, signal) {
|
|
5874
6903
|
const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
|
|
5875
|
-
|
|
6904
|
+
const startOrdinal = this.refillOrdinal(remote, ackOrdinal);
|
|
6905
|
+
let maxOrdinal = startOrdinal;
|
|
5876
6906
|
const lastJobByDoc = /* @__PURE__ */ new Map();
|
|
5877
6907
|
let prevChainJobId;
|
|
5878
6908
|
const sinceTimestamp = remote.meta.options.sinceTimestampUtcMs;
|
|
@@ -5892,8 +6922,9 @@ var SyncManager = class {
|
|
|
5892
6922
|
if (mode === OutboxMode.BatchTriggered) prevChainJobId = jobId;
|
|
5893
6923
|
}
|
|
5894
6924
|
remote.channel.outbox.add(...syncOps);
|
|
6925
|
+
this.evictPastOutboxBound(remote);
|
|
5895
6926
|
};
|
|
5896
|
-
let page = await this.operationIndex.find(remote.meta.collectionId.key,
|
|
6927
|
+
let page = await this.operationIndex.find(remote.meta.collectionId.key, startOrdinal, { excludeSourceRemote: remote.meta.name }, void 0, composedSignal);
|
|
5897
6928
|
let carry = [];
|
|
5898
6929
|
let hasMore;
|
|
5899
6930
|
do {
|
|
@@ -5909,9 +6940,17 @@ var SyncManager = class {
|
|
|
5909
6940
|
operations = operations.filter((op) => !this.quarantinedDocumentIds.has(op.context.documentId));
|
|
5910
6941
|
hasMore = !!page.next;
|
|
5911
6942
|
if (operations.length > 0) {
|
|
6943
|
+
const created = /* @__PURE__ */ new Set();
|
|
6944
|
+
for (const entry of operations) if (entry.operation.action.type === "CREATE_DOCUMENT") created.add(entry.context.documentId);
|
|
5912
6945
|
operations.sort((a, b) => {
|
|
5913
6946
|
if (a.context.documentId !== b.context.documentId) return a.context.documentId < b.context.documentId ? -1 : 1;
|
|
5914
|
-
if (a.context.scope !== b.context.scope)
|
|
6947
|
+
if (a.context.scope !== b.context.scope) {
|
|
6948
|
+
if (created.has(a.context.documentId)) {
|
|
6949
|
+
if (a.context.scope === "document") return -1;
|
|
6950
|
+
if (b.context.scope === "document") return 1;
|
|
6951
|
+
}
|
|
6952
|
+
return a.context.scope < b.context.scope ? -1 : 1;
|
|
6953
|
+
}
|
|
5915
6954
|
return a.context.ordinal - b.context.ordinal;
|
|
5916
6955
|
});
|
|
5917
6956
|
if (hasMore) {
|
|
@@ -5958,6 +6997,10 @@ var SyncBuilder = class {
|
|
|
5958
6997
|
this.config.maxInboxBatchSize = limit;
|
|
5959
6998
|
return this;
|
|
5960
6999
|
}
|
|
7000
|
+
withMaxHeldOperationsPerRemote(limit) {
|
|
7001
|
+
this.config.maxHeldOperationsPerRemote = limit;
|
|
7002
|
+
return this;
|
|
7003
|
+
}
|
|
5961
7004
|
build(reactor, logger, operationIndex, eventBus, db, driveContainerTypes) {
|
|
5962
7005
|
return this.buildModule(reactor, logger, operationIndex, eventBus, db, driveContainerTypes).syncManager;
|
|
5963
7006
|
}
|
|
@@ -6190,7 +7233,7 @@ var Reactor = class {
|
|
|
6190
7233
|
documentId: document.header.id,
|
|
6191
7234
|
model: document.header.documentType,
|
|
6192
7235
|
fromVersion: 0,
|
|
6193
|
-
toVersion: document.state.document
|
|
7236
|
+
toVersion: normalizeDocumentModelVersion(document.state.document?.version),
|
|
6194
7237
|
initialState: document.state
|
|
6195
7238
|
})];
|
|
6196
7239
|
if (signer) actions = await signActions(actions, signer, signal);
|
|
@@ -6572,7 +7615,10 @@ var ReactorBuilder = class {
|
|
|
6572
7615
|
logger;
|
|
6573
7616
|
documentModelSources = [];
|
|
6574
7617
|
upgradeManifests = [];
|
|
6575
|
-
features = {
|
|
7618
|
+
features = {
|
|
7619
|
+
legacyStorageEnabled: false,
|
|
7620
|
+
legacyProcessorIds: true
|
|
7621
|
+
};
|
|
6576
7622
|
readModels = [];
|
|
6577
7623
|
readModelFactories = [];
|
|
6578
7624
|
executorManager;
|
|
@@ -6769,6 +7815,18 @@ var ReactorBuilder = class {
|
|
|
6769
7815
|
}
|
|
6770
7816
|
async buildModule() {
|
|
6771
7817
|
if (!this.logger) this.logger = new ConsoleLogger(["reactor"]);
|
|
7818
|
+
const featureFlags = resolveFeatureFlags(this.executorConfig.featureFlags);
|
|
7819
|
+
if (this.projectionShardConfig !== void 0 && this.readModelFactories.length > 0) throw new Error("withProjectionShards does not support read models registered through withReadModelFactory; projection workers cannot receive host-only factory dependencies");
|
|
7820
|
+
if (this.projectionShardConfig !== void 0 && this.readModels.length > 0) throw new Error("withProjectionShards does not support read models registered through withReadModel; projection workers build their own read models from the shard config and would silently omit these");
|
|
7821
|
+
if (this.projectionShardConfig !== void 0) {
|
|
7822
|
+
const named = [...this.projectionShardConfig.preReadyKinds, ...this.projectionShardConfig.postReadyKinds];
|
|
7823
|
+
const missing = BUILT_IN_READ_MODEL_KINDS.filter((kind) => !named.includes(kind));
|
|
7824
|
+
const duplicated = BUILT_IN_READ_MODEL_KINDS.filter((kind) => named.filter((entry) => entry === kind).length > 1);
|
|
7825
|
+
if (missing.length > 0 || duplicated.length > 0) {
|
|
7826
|
+
const problems = [missing.length > 0 ? `never named: ${missing.join(", ")} (would be indexed by no shard, leaving those reads permanently stale and their consistency-token waits unresolvable)` : void 0, duplicated.length > 0 ? `named more than once: ${duplicated.join(", ")} (would be indexed twice per operation)` : void 0].filter((problem) => problem !== void 0);
|
|
7827
|
+
throw new Error(`withProjectionShards requires preReadyKinds and postReadyKinds to name each built-in read model (${BUILT_IN_READ_MODEL_KINDS.join(", ")}) exactly once between them; ${problems.join("; ")}`);
|
|
7828
|
+
}
|
|
7829
|
+
}
|
|
6772
7830
|
const resolvedSources = await resolveModelSources(this.documentModelSources);
|
|
6773
7831
|
if (this.workerPool) {
|
|
6774
7832
|
if (resolvedSources.manifest.length === 0) throw new Error("withWorkerPool requires at least one worker-importable document-model source ({ filePath } or { packageName }).");
|
|
@@ -6818,15 +7876,15 @@ var ReactorBuilder = class {
|
|
|
6818
7876
|
if (pool.db === void 0) throw new Error("unreachable: worker pool configured without db or factory");
|
|
6819
7877
|
factory = await this.createDefaultWorkerFactory(pool.numWorkers, pool.db, pool.verifier);
|
|
6820
7878
|
}
|
|
6821
|
-
const poolManager = new WorkerPoolJobExecutorManager(factory, eventBus, queue, jobTracker, this.logger, resolver,
|
|
7879
|
+
const poolManager = new WorkerPoolJobExecutorManager(factory, eventBus, queue, jobTracker, this.logger, resolver, operationIndex, this.executorConfig.jobTimeoutMs, this.executorConfig.deferredJobTtlMs);
|
|
6822
7880
|
executorManager = poolManager;
|
|
6823
7881
|
executorStartCount = pool.numWorkers;
|
|
6824
7882
|
if (resolver instanceof DocumentModelResolver) resolver.setBroadcastHook((entry) => poolManager.loadModel(entry));
|
|
6825
|
-
} else executorManager = new SimpleJobExecutorManager(() => new SimpleJobExecutor(this.logger, documentModelRegistry, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, collectionMembershipCache, this.driveContainerTypes, this.executorConfig, this.signatureVerifier, executionScope), eventBus, queue, jobTracker, this.logger, resolver, this.executorConfig.jobTimeoutMs);
|
|
7883
|
+
} else executorManager = new SimpleJobExecutorManager(() => new SimpleJobExecutor(this.logger, documentModelRegistry, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, collectionMembershipCache, this.driveContainerTypes, this.executorConfig, this.signatureVerifier, executionScope), eventBus, queue, jobTracker, this.logger, resolver, this.executorConfig.jobTimeoutMs, this.executorConfig.deferredJobTtlMs);
|
|
6826
7884
|
await executorManager.start(executorStartCount);
|
|
6827
7885
|
const readModelInstances = Array.from(new Set([...this.readModels]));
|
|
6828
7886
|
const documentViewConsistencyTracker = new ConsistencyTracker();
|
|
6829
|
-
const documentView = new KyselyDocumentView(database, operationStore, operationIndex, writeCache, documentViewConsistencyTracker);
|
|
7887
|
+
const documentView = new KyselyDocumentView(database, operationStore, operationIndex, writeCache, documentViewConsistencyTracker, featureFlags.documentDecisions);
|
|
6830
7888
|
try {
|
|
6831
7889
|
await documentView.init();
|
|
6832
7890
|
} catch (error) {
|
|
@@ -6844,7 +7902,7 @@ var ReactorBuilder = class {
|
|
|
6844
7902
|
const subscriptionManager = new ReactorSubscriptionManager(new DefaultSubscriptionErrorHandler());
|
|
6845
7903
|
const subscriptionNotificationReadModel = new SubscriptionNotificationReadModel(subscriptionManager, documentView);
|
|
6846
7904
|
const processorManagerConsistencyTracker = new ConsistencyTracker();
|
|
6847
|
-
const processorManager = new ProcessorManager(database, operationIndex, writeCache, processorManagerConsistencyTracker, this.logger, this.driveContainerTypes);
|
|
7905
|
+
const processorManager = new ProcessorManager(database, operationIndex, writeCache, processorManagerConsistencyTracker, this.logger, this.driveContainerTypes, { legacyProcessorIds: this.features.legacyProcessorIds !== false });
|
|
6848
7906
|
try {
|
|
6849
7907
|
await processorManager.init();
|
|
6850
7908
|
} catch (error) {
|
|
@@ -6852,13 +7910,17 @@ var ReactorBuilder = class {
|
|
|
6852
7910
|
}
|
|
6853
7911
|
for (const factory of this.readModelFactories) {
|
|
6854
7912
|
const readModel = await factory({
|
|
7913
|
+
documentModelRegistry,
|
|
6855
7914
|
operationIndex,
|
|
6856
7915
|
writeCache,
|
|
6857
7916
|
processorManagerConsistencyTracker
|
|
6858
7917
|
});
|
|
6859
7918
|
readModelInstances.push(readModel);
|
|
6860
7919
|
}
|
|
6861
|
-
const readModelCoordinator = this.readModelCoordinator ? this.readModelCoordinator : this.projectionShardConfig ? await this.createProjectionShardManager(this.projectionShardConfig, eventBus
|
|
7920
|
+
const readModelCoordinator = this.readModelCoordinator ? this.readModelCoordinator : this.projectionShardConfig ? await this.createProjectionShardManager(this.projectionShardConfig, eventBus, {
|
|
7921
|
+
"document-view": documentViewConsistencyTracker,
|
|
7922
|
+
"document-indexer": documentIndexerConsistencyTracker
|
|
7923
|
+
}) : new ReadModelCoordinator(eventBus, readModelInstances, [subscriptionNotificationReadModel, processorManager]);
|
|
6862
7924
|
const reactor = new Reactor(this.logger, documentModelRegistry, queue, jobTracker, readModelCoordinator, this.features, documentView, documentIndexer, operationStore, eventBus, executorManager);
|
|
6863
7925
|
let syncModule = void 0;
|
|
6864
7926
|
if (this.channelScheme) {
|
|
@@ -6869,9 +7931,15 @@ var ReactorBuilder = class {
|
|
|
6869
7931
|
syncModule = this.syncBuilder.buildModule(reactor, this.logger, operationIndex, eventBus, database, this.driveContainerTypes);
|
|
6870
7932
|
await syncModule.syncManager.startup();
|
|
6871
7933
|
}
|
|
7934
|
+
let groupReevaluationTrigger;
|
|
7935
|
+
if (featureFlags.authGroups) {
|
|
7936
|
+
groupReevaluationTrigger = new GroupReevaluationTrigger(this.logger, eventBus, queue, operationIndex);
|
|
7937
|
+
groupReevaluationTrigger.startup();
|
|
7938
|
+
}
|
|
6872
7939
|
const module = {
|
|
6873
7940
|
eventBus,
|
|
6874
7941
|
documentModelRegistry,
|
|
7942
|
+
featureFlags,
|
|
6875
7943
|
queue,
|
|
6876
7944
|
jobTracker,
|
|
6877
7945
|
executorManager,
|
|
@@ -6890,6 +7958,7 @@ var ReactorBuilder = class {
|
|
|
6890
7958
|
processorManagerConsistencyTracker,
|
|
6891
7959
|
syncModule,
|
|
6892
7960
|
reactor,
|
|
7961
|
+
groupReevaluationTrigger,
|
|
6893
7962
|
pools: this.instrumentedPools
|
|
6894
7963
|
};
|
|
6895
7964
|
if (this.signalHandlersEnabled) this.attachSignalHandlers(module);
|
|
@@ -6914,8 +7983,14 @@ var ReactorBuilder = class {
|
|
|
6914
7983
|
* injected via {@link withProjectionWorkerFactory}. Calls
|
|
6915
7984
|
* `manager.startup()` so all N workers reach READY before the reactor
|
|
6916
7985
|
* is returned to the caller.
|
|
7986
|
+
*
|
|
7987
|
+
* @param consistencyTrackers The host's trackers for the built-in read
|
|
7988
|
+
* models, keyed by kind. Under sharding the host's own copies of those
|
|
7989
|
+
* models never index an operation, so the manager advances these from the
|
|
7990
|
+
* shards' relayed indexing reports; without them every read carrying a
|
|
7991
|
+
* consistency token waits forever.
|
|
6917
7992
|
*/
|
|
6918
|
-
async createProjectionShardManager(config, eventBus) {
|
|
7993
|
+
async createProjectionShardManager(config, eventBus, consistencyTrackers) {
|
|
6919
7994
|
const baseDb = this.resolveReactorDbConfig();
|
|
6920
7995
|
if (!baseDb) throw new Error("withProjectionShards requires a db (or an executor worker pool configured with one); projection workers need connection info to open their own pools.");
|
|
6921
7996
|
const models = this.resolvedModelManifest ?? [];
|
|
@@ -6931,7 +8006,7 @@ var ReactorBuilder = class {
|
|
|
6931
8006
|
poolInstrumentations.push(forwarder);
|
|
6932
8007
|
this.instrumentedPools.push(forwarder);
|
|
6933
8008
|
}
|
|
6934
|
-
const { ProjectionShardManager } = await import("./projection-shard-manager-
|
|
8009
|
+
const { ProjectionShardManager } = await import("./projection-shard-manager-D6KYcEZi.js");
|
|
6935
8010
|
const manager = new ProjectionShardManager({
|
|
6936
8011
|
shardCount: config.shardCount,
|
|
6937
8012
|
db,
|
|
@@ -6945,7 +8020,8 @@ var ReactorBuilder = class {
|
|
|
6945
8020
|
shutdownGraceMs: config.shutdownGraceMs,
|
|
6946
8021
|
drainTimeoutMs: config.drainTimeoutMs,
|
|
6947
8022
|
chainDepthReportIntervalMs: config.chainDepthReportIntervalMs,
|
|
6948
|
-
poolInstrumentations
|
|
8023
|
+
poolInstrumentations,
|
|
8024
|
+
consistencyTrackers
|
|
6949
8025
|
});
|
|
6950
8026
|
await manager.startup();
|
|
6951
8027
|
this.shutdownHooks.push(() => manager.shutdown());
|
|
@@ -6962,9 +8038,9 @@ var ReactorBuilder = class {
|
|
|
6962
8038
|
*/
|
|
6963
8039
|
async createDefaultWorkerFactory(numWorkers, db, signatureVerifier) {
|
|
6964
8040
|
const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
|
|
6965
|
-
import("./worker-handle-
|
|
8041
|
+
import("./worker-handle-CrERzl8s.js"),
|
|
6966
8042
|
import("./transport-ByGviWdZ.js"),
|
|
6967
|
-
import("./worker-
|
|
8043
|
+
import("./worker-HPysBfhx.js").then((n) => n.n)
|
|
6968
8044
|
]);
|
|
6969
8045
|
const poolConfig = {
|
|
6970
8046
|
enabled: true,
|
|
@@ -6985,7 +8061,8 @@ var ReactorBuilder = class {
|
|
|
6985
8061
|
poolConfig,
|
|
6986
8062
|
db,
|
|
6987
8063
|
signatureVerifier,
|
|
6988
|
-
models
|
|
8064
|
+
models,
|
|
8065
|
+
executorConfig: this.executorConfig
|
|
6989
8066
|
},
|
|
6990
8067
|
logger,
|
|
6991
8068
|
poolInstrumentation
|
|
@@ -7106,6 +8183,7 @@ var ReactorClientBuilder = class {
|
|
|
7106
8183
|
subscriptionManager;
|
|
7107
8184
|
jobAwaiter;
|
|
7108
8185
|
documentModelLoader;
|
|
8186
|
+
readGate;
|
|
7109
8187
|
/**
|
|
7110
8188
|
* Sets the logger for the ReactorClient.
|
|
7111
8189
|
* @param logger - The logger to use.
|
|
@@ -7158,6 +8236,47 @@ var ReactorClientBuilder = class {
|
|
|
7158
8236
|
this.documentModelLoader = loader;
|
|
7159
8237
|
return this;
|
|
7160
8238
|
}
|
|
8239
|
+
/**
|
|
8240
|
+
* Overrides how reads are gated. A client built from a ReactorBuilder derives
|
|
8241
|
+
* this from that reactor's flags; one built from `withReactor` cannot, because
|
|
8242
|
+
* it is handed no flags and no registry, so it gates on the policy alone
|
|
8243
|
+
* unless a gate is supplied here.
|
|
8244
|
+
*/
|
|
8245
|
+
withReadGate(readGate) {
|
|
8246
|
+
this.readGate = readGate;
|
|
8247
|
+
return this;
|
|
8248
|
+
}
|
|
8249
|
+
/**
|
|
8250
|
+
* The gate the resolved model calls for. Below authEnforcement there is no
|
|
8251
|
+
* model to enforce -- the registered one ignores the auth scope -- so the
|
|
8252
|
+
* policy is evaluated on its own, which is what reads did before the model
|
|
8253
|
+
* existed. Group serving turns on with authGroups, because below it a
|
|
8254
|
+
* `{ group }` grant does not match, so a served roster is one no grant can
|
|
8255
|
+
* use.
|
|
8256
|
+
*/
|
|
8257
|
+
resolveReadGate(reactorModule, documentView, model) {
|
|
8258
|
+
if (!reactorModule || model === void 0) return new BareReadGate();
|
|
8259
|
+
return new ModelReadGate(model, documentView, reactorModule.featureFlags.authGroups, reactorModule.operationIndex, this.logger);
|
|
8260
|
+
}
|
|
8261
|
+
/**
|
|
8262
|
+
* What the client answers an authorization preflight from, or undefined when
|
|
8263
|
+
* it answers none.
|
|
8264
|
+
*
|
|
8265
|
+
* Resolved from the same model reads enforce, so a preflight and a read can
|
|
8266
|
+
* never decide against different models. Undefined below authEnforcement, and
|
|
8267
|
+
* undefined on the `withReactor` path, where there are no flags and no
|
|
8268
|
+
* registry to select a model with -- a client with no model refuses the
|
|
8269
|
+
* preflight rather than answering it from the legacy host-side permission
|
|
8270
|
+
* tables. Deliberately not derived from the read gate: `withReadGate`
|
|
8271
|
+
* overrides that, so sniffing the gate's type would report enforcement from a
|
|
8272
|
+
* caller's substitution.
|
|
8273
|
+
*/
|
|
8274
|
+
resolveActionEvaluation(reactorModule, model) {
|
|
8275
|
+
return reactorModule && model ? {
|
|
8276
|
+
model,
|
|
8277
|
+
flags: reactorModule.featureFlags
|
|
8278
|
+
} : void 0;
|
|
8279
|
+
}
|
|
7161
8280
|
async build() {
|
|
7162
8281
|
return (await this.buildModule()).client;
|
|
7163
8282
|
}
|
|
@@ -7186,8 +8305,9 @@ var ReactorClientBuilder = class {
|
|
|
7186
8305
|
const signer = this.signer ?? new PassthroughSigner();
|
|
7187
8306
|
const subscriptionManager = this.subscriptionManager ?? reactorModule?.subscriptionManager ?? new ReactorSubscriptionManager(new DefaultSubscriptionErrorHandler());
|
|
7188
8307
|
const jobAwaiter = this.jobAwaiter ?? new JobAwaiter(eventBus, (jobId, signal) => reactor.getJobStatus(jobId, signal));
|
|
8308
|
+
const decisionModel = reactorModule ? readDecisionModel(reactorModule.featureFlags, reactorModule.documentModelRegistry) : void 0;
|
|
7189
8309
|
return {
|
|
7190
|
-
client: new ReactorClient(this.logger, reactor, signer, subscriptionManager, jobAwaiter, documentIndexer, documentView),
|
|
8310
|
+
client: new ReactorClient(this.logger, reactor, signer, subscriptionManager, jobAwaiter, documentIndexer, documentView, this.readGate ?? this.resolveReadGate(reactorModule, documentView, decisionModel), this.resolveActionEvaluation(reactorModule, decisionModel)),
|
|
7191
8311
|
reactor,
|
|
7192
8312
|
eventBus,
|
|
7193
8313
|
documentIndexer,
|
|
@@ -7225,6 +8345,107 @@ function driveIdFromUrl(url) {
|
|
|
7225
8345
|
return url.split("/").pop() ?? "";
|
|
7226
8346
|
}
|
|
7227
8347
|
//#endregion
|
|
8348
|
+
//#region src/decision/sync-scope-gate.ts
|
|
8349
|
+
/** What a subject may read of a document that is not there to be read. */
|
|
8350
|
+
const META_ONLY = (scope) => ALWAYS_READABLE_SCOPES.has(scope);
|
|
8351
|
+
/**
|
|
8352
|
+
* A read gate asked by document id rather than by document.
|
|
8353
|
+
*
|
|
8354
|
+
* Serving works from an id: an outbox entry names a document, a branch and a
|
|
8355
|
+
* scope, and carries no state. Fetching the document is therefore the serving
|
|
8356
|
+
* path's own job, and it is the only thing this adds to the gate it wraps.
|
|
8357
|
+
*/
|
|
8358
|
+
var SyncScopeGate = class {
|
|
8359
|
+
constructor(gate, documentView, logger) {
|
|
8360
|
+
this.gate = gate;
|
|
8361
|
+
this.documentView = documentView;
|
|
8362
|
+
this.logger = logger;
|
|
8363
|
+
}
|
|
8364
|
+
/**
|
|
8365
|
+
* Which scopes of one document the subject may be served.
|
|
8366
|
+
*
|
|
8367
|
+
* A document this replica cannot produce yields the metadata scopes and
|
|
8368
|
+
* nothing else. That is the fail-closed direction, and it is safe to fail
|
|
8369
|
+
* closed here precisely because serving withholds rather than consumes: the
|
|
8370
|
+
* entry stays in the outbox and the next poll asks again, so a document that
|
|
8371
|
+
* is merely not indexed yet is delayed rather than lost.
|
|
8372
|
+
*
|
|
8373
|
+
* Any other failure is rethrown. A read side that is down must not read as a
|
|
8374
|
+
* silent, universal denial, because a denial that looks like a policy is one
|
|
8375
|
+
* nobody investigates.
|
|
8376
|
+
*/
|
|
8377
|
+
async scopePredicateById(documentId, subject, branch, signal) {
|
|
8378
|
+
let document;
|
|
8379
|
+
try {
|
|
8380
|
+
document = await this.documentView.get(documentId, { branch }, void 0, signal);
|
|
8381
|
+
} catch (error) {
|
|
8382
|
+
if (!DocumentNotFoundError.isError(error)) throw error;
|
|
8383
|
+
this.logger?.verbose(`Withholding ${documentId} on branch ${branch}: the read side does not hold it`);
|
|
8384
|
+
return META_ONLY;
|
|
8385
|
+
}
|
|
8386
|
+
return this.gate.scopePredicate(document, subject, branch, signal);
|
|
8387
|
+
}
|
|
8388
|
+
};
|
|
8389
|
+
//#endregion
|
|
8390
|
+
//#region src/decision/stream-order.ts
|
|
8391
|
+
/**
|
|
8392
|
+
* The first pair of effective operations whose stored order contradicts their
|
|
8393
|
+
* timestamps, or undefined when the stream is in position order.
|
|
8394
|
+
*
|
|
8395
|
+
* Such a stream cannot be walked, and the auth stream is never reshuffled once
|
|
8396
|
+
* the monotonic rule is on, so run this before enabling enforcement on a fleet.
|
|
8397
|
+
*
|
|
8398
|
+
* `requireStrict` additionally rejects a tie, which is what the auth stream's
|
|
8399
|
+
* monotonic rule requires and what the walk alone does not care about.
|
|
8400
|
+
*/
|
|
8401
|
+
function firstOutOfOrderPair(operations, options) {
|
|
8402
|
+
const requireStrict = options?.requireStrict ?? false;
|
|
8403
|
+
const effective = garbageCollect(sortOperations([...operations]));
|
|
8404
|
+
for (let i = 1; i < effective.length; i++) {
|
|
8405
|
+
const previous = effective[i - 1];
|
|
8406
|
+
const current = effective[i];
|
|
8407
|
+
const previousAt = Date.parse(previous.timestampUtcMs);
|
|
8408
|
+
const currentAt = Date.parse(current.timestampUtcMs);
|
|
8409
|
+
if (currentAt < previousAt) return {
|
|
8410
|
+
previous,
|
|
8411
|
+
current,
|
|
8412
|
+
kind: "descending"
|
|
8413
|
+
};
|
|
8414
|
+
if (requireStrict && currentAt === previousAt) return {
|
|
8415
|
+
previous,
|
|
8416
|
+
current,
|
|
8417
|
+
kind: "tied"
|
|
8418
|
+
};
|
|
8419
|
+
}
|
|
8420
|
+
}
|
|
8421
|
+
//#endregion
|
|
8422
|
+
//#region src/read-models/interfaces.ts
|
|
8423
|
+
function supportsLiveReadModelRegistration(coordinator) {
|
|
8424
|
+
return "addReadModel" in coordinator && typeof coordinator.addReadModel === "function";
|
|
8425
|
+
}
|
|
8426
|
+
//#endregion
|
|
8427
|
+
//#region src/processors/host-module.ts
|
|
8428
|
+
/** Builds the reactor-level host module; hosts spread it and add their own fields. */
|
|
8429
|
+
function createReactorHostModuleBase(options) {
|
|
8430
|
+
const { client, readModels, ...core } = options;
|
|
8431
|
+
return {
|
|
8432
|
+
...core,
|
|
8433
|
+
client,
|
|
8434
|
+
dispatch: { async execute(docId, branch, actions, signal) {
|
|
8435
|
+
const jobInfo = await client.executeAsync(docId, branch, actions, signal);
|
|
8436
|
+
return {
|
|
8437
|
+
id: jobInfo.id,
|
|
8438
|
+
status: jobInfo.status
|
|
8439
|
+
};
|
|
8440
|
+
} },
|
|
8441
|
+
getReadModel(name) {
|
|
8442
|
+
const model = readModels.find((m) => m.name === name);
|
|
8443
|
+
if (!model) throw new Error(`Read model "${name}" not found`);
|
|
8444
|
+
return model;
|
|
8445
|
+
}
|
|
8446
|
+
};
|
|
8447
|
+
}
|
|
8448
|
+
//#endregion
|
|
7228
8449
|
//#region src/admin/passthrough-keyframe-store.ts
|
|
7229
8450
|
const passthroughKeyframeStore = {
|
|
7230
8451
|
putKeyframe: () => Promise.resolve(),
|
|
@@ -7250,6 +8471,7 @@ var DocumentIntegrityService = class {
|
|
|
7250
8471
|
async validateDocument(documentId, branch = "main", signal) {
|
|
7251
8472
|
const keyframeIssues = [];
|
|
7252
8473
|
const snapshotIssues = [];
|
|
8474
|
+
const streamOrderIssues = await this.findStreamOrderIssues(documentId, branch, signal);
|
|
7253
8475
|
const replayCache = new KyselyWriteCache(passthroughKeyframeStore, this.operationStore, this.documentModelRegistry, {
|
|
7254
8476
|
maxDocuments: 1,
|
|
7255
8477
|
ringBufferSize: 1,
|
|
@@ -7276,9 +8498,10 @@ var DocumentIntegrityService = class {
|
|
|
7276
8498
|
} catch {
|
|
7277
8499
|
return {
|
|
7278
8500
|
documentId,
|
|
7279
|
-
isConsistent: keyframeIssues.length === 0,
|
|
8501
|
+
isConsistent: keyframeIssues.length === 0 && streamOrderIssues.length === 0,
|
|
7280
8502
|
keyframeIssues,
|
|
7281
|
-
snapshotIssues
|
|
8503
|
+
snapshotIssues,
|
|
8504
|
+
streamOrderIssues
|
|
7282
8505
|
};
|
|
7283
8506
|
}
|
|
7284
8507
|
const revisions = await this.operationStore.getRevisions(documentId, branch, signal);
|
|
@@ -7304,9 +8527,10 @@ var DocumentIntegrityService = class {
|
|
|
7304
8527
|
}
|
|
7305
8528
|
return {
|
|
7306
8529
|
documentId,
|
|
7307
|
-
isConsistent: keyframeIssues.length === 0 && snapshotIssues.length === 0,
|
|
8530
|
+
isConsistent: keyframeIssues.length === 0 && snapshotIssues.length === 0 && streamOrderIssues.length === 0,
|
|
7308
8531
|
keyframeIssues,
|
|
7309
|
-
snapshotIssues
|
|
8532
|
+
snapshotIssues,
|
|
8533
|
+
streamOrderIssues
|
|
7310
8534
|
};
|
|
7311
8535
|
}
|
|
7312
8536
|
async rebuildKeyframes(documentId, branch = "main", signal) {
|
|
@@ -7328,12 +8552,26 @@ var DocumentIntegrityService = class {
|
|
|
7328
8552
|
scopesInvalidated: scopes.length
|
|
7329
8553
|
};
|
|
7330
8554
|
}
|
|
8555
|
+
async findStreamOrderIssues(documentId, branch, signal) {
|
|
8556
|
+
const scopes = await this.discoverScopes(documentId, branch, signal);
|
|
8557
|
+
const issues = [];
|
|
8558
|
+
for (const scope of scopes) {
|
|
8559
|
+
throwIfAborted(signal);
|
|
8560
|
+
const pair = firstOutOfOrderPair((await this.operationStore.getSince(documentId, scope, branch, -1, void 0, void 0, signal)).results, { requireStrict: scope === "auth" });
|
|
8561
|
+
if (pair !== void 0) issues.push({
|
|
8562
|
+
scope,
|
|
8563
|
+
branch,
|
|
8564
|
+
...pair
|
|
8565
|
+
});
|
|
8566
|
+
}
|
|
8567
|
+
return issues;
|
|
8568
|
+
}
|
|
7331
8569
|
async discoverScopes(documentId, branch, signal) {
|
|
7332
8570
|
const revisions = await this.operationStore.getRevisions(documentId, branch, signal);
|
|
7333
8571
|
return Object.keys(revisions.revision);
|
|
7334
8572
|
}
|
|
7335
8573
|
};
|
|
7336
8574
|
//#endregion
|
|
7337
|
-
export { BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentChangeType, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RevisionMismatchError, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncStatus, SyncStatusTracker, addRelationshipAction, batchOperationsByDocument, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createRelationalDb, deleteDocumentAction, documentActions, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, getMigrationStatus, instrumentPgPool, isDriveAuthError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, removeRelationshipAction, runMigrations, sanitizeArg, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
8575
|
+
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SeededStateReader, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createReactorHostModuleBase, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
7338
8576
|
|
|
7339
8577
|
//# sourceMappingURL=index.js.map
|