@powerhousedao/reactor 6.2.2-dev.53 → 6.2.2-dev.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{build-worker-executor-NT9b3rNm.js → build-worker-executor-DBHkoWBR.js} +2 -2
- package/dist/{build-worker-executor-NT9b3rNm.js.map → build-worker-executor-DBHkoWBR.js.map} +1 -1
- package/dist/{drive-container-types-RZa1wukO.js → drive-container-types-bVQ_8YwX.js} +145 -47
- package/dist/drive-container-types-bVQ_8YwX.js.map +1 -0
- package/dist/entry.js +2 -2
- package/dist/index.d.ts +160 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +411 -147
- package/dist/index.js.map +1 -1
- package/dist/projection-entry.js +2 -2
- package/dist/{worker-sw2vjrd3.js → worker-DXJpyHLW.js} +2 -2
- package/dist/{worker-sw2vjrd3.js.map → worker-DXJpyHLW.js.map} +1 -1
- package/package.json +4 -4
- package/dist/drive-container-types-RZa1wukO.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as GATED_DOCUMENT_ACTIONS, B as AuthTimestampNotMonotonicError, C as DuplicateOperationError, D as DuplicateModuleError, E as DuplicateManifestError, F as selectDecisionModel, G as InvalidOperationTimestampError, H as DocumentDeletedError, I as documentDecisionModel, J as parsePagingOptions, K as UpgradePreconditionFailedError, L as authDecisionModel, M as createEmptyConsistencyToken, N as targetDocumentId, O as InvalidModuleError, P as decideAtHead, R as buildDecisionModel, S as AppendConditionFailedError, T as RevisionMismatchError, U as DocumentNotFoundError, V as AuthorizationDeniedError, W as ExcessiveReshuffleError, Y as throwIfAborted, _ as DocumentMetaCache, a as createForwardingPoolInstrumentation, b as JobExecutorEventTypes, c as KyselyKeyframeStore, d as DriveCollectionId, f as KyselyExecutionScope, g as KyselyOperationIndex, h as KyselyWriteCache, i as runMigrations, j as createConsistencyToken, k as ModuleNotFoundError, l as DocumentModelRegistry, m as EventBus, n as REACTOR_SCHEMA, o as instrumentPgPool, p as resolveFeatureFlags, q as matchesScope, r as getMigrationStatus, s as KyselyOperationStore, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as SimpleJobExecutor, v as CollectionMembershipCache, w as OptimisticLockError, x as APPEND_CONDITION_FAILED_PREFIX, y as DEFAULT_DEFERRED_JOB_TTL_MS, z as AuthEnforcementDisabledError } from "./drive-container-types-bVQ_8YwX.js";
|
|
2
2
|
import { n as ReactorEventTypes, t as EventBusAggregateError } from "./types-DMKLa0Ok.js";
|
|
3
3
|
import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-D3S6Eysd.js";
|
|
4
4
|
import { a as ReadModelCoordinator, i as KyselyDocumentView, n as ConsistencyTracker, o as BaseReadModel, r as makeConsistencyKey, t as KyselyDocumentIndexer } from "./document-indexer-DlpJB8AK.js";
|
|
5
5
|
import { n as errorToInfo, r as sanitizeArg, t as createForwardingLogger } from "./forwarding-logger-BBkMSxuJ.js";
|
|
6
|
-
import { t as workerEntryPath } from "./worker-
|
|
7
|
-
import { DowngradeNotSupportedError, MAX_AUTH_GRANTS, UnsupportedDocumentModelVersionError, actions, actions as documentActions, createPresignedHeader, decide, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, normalizeDocumentModelVersion, sortOperations } from "@powerhousedao/shared/document-model";
|
|
6
|
+
import { t as workerEntryPath } from "./worker-DXJpyHLW.js";
|
|
7
|
+
import { DowngradeNotSupportedError, MAX_AUTH_GRANTS, UnsupportedDocumentModelVersionError, actions, actions as documentActions, createPresignedHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, normalizeDocumentModelVersion, sortOperations, toTransportAction } from "@powerhousedao/shared/document-model";
|
|
8
8
|
import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
|
|
9
9
|
import { v4 } from "uuid";
|
|
10
10
|
import { ConsoleLogger, childLogger } from "document-model";
|
|
@@ -695,6 +695,8 @@ let DocumentChangeType = /* @__PURE__ */ function(DocumentChangeType) {
|
|
|
695
695
|
* argument covers. Grants gate domain-scope reads only.
|
|
696
696
|
*/
|
|
697
697
|
const ALWAYS_READABLE_SCOPES = new Set(["auth", "document"]);
|
|
698
|
+
/** What a gate does with an uninitialized policy when told nothing. */
|
|
699
|
+
const OPEN_BY_DEFAULT = { withholdUninitialized: false };
|
|
698
700
|
/**
|
|
699
701
|
* The branch a group is read on. A group's member list lives on its main
|
|
700
702
|
* branch whatever branch the document naming it is on, and the reference
|
|
@@ -816,12 +818,13 @@ var SeededStateReader = class {
|
|
|
816
818
|
* from state no commit ever invalidates.
|
|
817
819
|
*/
|
|
818
820
|
var ModelReadGate = class {
|
|
819
|
-
constructor(model, documentView, servesGroups, operationIndex, logger) {
|
|
821
|
+
constructor(model, documentView, servesGroups, operationIndex, logger, options = OPEN_BY_DEFAULT) {
|
|
820
822
|
this.model = model;
|
|
821
823
|
this.documentView = documentView;
|
|
822
824
|
this.servesGroups = servesGroups;
|
|
823
825
|
this.operationIndex = operationIndex;
|
|
824
826
|
this.logger = logger;
|
|
827
|
+
this.options = options;
|
|
825
828
|
}
|
|
826
829
|
/**
|
|
827
830
|
* A served group yields its member list and nothing else. What the audience
|
|
@@ -914,15 +917,19 @@ var ModelReadGate = class {
|
|
|
914
917
|
/**
|
|
915
918
|
* What this document's own policy says, with no group serving applied.
|
|
916
919
|
*
|
|
917
|
-
* An unpoliced document is readable in full
|
|
918
|
-
*
|
|
919
|
-
* `evaluate` makes: a legacy `{}` auth
|
|
920
|
-
* uninitialized, and "no grants" does not,
|
|
921
|
-
* an empty grant list denies everything.
|
|
920
|
+
* An unpoliced document is readable in full unless the host closes by default,
|
|
921
|
+
* and either way it is the common case and the one worth answering without
|
|
922
|
+
* building anything. The test is the one `evaluate` makes: a legacy `{}` auth
|
|
923
|
+
* scope and version 0 both mean uninitialized, and "no grants" does not,
|
|
924
|
+
* because a policy with a version and an empty grant list denies everything.
|
|
925
|
+
*
|
|
926
|
+
* Closing here rather than around the gate is what keeps a policy able to
|
|
927
|
+
* publish an unpoliced group it names: the referencer walk asks this question
|
|
928
|
+
* of the referencing document, whose real policy answers it.
|
|
922
929
|
*/
|
|
923
930
|
async ownPolicyPredicate(document, subject, branch, signal) {
|
|
924
931
|
const auth = authOf(document);
|
|
925
|
-
if (!auth || !auth.version) return () => true;
|
|
932
|
+
if (!auth || !auth.version) return this.options.withholdUninitialized ? (scope) => ALWAYS_READABLE_SCOPES.has(scope) : () => true;
|
|
926
933
|
const target = {
|
|
927
934
|
documentId: document.header.id,
|
|
928
935
|
branch
|
|
@@ -1746,12 +1753,12 @@ var JobResultHandler = class {
|
|
|
1746
1753
|
return;
|
|
1747
1754
|
} catch {}
|
|
1748
1755
|
}
|
|
1749
|
-
if (result.error && DocumentNotFoundError.isError(result.error)) {
|
|
1756
|
+
if (result.error && DocumentNotFoundError.isError(result.error) && handle.job.kind === "load") {
|
|
1750
1757
|
handle.defer();
|
|
1751
|
-
callbacks.deferJob(
|
|
1758
|
+
callbacks.deferJob(result.error.documentId, handle.job);
|
|
1752
1759
|
return;
|
|
1753
1760
|
}
|
|
1754
|
-
if (result.error && (DocumentDeletedError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error))) {
|
|
1761
|
+
if (result.error && (DocumentDeletedError.isError(result.error) || DocumentNotFoundError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error) || UpgradePreconditionFailedError.isError(result.error))) {
|
|
1755
1762
|
const errorInfo = toErrorInfo(result.error);
|
|
1756
1763
|
this.jobTracker.markFailed(handle.job.id, errorInfo, handle.job);
|
|
1757
1764
|
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
@@ -1817,16 +1824,127 @@ var JobResultHandler = class {
|
|
|
1817
1824
|
}
|
|
1818
1825
|
};
|
|
1819
1826
|
//#endregion
|
|
1820
|
-
//#region src/executor/
|
|
1827
|
+
//#region src/executor/deferred-jobs.ts
|
|
1821
1828
|
/**
|
|
1822
|
-
*
|
|
1829
|
+
* The jobs held back because the document they write to was not there yet.
|
|
1830
|
+
*
|
|
1831
|
+
* A job whose document is missing is deferred rather than failed, because the
|
|
1832
|
+
* operations that create that document may still be on their way - out of order
|
|
1833
|
+
* from a peer, or a sibling job in the same batch. Deferring emits no event and
|
|
1834
|
+
* moves no job status, which is what makes it cheap: a job that gets flushed
|
|
1835
|
+
* runs as if it had merely waited its turn.
|
|
1836
|
+
*
|
|
1837
|
+
* That silence is also why the wait has to be bounded. A deferred job is only
|
|
1838
|
+
* released when a job carrying CREATE_DOCUMENT for that exact document id
|
|
1839
|
+
* completes, so a job deferred against an id nothing will ever create is
|
|
1840
|
+
* released by nothing. Its status stays RUNNING, which is not terminal, so a
|
|
1841
|
+
* caller awaiting it waits forever - there is no timeout on that side. The TTL
|
|
1842
|
+
* converts that silence into the failure the caller can act on.
|
|
1843
|
+
*
|
|
1844
|
+
* Failing through the queue rather than only the tracker matters: the queue
|
|
1845
|
+
* records the job as resolved, which is what unblocks anything that declared a
|
|
1846
|
+
* dependency on it. Deferring alone does not, so a dependent of a deferred job
|
|
1847
|
+
* is stranded at PENDING for as long as the deferral lasts.
|
|
1823
1848
|
*/
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1849
|
+
var DeferredJobs = class {
|
|
1850
|
+
#byDocumentId = /* @__PURE__ */ new Map();
|
|
1851
|
+
#timers = /* @__PURE__ */ new Map();
|
|
1852
|
+
constructor(queue, jobTracker, eventBus, logger, ttlMs = DEFAULT_DEFERRED_JOB_TTL_MS, onReleased = () => Promise.resolve()) {
|
|
1853
|
+
this.queue = queue;
|
|
1854
|
+
this.jobTracker = jobTracker;
|
|
1855
|
+
this.eventBus = eventBus;
|
|
1856
|
+
this.logger = logger;
|
|
1857
|
+
this.ttlMs = ttlMs;
|
|
1858
|
+
this.onReleased = onReleased;
|
|
1859
|
+
}
|
|
1860
|
+
/**
|
|
1861
|
+
* Holds a job until its document arrives, or until the TTL elapses.
|
|
1862
|
+
*
|
|
1863
|
+
* A job deferred again after a flush replaces its own timer rather than
|
|
1864
|
+
* accumulating one, so the wait is measured from the most recent attempt.
|
|
1865
|
+
*/
|
|
1866
|
+
add(documentId, job) {
|
|
1867
|
+
const existing = this.#byDocumentId.get(documentId) ?? [];
|
|
1868
|
+
existing.push(job);
|
|
1869
|
+
this.#byDocumentId.set(documentId, existing);
|
|
1870
|
+
this.#clearTimer(job.id);
|
|
1871
|
+
const timer = setTimeout(() => {
|
|
1872
|
+
this.#expire(documentId, job);
|
|
1873
|
+
}, this.ttlMs);
|
|
1874
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
1875
|
+
this.#timers.set(job.id, timer);
|
|
1876
|
+
}
|
|
1877
|
+
/** Re-enqueues everything waiting on a document that has now arrived. */
|
|
1878
|
+
async flush(documentId) {
|
|
1879
|
+
const jobs = this.#byDocumentId.get(documentId);
|
|
1880
|
+
if (!jobs || jobs.length === 0) return;
|
|
1881
|
+
this.#byDocumentId.delete(documentId);
|
|
1882
|
+
for (const job of jobs) {
|
|
1883
|
+
this.#clearTimer(job.id);
|
|
1884
|
+
try {
|
|
1885
|
+
await this.queue.enqueue(job);
|
|
1886
|
+
} catch (error) {
|
|
1887
|
+
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
/**
|
|
1892
|
+
* Fails everything still deferred, for a reactor that is shutting down.
|
|
1893
|
+
*
|
|
1894
|
+
* The queue is deliberately not told: it is going away too, and nothing is
|
|
1895
|
+
* left to unblock.
|
|
1896
|
+
*/
|
|
1897
|
+
failAll() {
|
|
1898
|
+
for (const [, jobs] of this.#byDocumentId) for (const job of jobs) {
|
|
1899
|
+
this.#clearTimer(job.id);
|
|
1900
|
+
this.#markFailed(job);
|
|
1901
|
+
}
|
|
1902
|
+
this.#byDocumentId.clear();
|
|
1903
|
+
}
|
|
1904
|
+
/** The jobs still waiting on a document. Exposed for assertions. */
|
|
1905
|
+
waitingOn(documentId) {
|
|
1906
|
+
return this.#byDocumentId.get(documentId) ?? [];
|
|
1907
|
+
}
|
|
1908
|
+
async #expire(documentId, job) {
|
|
1909
|
+
const jobs = this.#byDocumentId.get(documentId);
|
|
1910
|
+
if (!jobs?.includes(job)) return;
|
|
1911
|
+
const remaining = jobs.filter((held) => held !== job);
|
|
1912
|
+
if (remaining.length === 0) this.#byDocumentId.delete(documentId);
|
|
1913
|
+
else this.#byDocumentId.set(documentId, remaining);
|
|
1914
|
+
this.#timers.delete(job.id);
|
|
1915
|
+
this.logger.error("Deferred job @jobId gave up waiting for document @documentId after @ttlMs ms", job.id, documentId, this.ttlMs);
|
|
1916
|
+
const errorInfo = this.#markFailed(job);
|
|
1917
|
+
try {
|
|
1918
|
+
await this.queue.failJob(job.id, errorInfo);
|
|
1919
|
+
} catch (error) {
|
|
1920
|
+
this.logger.error("Error failing an expired deferred job: @Error", error);
|
|
1921
|
+
return;
|
|
1922
|
+
}
|
|
1923
|
+
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1924
|
+
try {
|
|
1925
|
+
await this.onReleased();
|
|
1926
|
+
} catch (error) {
|
|
1927
|
+
this.logger.error("Error dispatching after a deferred job expired: @Error", error);
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
#markFailed(job) {
|
|
1931
|
+
const error = new DocumentNotFoundError(job.documentId);
|
|
1932
|
+
const errorInfo = toErrorInfo(error);
|
|
1933
|
+
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1934
|
+
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
1935
|
+
jobId: job.id,
|
|
1936
|
+
error,
|
|
1937
|
+
job
|
|
1938
|
+
}).catch(() => {});
|
|
1939
|
+
return errorInfo;
|
|
1940
|
+
}
|
|
1941
|
+
#clearTimer(jobId) {
|
|
1942
|
+
const timer = this.#timers.get(jobId);
|
|
1943
|
+
if (timer !== void 0) {
|
|
1944
|
+
clearTimeout(timer);
|
|
1945
|
+
this.#timers.delete(jobId);
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1830
1948
|
};
|
|
1831
1949
|
//#endregion
|
|
1832
1950
|
//#region src/executor/worker-pool-router.ts
|
|
@@ -1898,10 +2016,10 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1898
2016
|
activeJobs = 0;
|
|
1899
2017
|
totalJobsProcessed = 0;
|
|
1900
2018
|
unsubscribe;
|
|
1901
|
-
deferredJobs
|
|
2019
|
+
deferredJobs;
|
|
1902
2020
|
resultHandler;
|
|
1903
2021
|
jobTimeoutMs;
|
|
1904
|
-
constructor(workerFactory, eventBus, queue, jobTracker, logger, resolver, collectionMembershipCache, jobTimeoutMs = 3e4) {
|
|
2022
|
+
constructor(workerFactory, eventBus, queue, jobTracker, logger, resolver, collectionMembershipCache, jobTimeoutMs = 3e4, deferredJobTtlMs = DEFAULT_DEFERRED_JOB_TTL_MS) {
|
|
1905
2023
|
this.workerFactory = workerFactory;
|
|
1906
2024
|
this.eventBus = eventBus;
|
|
1907
2025
|
this.queue = queue;
|
|
@@ -1910,6 +2028,7 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1910
2028
|
this.resolver = resolver;
|
|
1911
2029
|
this.collectionMembershipCache = collectionMembershipCache;
|
|
1912
2030
|
this.jobTimeoutMs = jobTimeoutMs;
|
|
2031
|
+
this.deferredJobs = new DeferredJobs(queue, jobTracker, eventBus, logger, deferredJobTtlMs, () => this.tryDispatchAll());
|
|
1913
2032
|
this.resultHandler = new JobResultHandler(queue, jobTracker, eventBus, resolver, logger);
|
|
1914
2033
|
}
|
|
1915
2034
|
async start(numWorkers) {
|
|
@@ -1930,16 +2049,7 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
1930
2049
|
this.unsubscribe = void 0;
|
|
1931
2050
|
}
|
|
1932
2051
|
if (graceful) while (this.activeJobs > 0) await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1933
|
-
|
|
1934
|
-
const errorInfo = toErrorInfo(new DocumentNotFoundError(job.documentId));
|
|
1935
|
-
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
1936
|
-
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
1937
|
-
jobId: job.id,
|
|
1938
|
-
error: new DocumentNotFoundError(job.documentId),
|
|
1939
|
-
job
|
|
1940
|
-
}).catch(() => {});
|
|
1941
|
-
}
|
|
1942
|
-
this.deferredJobs.clear();
|
|
2052
|
+
this.deferredJobs.failAll();
|
|
1943
2053
|
await Promise.all(this.workers.map((w) => w.shutdown(graceful).catch((err) => {
|
|
1944
2054
|
this.logger.warn("worker shutdown failed: @Error", err);
|
|
1945
2055
|
})));
|
|
@@ -2061,12 +2171,8 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
2061
2171
|
this.logger.error("emitWriteReady failed for job @jobId: @Error", { jobId: handle.job.id }, error);
|
|
2062
2172
|
});
|
|
2063
2173
|
await this.resultHandler.handleResult(handle, outcome.result, {
|
|
2064
|
-
deferJob: (documentId, job) =>
|
|
2065
|
-
|
|
2066
|
-
existing.push(job);
|
|
2067
|
-
this.deferredJobs.set(documentId, existing);
|
|
2068
|
-
},
|
|
2069
|
-
flushDeferredFor: (documentId) => this.flushDeferredJobs(documentId)
|
|
2174
|
+
deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
|
|
2175
|
+
flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
|
|
2070
2176
|
});
|
|
2071
2177
|
this.activeJobs--;
|
|
2072
2178
|
await this.tryDispatchFor(worker);
|
|
@@ -2148,16 +2254,6 @@ var WorkerPoolJobExecutorManager = class {
|
|
|
2148
2254
|
this.workers[deadIndex] = fresh;
|
|
2149
2255
|
await this.tryDispatchFor(fresh);
|
|
2150
2256
|
}
|
|
2151
|
-
async flushDeferredJobs(documentId) {
|
|
2152
|
-
const jobs = this.deferredJobs.get(documentId);
|
|
2153
|
-
if (!jobs || jobs.length === 0) return;
|
|
2154
|
-
this.deferredJobs.delete(documentId);
|
|
2155
|
-
for (const job of jobs) try {
|
|
2156
|
-
await this.queue.enqueue(job);
|
|
2157
|
-
} catch (error) {
|
|
2158
|
-
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
2159
|
-
}
|
|
2160
|
-
}
|
|
2161
2257
|
};
|
|
2162
2258
|
function isWorkerTransportError(error) {
|
|
2163
2259
|
return error instanceof WorkerExitedError || error instanceof WorkerInitFailedError || error instanceof WorkerAbortTimeoutError;
|
|
@@ -2186,10 +2282,10 @@ var SimpleJobExecutorManager = class {
|
|
|
2186
2282
|
activeJobs = 0;
|
|
2187
2283
|
totalJobsProcessed = 0;
|
|
2188
2284
|
unsubscribe;
|
|
2189
|
-
deferredJobs
|
|
2285
|
+
deferredJobs;
|
|
2190
2286
|
resultHandler;
|
|
2191
2287
|
jobTimeoutMs;
|
|
2192
|
-
constructor(executorFactory, eventBus, queue, jobTracker, logger, resolver, jobTimeoutMs = 3e4) {
|
|
2288
|
+
constructor(executorFactory, eventBus, queue, jobTracker, logger, resolver, jobTimeoutMs = 3e4, deferredJobTtlMs = DEFAULT_DEFERRED_JOB_TTL_MS) {
|
|
2193
2289
|
this.executorFactory = executorFactory;
|
|
2194
2290
|
this.eventBus = eventBus;
|
|
2195
2291
|
this.queue = queue;
|
|
@@ -2197,6 +2293,7 @@ var SimpleJobExecutorManager = class {
|
|
|
2197
2293
|
this.logger = logger;
|
|
2198
2294
|
this.resolver = resolver;
|
|
2199
2295
|
this.jobTimeoutMs = jobTimeoutMs;
|
|
2296
|
+
this.deferredJobs = new DeferredJobs(queue, jobTracker, eventBus, logger, deferredJobTtlMs, () => this.checkForMoreJobs());
|
|
2200
2297
|
this.resultHandler = new JobResultHandler(queue, jobTracker, eventBus, resolver, logger);
|
|
2201
2298
|
}
|
|
2202
2299
|
async start(numExecutors) {
|
|
@@ -2217,16 +2314,7 @@ var SimpleJobExecutorManager = class {
|
|
|
2217
2314
|
this.unsubscribe = void 0;
|
|
2218
2315
|
}
|
|
2219
2316
|
if (graceful) while (this.activeJobs > 0) await new Promise((resolve) => setTimeout(resolve, 100));
|
|
2220
|
-
|
|
2221
|
-
const errorInfo = toErrorInfo(new DocumentNotFoundError(job.documentId));
|
|
2222
|
-
this.jobTracker.markFailed(job.id, errorInfo, job);
|
|
2223
|
-
this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
|
|
2224
|
-
jobId: job.id,
|
|
2225
|
-
error: new DocumentNotFoundError(job.documentId),
|
|
2226
|
-
job
|
|
2227
|
-
}).catch(() => {});
|
|
2228
|
-
}
|
|
2229
|
-
this.deferredJobs.clear();
|
|
2317
|
+
this.deferredJobs.failAll();
|
|
2230
2318
|
this.executors = [];
|
|
2231
2319
|
this.isRunning = false;
|
|
2232
2320
|
}
|
|
@@ -2319,12 +2407,8 @@ var SimpleJobExecutorManager = class {
|
|
|
2319
2407
|
this.eventBus.emit(JobExecutorEventTypes.JOB_FAILED, failedEvent).catch(() => {});
|
|
2320
2408
|
}
|
|
2321
2409
|
await this.resultHandler.handleResult(handle, result, {
|
|
2322
|
-
deferJob: (documentId, job) =>
|
|
2323
|
-
|
|
2324
|
-
existing.push(job);
|
|
2325
|
-
this.deferredJobs.set(documentId, existing);
|
|
2326
|
-
},
|
|
2327
|
-
flushDeferredFor: (documentId) => this.flushDeferredJobs(documentId)
|
|
2410
|
+
deferJob: (documentId, job) => this.deferredJobs.add(documentId, job),
|
|
2411
|
+
flushDeferredFor: (documentId) => this.deferredJobs.flush(documentId)
|
|
2328
2412
|
});
|
|
2329
2413
|
this.activeJobs--;
|
|
2330
2414
|
await this.checkForMoreJobs();
|
|
@@ -2358,16 +2442,6 @@ var SimpleJobExecutorManager = class {
|
|
|
2358
2442
|
}
|
|
2359
2443
|
}
|
|
2360
2444
|
}
|
|
2361
|
-
async flushDeferredJobs(documentId) {
|
|
2362
|
-
const jobs = this.deferredJobs.get(documentId);
|
|
2363
|
-
if (!jobs || jobs.length === 0) return;
|
|
2364
|
-
this.deferredJobs.delete(documentId);
|
|
2365
|
-
for (const job of jobs) try {
|
|
2366
|
-
await this.queue.enqueue(job);
|
|
2367
|
-
} catch (error) {
|
|
2368
|
-
this.logger.error("Error re-enqueuing deferred job: @Error", error);
|
|
2369
|
-
}
|
|
2370
|
-
}
|
|
2371
2445
|
};
|
|
2372
2446
|
//#endregion
|
|
2373
2447
|
//#region src/job-tracker/in-memory-job-tracker.ts
|
|
@@ -4057,13 +4131,42 @@ var BufferedMailbox = class {
|
|
|
4057
4131
|
var GraphQLRequestError = class extends Error {
|
|
4058
4132
|
statusCode;
|
|
4059
4133
|
category;
|
|
4060
|
-
|
|
4134
|
+
/**
|
|
4135
|
+
* One entry per error the response carried, in order, holding its
|
|
4136
|
+
* `extensions.code` - undefined where it declared none. Kept per error rather
|
|
4137
|
+
* than as a set, because a response that mixes a classified error with an
|
|
4138
|
+
* unclassified one must not be read as if only the classified one arrived.
|
|
4139
|
+
*/
|
|
4140
|
+
codes;
|
|
4141
|
+
constructor(message, category, statusCode, codes = []) {
|
|
4061
4142
|
super(message);
|
|
4062
4143
|
this.name = "GraphQLRequestError";
|
|
4063
4144
|
this.category = category;
|
|
4064
4145
|
this.statusCode = statusCode;
|
|
4146
|
+
this.codes = codes;
|
|
4065
4147
|
}
|
|
4066
4148
|
};
|
|
4149
|
+
/**
|
|
4150
|
+
* Extension codes a remote uses to say a failure is worth polling through.
|
|
4151
|
+
*
|
|
4152
|
+
* Shared with reactor-api so the server throws what this check reads and the two
|
|
4153
|
+
* cannot drift. A `graphql` category error is otherwise permanent: it stops the
|
|
4154
|
+
* poll timer, and nothing restarts it, so a code that lands here is the
|
|
4155
|
+
* difference between a channel that recovers and one that is dead for the
|
|
4156
|
+
* process lifetime.
|
|
4157
|
+
*/
|
|
4158
|
+
const RECOVERABLE_GRAPHQL_ERROR_CODES = { malformedStoredOperation: "MALFORMED_STORED_OPERATION" };
|
|
4159
|
+
const RECOVERABLE_CODES = new Set(Object.values(RECOVERABLE_GRAPHQL_ERROR_CODES));
|
|
4160
|
+
/**
|
|
4161
|
+
* True when every error the response carried named a recoverable code.
|
|
4162
|
+
*
|
|
4163
|
+
* Unanimity is the requirement: one unclassified error alongside a recoverable
|
|
4164
|
+
* one means something else also went wrong, and polling through that would be
|
|
4165
|
+
* guessing.
|
|
4166
|
+
*/
|
|
4167
|
+
function isRecoverableGraphQLError(error) {
|
|
4168
|
+
return error.codes.length > 0 && error.codes.every((code) => code !== void 0 && RECOVERABLE_CODES.has(code));
|
|
4169
|
+
}
|
|
4067
4170
|
/** Auth-rejection message fragments the switchboard emits. Shared with
|
|
4068
4171
|
* reactor-api so server throws and this client check can't drift. */
|
|
4069
4172
|
const DRIVE_AUTH_ERROR_MESSAGES = {
|
|
@@ -4609,36 +4712,15 @@ var IntervalPollTimer = class {
|
|
|
4609
4712
|
//#region src/sync/channels/utils.ts
|
|
4610
4713
|
let syncOpCounter = 0;
|
|
4611
4714
|
/**
|
|
4612
|
-
* Serializes an action for GraphQL transport
|
|
4715
|
+
* Serializes an action for GraphQL transport.
|
|
4613
4716
|
*
|
|
4614
|
-
*
|
|
4615
|
-
*
|
|
4616
|
-
*
|
|
4617
|
-
*
|
|
4717
|
+
* Projects onto exactly the fields `ActionInput` declares, which is what keeps a
|
|
4718
|
+
* stale runtime-only field - a legacy `attachments` array, say - from riding
|
|
4719
|
+
* along into the mutation variables, where the schema would reject the whole
|
|
4720
|
+
* request rather than the field.
|
|
4618
4721
|
*/
|
|
4619
4722
|
function serializeAction(action) {
|
|
4620
|
-
|
|
4621
|
-
id: action.id,
|
|
4622
|
-
type: action.type,
|
|
4623
|
-
timestampUtcMs: action.timestampUtcMs,
|
|
4624
|
-
input: action.input,
|
|
4625
|
-
scope: action.scope
|
|
4626
|
-
};
|
|
4627
|
-
const signer = action.context?.signer;
|
|
4628
|
-
if (!signer?.signatures) return action.context ? {
|
|
4629
|
-
...base,
|
|
4630
|
-
context: action.context
|
|
4631
|
-
} : base;
|
|
4632
|
-
return {
|
|
4633
|
-
...base,
|
|
4634
|
-
context: {
|
|
4635
|
-
...action.context,
|
|
4636
|
-
signer: {
|
|
4637
|
-
...signer,
|
|
4638
|
-
signatures: signer.signatures.map((sig) => Array.isArray(sig) ? sig.join(", ") : sig)
|
|
4639
|
-
}
|
|
4640
|
-
}
|
|
4641
|
-
};
|
|
4723
|
+
return toTransportAction(action);
|
|
4642
4724
|
}
|
|
4643
4725
|
/**
|
|
4644
4726
|
* Serializes a SyncEnvelope for GraphQL transport.
|
|
@@ -4676,16 +4758,6 @@ function serializeEnvelope(envelope) {
|
|
|
4676
4758
|
};
|
|
4677
4759
|
}
|
|
4678
4760
|
/**
|
|
4679
|
-
* Deserializes a signature from a comma-separated string back to a tuple.
|
|
4680
|
-
*
|
|
4681
|
-
* GraphQL serializes Signature tuples as comma-separated strings for transport.
|
|
4682
|
-
* This function converts them back to the expected [string, string, string, string, string] format.
|
|
4683
|
-
*/
|
|
4684
|
-
function deserializeSignature(sig) {
|
|
4685
|
-
if (Array.isArray(sig)) return sig;
|
|
4686
|
-
return sig.split(", ");
|
|
4687
|
-
}
|
|
4688
|
-
/**
|
|
4689
4761
|
* `isDenied` tests strictly against undefined, so a null left in place would mark
|
|
4690
4762
|
* every synced operation denied. The key is removed rather than set to undefined,
|
|
4691
4763
|
* so an operation that arrived without it stays identical to the one sent.
|
|
@@ -5279,7 +5351,7 @@ var GqlRequestChannel = class {
|
|
|
5279
5351
|
if (error.statusCode !== void 0 && error.statusCode >= 500) return "recoverable";
|
|
5280
5352
|
return "unrecoverable";
|
|
5281
5353
|
case "parse": return "recoverable";
|
|
5282
|
-
case "graphql": return "unrecoverable";
|
|
5354
|
+
case "graphql": return isRecoverableGraphQLError(error) ? "recoverable" : "unrecoverable";
|
|
5283
5355
|
case "missing-data": return "unrecoverable";
|
|
5284
5356
|
}
|
|
5285
5357
|
}
|
|
@@ -5364,7 +5436,7 @@ var GqlRequestChannel = class {
|
|
|
5364
5436
|
throw new GraphQLRequestError(`Failed to parse GraphQL response: ${error instanceof Error ? error.message : String(error)}`, "parse");
|
|
5365
5437
|
}
|
|
5366
5438
|
this.logger.verbose("GQL response @channelId @operation status=@status data=@data errors=@errors", this.channelId, operationName, response.status, JSON.stringify(result.data), result.errors ? JSON.stringify(result.errors) : "none");
|
|
5367
|
-
if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql");
|
|
5439
|
+
if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql", void 0, result.errors.map((error) => error.extensions?.code));
|
|
5368
5440
|
if (!result.data) throw new GraphQLRequestError("GraphQL response missing data field", "missing-data");
|
|
5369
5441
|
return result.data;
|
|
5370
5442
|
}
|
|
@@ -5456,6 +5528,10 @@ var GqlRequestChannelFactory = class {
|
|
|
5456
5528
|
};
|
|
5457
5529
|
//#endregion
|
|
5458
5530
|
//#region src/sync/channels/gql-res-channel.ts
|
|
5531
|
+
/** Where a sync operation's run of ordinals begins. */
|
|
5532
|
+
function firstOrdinalOf$1(syncOp) {
|
|
5533
|
+
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
5534
|
+
}
|
|
5459
5535
|
/**
|
|
5460
5536
|
* This class is used server-side to accumulate inbox + outbox operations.
|
|
5461
5537
|
*
|
|
@@ -5471,6 +5547,8 @@ var GqlResponseChannel = class {
|
|
|
5471
5547
|
isShutdown;
|
|
5472
5548
|
lastPersistedInboxOrdinal = 0;
|
|
5473
5549
|
lastPersistedOutboxOrdinal = 0;
|
|
5550
|
+
evictedOutboxFloor = Number.POSITIVE_INFINITY;
|
|
5551
|
+
appliedOutboxOrdinal = 0;
|
|
5474
5552
|
connectionState = "connecting";
|
|
5475
5553
|
connectionStateCallbacks = /* @__PURE__ */ new Set();
|
|
5476
5554
|
constructor(logger, channelId, remoteName, cursorStorage) {
|
|
@@ -5483,18 +5561,11 @@ var GqlResponseChannel = class {
|
|
|
5483
5561
|
this.outbox = new Mailbox();
|
|
5484
5562
|
this.deadLetter = new Mailbox();
|
|
5485
5563
|
this.outbox.onRemoved((syncOps) => {
|
|
5486
|
-
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
cursorType: "outbox",
|
|
5492
|
-
cursorOrdinal: maxOrdinal,
|
|
5493
|
-
lastSyncedAtUtcMs: Date.now()
|
|
5494
|
-
}).catch((error) => {
|
|
5495
|
-
this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
|
|
5496
|
-
});
|
|
5497
|
-
}
|
|
5564
|
+
this.rememberUnserved(syncOps);
|
|
5565
|
+
this.persistOutboxCursor(getLatestAppliedOrdinal(syncOps));
|
|
5566
|
+
});
|
|
5567
|
+
this.outbox.onAdded((syncOps) => {
|
|
5568
|
+
this.forgetEvictedBelow(syncOps);
|
|
5498
5569
|
});
|
|
5499
5570
|
this.inbox.onRemoved((syncOps) => {
|
|
5500
5571
|
const maxOrdinal = getLatestAppliedOrdinal(syncOps);
|
|
@@ -5536,6 +5607,17 @@ var GqlResponseChannel = class {
|
|
|
5536
5607
|
}
|
|
5537
5608
|
/** Response channels are push-driven; resolvers populate mailboxes directly. */
|
|
5538
5609
|
triggerPull() {}
|
|
5610
|
+
async init() {
|
|
5611
|
+
const cursors = await this.cursorStorage.list(this.remoteName);
|
|
5612
|
+
const inboxOrdinal = cursors.find((c) => c.cursorType === "inbox")?.cursorOrdinal ?? 0;
|
|
5613
|
+
const outboxOrdinal = cursors.find((c) => c.cursorType === "outbox")?.cursorOrdinal ?? 0;
|
|
5614
|
+
this.inbox.init(inboxOrdinal);
|
|
5615
|
+
this.outbox.init(outboxOrdinal);
|
|
5616
|
+
this.lastPersistedInboxOrdinal = inboxOrdinal;
|
|
5617
|
+
this.lastPersistedOutboxOrdinal = outboxOrdinal;
|
|
5618
|
+
this.appliedOutboxOrdinal = outboxOrdinal;
|
|
5619
|
+
this.transitionConnectionState("connected");
|
|
5620
|
+
}
|
|
5539
5621
|
transitionConnectionState(next) {
|
|
5540
5622
|
if (this.connectionState === next) return;
|
|
5541
5623
|
this.connectionState = next;
|
|
@@ -5546,15 +5628,68 @@ var GqlResponseChannel = class {
|
|
|
5546
5628
|
this.logger.error("Connection state change callback error: @Error", error);
|
|
5547
5629
|
}
|
|
5548
5630
|
}
|
|
5549
|
-
|
|
5550
|
-
|
|
5551
|
-
|
|
5552
|
-
|
|
5553
|
-
|
|
5554
|
-
|
|
5555
|
-
|
|
5556
|
-
|
|
5557
|
-
|
|
5631
|
+
/**
|
|
5632
|
+
* Records the ordinals of entries that left the outbox without being served,
|
|
5633
|
+
* so the cursor cannot advance past them.
|
|
5634
|
+
*
|
|
5635
|
+
* An entry can leave unserved because a bound evicted it, and an evicted entry
|
|
5636
|
+
* is exactly one this channel intends to re-derive: it is still owed to the
|
|
5637
|
+
* remote. Remembering the floor across the whole run rather than only while
|
|
5638
|
+
* the entry is present is what makes that true after a later ack would
|
|
5639
|
+
* otherwise have swept the cursor past it.
|
|
5640
|
+
*/
|
|
5641
|
+
rememberUnserved(syncOps) {
|
|
5642
|
+
for (const syncOp of syncOps) {
|
|
5643
|
+
if (syncOp.status === SyncOperationStatus.Applied) continue;
|
|
5644
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5645
|
+
if (first > 0 && first < this.evictedOutboxFloor) this.evictedOutboxFloor = first;
|
|
5646
|
+
}
|
|
5647
|
+
}
|
|
5648
|
+
/**
|
|
5649
|
+
* Persists the outbox cursor, never past an operation this remote has not
|
|
5650
|
+
* been served.
|
|
5651
|
+
*
|
|
5652
|
+
* The cursor is where a restart resumes deriving the outbox from, so an
|
|
5653
|
+
* ordinal persisted past an unserved entry loses that entry for good: the
|
|
5654
|
+
* rebuild starts beyond it and nothing else remembers it was owed. Acks
|
|
5655
|
+
* arrive out of order with respect to what is withheld -- a later entry can
|
|
5656
|
+
* be acknowledged while an earlier one is still being withheld from this
|
|
5657
|
+
* subject -- so the applied high-water mark alone is not a safe cursor.
|
|
5658
|
+
*/
|
|
5659
|
+
persistOutboxCursor(appliedOrdinal) {
|
|
5660
|
+
this.appliedOutboxOrdinal = Math.max(this.appliedOutboxOrdinal, appliedOrdinal);
|
|
5661
|
+
const ordinal = Math.min(this.appliedOutboxOrdinal, this.unservedFloor() - 1);
|
|
5662
|
+
if (ordinal <= this.lastPersistedOutboxOrdinal) return;
|
|
5663
|
+
this.lastPersistedOutboxOrdinal = ordinal;
|
|
5664
|
+
this.cursorStorage.upsert({
|
|
5665
|
+
remoteName: this.remoteName,
|
|
5666
|
+
cursorType: "outbox",
|
|
5667
|
+
cursorOrdinal: ordinal,
|
|
5668
|
+
lastSyncedAtUtcMs: Date.now()
|
|
5669
|
+
}).catch((error) => {
|
|
5670
|
+
this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
|
|
5671
|
+
});
|
|
5672
|
+
}
|
|
5673
|
+
/** Drops the evicted floor once the entries it stood for are queued again. */
|
|
5674
|
+
forgetEvictedBelow(syncOps) {
|
|
5675
|
+
if (this.evictedOutboxFloor === Number.POSITIVE_INFINITY) return;
|
|
5676
|
+
for (const syncOp of syncOps) {
|
|
5677
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5678
|
+
if (first > 0 && first <= this.evictedOutboxFloor) {
|
|
5679
|
+
this.evictedOutboxFloor = Number.POSITIVE_INFINITY;
|
|
5680
|
+
return;
|
|
5681
|
+
}
|
|
5682
|
+
}
|
|
5683
|
+
}
|
|
5684
|
+
/** The lowest ordinal still owed to this remote, evicted or still queued. */
|
|
5685
|
+
unservedFloor() {
|
|
5686
|
+
let floor = this.evictedOutboxFloor;
|
|
5687
|
+
for (const syncOp of this.outbox.items) {
|
|
5688
|
+
if (syncOp.status === SyncOperationStatus.Applied) continue;
|
|
5689
|
+
const first = firstOrdinalOf$1(syncOp);
|
|
5690
|
+
if (first > 0 && first < floor) floor = first;
|
|
5691
|
+
}
|
|
5692
|
+
return floor;
|
|
5558
5693
|
}
|
|
5559
5694
|
};
|
|
5560
5695
|
//#endregion
|
|
@@ -5738,7 +5873,10 @@ function rowToRemoteRecord(row) {
|
|
|
5738
5873
|
scope: row.filter_scopes ?? [],
|
|
5739
5874
|
branch: row.filter_branch
|
|
5740
5875
|
},
|
|
5741
|
-
options: {
|
|
5876
|
+
options: {
|
|
5877
|
+
sinceTimestampUtcMs: "0",
|
|
5878
|
+
boundAddress: row.bound_address ?? void 0
|
|
5879
|
+
},
|
|
5742
5880
|
status: {
|
|
5743
5881
|
push: {
|
|
5744
5882
|
state: row.push_state,
|
|
@@ -5778,7 +5916,8 @@ function remoteRecordToRow(remote) {
|
|
|
5778
5916
|
pull_state: remote.status.pull.state,
|
|
5779
5917
|
pull_last_success_utc_ms: remote.status.pull.lastSuccessUtcMs ? new Date(remote.status.pull.lastSuccessUtcMs).toISOString() : null,
|
|
5780
5918
|
pull_last_failure_utc_ms: remote.status.pull.lastFailureUtcMs ? new Date(remote.status.pull.lastFailureUtcMs).toISOString() : null,
|
|
5781
|
-
pull_failure_count: remote.status.pull.failureCount
|
|
5919
|
+
pull_failure_count: remote.status.pull.failureCount,
|
|
5920
|
+
bound_address: remote.options.boundAddress ?? null
|
|
5782
5921
|
};
|
|
5783
5922
|
}
|
|
5784
5923
|
var KyselySyncRemoteStorage = class {
|
|
@@ -6146,9 +6285,14 @@ var OutboxMode = /* @__PURE__ */ function(OutboxMode) {
|
|
|
6146
6285
|
}(OutboxMode || {});
|
|
6147
6286
|
const defaultSyncManagerConfig = {
|
|
6148
6287
|
maxDeadLettersPerRemote: 100,
|
|
6149
|
-
maxInboxBatchSize: 32
|
|
6288
|
+
maxInboxBatchSize: 32,
|
|
6289
|
+
maxHeldOperationsPerRemote: 1e4
|
|
6150
6290
|
};
|
|
6151
6291
|
const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
|
|
6292
|
+
/** Where a sync operation's run of ordinals begins. */
|
|
6293
|
+
function firstOrdinalOf(syncOp) {
|
|
6294
|
+
return syncOp.operations.length > 0 ? syncOp.operations[0].context.ordinal : 0;
|
|
6295
|
+
}
|
|
6152
6296
|
var SyncManager = class {
|
|
6153
6297
|
logger;
|
|
6154
6298
|
remoteStorage;
|
|
@@ -6172,6 +6316,7 @@ var SyncManager = class {
|
|
|
6172
6316
|
quarantinedDocumentIds = /* @__PURE__ */ new Set();
|
|
6173
6317
|
backfillAbortControllers = /* @__PURE__ */ new Map();
|
|
6174
6318
|
planKeyToJobUuid = /* @__PURE__ */ new Map();
|
|
6319
|
+
evictedOutboxFloors = /* @__PURE__ */ new Map();
|
|
6175
6320
|
lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
|
|
6176
6321
|
inboxChunkChain = Promise.resolve();
|
|
6177
6322
|
constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, driveContainerTypes, config = {}) {
|
|
@@ -6279,6 +6424,28 @@ var SyncManager = class {
|
|
|
6279
6424
|
for (const remote of this.remotes.values()) if (remote.meta.id === id) return remote;
|
|
6280
6425
|
throw new Error(`Remote with id '${id}' does not exist`);
|
|
6281
6426
|
}
|
|
6427
|
+
async bindRemote(id, boundAddress) {
|
|
6428
|
+
const remote = this.getById(id);
|
|
6429
|
+
const bound = remote.meta.options.boundAddress;
|
|
6430
|
+
if (bound === boundAddress) return;
|
|
6431
|
+
if (bound !== void 0) throw new Error(`Remote with id '${id}' is already bound to another address`);
|
|
6432
|
+
remote.meta.options = {
|
|
6433
|
+
...remote.meta.options,
|
|
6434
|
+
boundAddress
|
|
6435
|
+
};
|
|
6436
|
+
await this.remoteStorage.upsert({
|
|
6437
|
+
id: remote.meta.id,
|
|
6438
|
+
name: remote.meta.name,
|
|
6439
|
+
collectionId: remote.meta.collectionId,
|
|
6440
|
+
channelConfig: remote.meta.channelConfig,
|
|
6441
|
+
filter: remote.meta.filter,
|
|
6442
|
+
options: remote.meta.options,
|
|
6443
|
+
status: {
|
|
6444
|
+
push: createIdleHealth(),
|
|
6445
|
+
pull: createIdleHealth()
|
|
6446
|
+
}
|
|
6447
|
+
});
|
|
6448
|
+
}
|
|
6282
6449
|
async add(name, collectionId, channelConfig, filter = {
|
|
6283
6450
|
documentId: [],
|
|
6284
6451
|
scope: [],
|
|
@@ -6621,9 +6788,59 @@ var SyncManager = class {
|
|
|
6621
6788
|
const message = error?.message || "Unknown error";
|
|
6622
6789
|
return new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), classifyJobFailure(error?.name ?? "Error"));
|
|
6623
6790
|
}
|
|
6791
|
+
/**
|
|
6792
|
+
* Where the next outbox derivation starts: at the evicted floor when there is
|
|
6793
|
+
* room to take those entries back, and at the caller's cursor otherwise.
|
|
6794
|
+
*
|
|
6795
|
+
* The floor is forgotten as it is folded in. If the refill overflows again the
|
|
6796
|
+
* eviction records a new one, and if it does not, the entries are queued and
|
|
6797
|
+
* the outbox itself holds the cursor down.
|
|
6798
|
+
*/
|
|
6799
|
+
refillOrdinal(remote, ackOrdinal) {
|
|
6800
|
+
const floor = this.evictedOutboxFloors.get(remote.meta.name);
|
|
6801
|
+
if (floor === void 0) return ackOrdinal;
|
|
6802
|
+
if (this.outboxOperationCount(remote) >= this.config.maxHeldOperationsPerRemote) return ackOrdinal;
|
|
6803
|
+
this.evictedOutboxFloors.delete(remote.meta.name);
|
|
6804
|
+
return Math.min(ackOrdinal, floor - 1);
|
|
6805
|
+
}
|
|
6806
|
+
/**
|
|
6807
|
+
* Evicts the newest entries of an outbox that has grown past its bound.
|
|
6808
|
+
*
|
|
6809
|
+
* Only entries that have never been taken up for transport are eligible, which
|
|
6810
|
+
* is what makes this a bound on withheld work rather than a way to drop
|
|
6811
|
+
* something already in flight. One entry is always kept, so an outbox whose
|
|
6812
|
+
* first entry is larger than the whole cap still makes progress.
|
|
6813
|
+
*/
|
|
6814
|
+
evictPastOutboxBound(remote) {
|
|
6815
|
+
const cap = this.config.maxHeldOperationsPerRemote;
|
|
6816
|
+
const queued = remote.channel.outbox.items.filter((syncOp) => syncOp.status === SyncOperationStatus.Unknown).sort((a, b) => firstOrdinalOf(a) - firstOrdinalOf(b));
|
|
6817
|
+
let kept = 0;
|
|
6818
|
+
let cut = queued.length;
|
|
6819
|
+
for (let i = 0; i < queued.length; i++) {
|
|
6820
|
+
const size = queued[i].operations.length;
|
|
6821
|
+
if (i > 0 && kept + size > cap) {
|
|
6822
|
+
cut = i;
|
|
6823
|
+
break;
|
|
6824
|
+
}
|
|
6825
|
+
kept += size;
|
|
6826
|
+
}
|
|
6827
|
+
const evicted = queued.slice(cut);
|
|
6828
|
+
if (evicted.length === 0) return;
|
|
6829
|
+
const floor = firstOrdinalOf(evicted[0]);
|
|
6830
|
+
const known = this.evictedOutboxFloors.get(remote.meta.name);
|
|
6831
|
+
this.evictedOutboxFloors.set(remote.meta.name, known === void 0 ? floor : Math.min(known, floor));
|
|
6832
|
+
this.logger.warn("Outbox for @RemoteName is past its bound of @Cap operations; evicting @Count entries from ordinal @Floor, to be derived again once it drains", remote.meta.name, cap, evicted.length, floor);
|
|
6833
|
+
remote.channel.outbox.remove(...evicted);
|
|
6834
|
+
}
|
|
6835
|
+
outboxOperationCount(remote) {
|
|
6836
|
+
let count = 0;
|
|
6837
|
+
for (const syncOp of remote.channel.outbox.items) count += syncOp.operations.length;
|
|
6838
|
+
return count;
|
|
6839
|
+
}
|
|
6624
6840
|
async updateOutbox(remote, ackOrdinal, mode = OutboxMode.Backfill, signal) {
|
|
6625
6841
|
const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
|
|
6626
|
-
|
|
6842
|
+
const startOrdinal = this.refillOrdinal(remote, ackOrdinal);
|
|
6843
|
+
let maxOrdinal = startOrdinal;
|
|
6627
6844
|
const lastJobByDoc = /* @__PURE__ */ new Map();
|
|
6628
6845
|
let prevChainJobId;
|
|
6629
6846
|
const sinceTimestamp = remote.meta.options.sinceTimestampUtcMs;
|
|
@@ -6643,8 +6860,9 @@ var SyncManager = class {
|
|
|
6643
6860
|
if (mode === OutboxMode.BatchTriggered) prevChainJobId = jobId;
|
|
6644
6861
|
}
|
|
6645
6862
|
remote.channel.outbox.add(...syncOps);
|
|
6863
|
+
this.evictPastOutboxBound(remote);
|
|
6646
6864
|
};
|
|
6647
|
-
let page = await this.operationIndex.find(remote.meta.collectionId.key,
|
|
6865
|
+
let page = await this.operationIndex.find(remote.meta.collectionId.key, startOrdinal, { excludeSourceRemote: remote.meta.name }, void 0, composedSignal);
|
|
6648
6866
|
let carry = [];
|
|
6649
6867
|
let hasMore;
|
|
6650
6868
|
do {
|
|
@@ -6709,6 +6927,10 @@ var SyncBuilder = class {
|
|
|
6709
6927
|
this.config.maxInboxBatchSize = limit;
|
|
6710
6928
|
return this;
|
|
6711
6929
|
}
|
|
6930
|
+
withMaxHeldOperationsPerRemote(limit) {
|
|
6931
|
+
this.config.maxHeldOperationsPerRemote = limit;
|
|
6932
|
+
return this;
|
|
6933
|
+
}
|
|
6712
6934
|
build(reactor, logger, operationIndex, eventBus, db, driveContainerTypes) {
|
|
6713
6935
|
return this.buildModule(reactor, logger, operationIndex, eventBus, db, driveContainerTypes).syncManager;
|
|
6714
6936
|
}
|
|
@@ -7571,11 +7793,11 @@ var ReactorBuilder = class {
|
|
|
7571
7793
|
if (pool.db === void 0) throw new Error("unreachable: worker pool configured without db or factory");
|
|
7572
7794
|
factory = await this.createDefaultWorkerFactory(pool.numWorkers, pool.db, pool.verifier);
|
|
7573
7795
|
}
|
|
7574
|
-
const poolManager = new WorkerPoolJobExecutorManager(factory, eventBus, queue, jobTracker, this.logger, resolver, collectionMembershipCache, this.executorConfig.jobTimeoutMs);
|
|
7796
|
+
const poolManager = new WorkerPoolJobExecutorManager(factory, eventBus, queue, jobTracker, this.logger, resolver, collectionMembershipCache, this.executorConfig.jobTimeoutMs, this.executorConfig.deferredJobTtlMs);
|
|
7575
7797
|
executorManager = poolManager;
|
|
7576
7798
|
executorStartCount = pool.numWorkers;
|
|
7577
7799
|
if (resolver instanceof DocumentModelResolver) resolver.setBroadcastHook((entry) => poolManager.loadModel(entry));
|
|
7578
|
-
} else executorManager = new SimpleJobExecutorManager(() => new SimpleJobExecutor(this.logger, documentModelRegistry, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, collectionMembershipCache, this.driveContainerTypes, this.executorConfig, this.signatureVerifier, executionScope), eventBus, queue, jobTracker, this.logger, resolver, this.executorConfig.jobTimeoutMs);
|
|
7800
|
+
} else executorManager = new SimpleJobExecutorManager(() => new SimpleJobExecutor(this.logger, documentModelRegistry, operationStore, eventBus, writeCache, operationIndex, documentMetaCache, collectionMembershipCache, this.driveContainerTypes, this.executorConfig, this.signatureVerifier, executionScope), eventBus, queue, jobTracker, this.logger, resolver, this.executorConfig.jobTimeoutMs, this.executorConfig.deferredJobTtlMs);
|
|
7579
7801
|
await executorManager.start(executorStartCount);
|
|
7580
7802
|
const readModelInstances = Array.from(new Set([...this.readModels]));
|
|
7581
7803
|
const documentViewConsistencyTracker = new ConsistencyTracker();
|
|
@@ -7725,7 +7947,7 @@ var ReactorBuilder = class {
|
|
|
7725
7947
|
const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
|
|
7726
7948
|
import("./worker-handle-CrERzl8s.js"),
|
|
7727
7949
|
import("./transport-ByGviWdZ.js"),
|
|
7728
|
-
import("./worker-
|
|
7950
|
+
import("./worker-DXJpyHLW.js").then((n) => n.n)
|
|
7729
7951
|
]);
|
|
7730
7952
|
const poolConfig = {
|
|
7731
7953
|
enabled: true,
|
|
@@ -8030,6 +8252,48 @@ function driveIdFromUrl(url) {
|
|
|
8030
8252
|
return url.split("/").pop() ?? "";
|
|
8031
8253
|
}
|
|
8032
8254
|
//#endregion
|
|
8255
|
+
//#region src/decision/sync-scope-gate.ts
|
|
8256
|
+
/** What a subject may read of a document that is not there to be read. */
|
|
8257
|
+
const META_ONLY = (scope) => ALWAYS_READABLE_SCOPES.has(scope);
|
|
8258
|
+
/**
|
|
8259
|
+
* A read gate asked by document id rather than by document.
|
|
8260
|
+
*
|
|
8261
|
+
* Serving works from an id: an outbox entry names a document, a branch and a
|
|
8262
|
+
* scope, and carries no state. Fetching the document is therefore the serving
|
|
8263
|
+
* path's own job, and it is the only thing this adds to the gate it wraps.
|
|
8264
|
+
*/
|
|
8265
|
+
var SyncScopeGate = class {
|
|
8266
|
+
constructor(gate, documentView, logger) {
|
|
8267
|
+
this.gate = gate;
|
|
8268
|
+
this.documentView = documentView;
|
|
8269
|
+
this.logger = logger;
|
|
8270
|
+
}
|
|
8271
|
+
/**
|
|
8272
|
+
* Which scopes of one document the subject may be served.
|
|
8273
|
+
*
|
|
8274
|
+
* A document this replica cannot produce yields the metadata scopes and
|
|
8275
|
+
* nothing else. That is the fail-closed direction, and it is safe to fail
|
|
8276
|
+
* closed here precisely because serving withholds rather than consumes: the
|
|
8277
|
+
* entry stays in the outbox and the next poll asks again, so a document that
|
|
8278
|
+
* is merely not indexed yet is delayed rather than lost.
|
|
8279
|
+
*
|
|
8280
|
+
* Any other failure is rethrown. A read side that is down must not read as a
|
|
8281
|
+
* silent, universal denial, because a denial that looks like a policy is one
|
|
8282
|
+
* nobody investigates.
|
|
8283
|
+
*/
|
|
8284
|
+
async scopePredicateById(documentId, subject, branch, signal) {
|
|
8285
|
+
let document;
|
|
8286
|
+
try {
|
|
8287
|
+
document = await this.documentView.get(documentId, { branch }, void 0, signal);
|
|
8288
|
+
} catch (error) {
|
|
8289
|
+
if (!DocumentNotFoundError.isError(error)) throw error;
|
|
8290
|
+
this.logger?.verbose(`Withholding ${documentId} on branch ${branch}: the read side does not hold it`);
|
|
8291
|
+
return META_ONLY;
|
|
8292
|
+
}
|
|
8293
|
+
return this.gate.scopePredicate(document, subject, branch, signal);
|
|
8294
|
+
}
|
|
8295
|
+
};
|
|
8296
|
+
//#endregion
|
|
8033
8297
|
//#region src/decision/stream-order.ts
|
|
8034
8298
|
/**
|
|
8035
8299
|
* The first pair of effective operations whose stored order contradicts their
|
|
@@ -8193,6 +8457,6 @@ var DocumentIntegrityService = class {
|
|
|
8193
8457
|
}
|
|
8194
8458
|
};
|
|
8195
8459
|
//#endregion
|
|
8196
|
-
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentChangeType, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SeededStateReader, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
8460
|
+
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentChangeType, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SeededStateReader, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, isRecoverableGraphQLError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
|
|
8197
8461
|
|
|
8198
8462
|
//# sourceMappingURL=index.js.map
|