@powerhousedao/reactor 6.2.3-dev.49 → 6.2.3-dev.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{document-indexer-Dc_Sv6yi.js → document-indexer-DwAMDVRI.js} +2 -2
- package/dist/{document-indexer-Dc_Sv6yi.js.map → document-indexer-DwAMDVRI.js.map} +1 -1
- package/dist/{drive-container-types-D380mcB3.js → drive-container-types-Ci1pdv4X.js} +9 -9
- package/dist/{drive-container-types-D380mcB3.js.map → drive-container-types-Ci1pdv4X.js.map} +1 -1
- package/dist/entry.js +3 -2
- package/dist/entry.js.map +1 -1
- package/dist/{load-spec-D9lnje-H.js → forwarding-logger-ByfBFgMS.js} +9 -248
- package/dist/forwarding-logger-ByfBFgMS.js.map +1 -0
- package/dist/{index-Dq2f7oAC.d.ts → index-CNhT69De.d.ts} +34 -5
- package/dist/index-CNhT69De.d.ts.map +1 -0
- package/dist/index.d.ts +126 -26
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1065 -157
- package/dist/index.js.map +1 -1
- package/dist/load-spec-ZLjuyhxz.js +246 -0
- package/dist/load-spec-ZLjuyhxz.js.map +1 -0
- package/dist/projection-entry.js +3 -2
- package/dist/projection-entry.js.map +1 -1
- package/dist/{projection-shard-manager-MRJ9BjVM.js → projection-shard-manager-CQ5FwQmv.js} +2 -2
- package/dist/{projection-shard-manager-MRJ9BjVM.js.map → projection-shard-manager-CQ5FwQmv.js.map} +1 -1
- package/dist/rpc.d.ts +2 -2
- package/dist/rpc.js +3 -2
- package/dist/rpc.js.map +1 -1
- package/dist/{worker-handle-B48V1hDm.js → worker-handle-CN2-oRv-.js} +2 -2
- package/dist/{worker-handle-B48V1hDm.js.map → worker-handle-CN2-oRv-.js.map} +1 -1
- package/package.json +13 -13
- package/dist/index-Dq2f7oAC.d.ts.map +0 -1
- package/dist/load-spec-D9lnje-H.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { _ as buildDecisionModel, a as KyselyExecutionScope, c as appliedDelete, d as DEFAULT_MAX_PURGE_OPERATIONS, f as JobExecutorEventTypes, g as authDecisionModel, h as documentDecisionModel, i as PassthroughSigner, l as resolveFeatureFlags, m as selectDecisionModel, n as SimpleJobExecutor, o as DEFAULT_PURGE_DELETE_BATCH, p as decideAtHead, r as verifyActionSignature, s as KyselyDocumentPurger, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as DEFAULT_DEFERRED_JOB_TTL_MS, v as DriveCollectionId } from "./drive-container-types-
|
|
1
|
+
import { $ as DocumentExistence, A as GATED_DOCUMENT_ACTIONS, C as describeWaitingSessions, Ct as matchesScope, D as CollectionMembershipCache, Dt as __exportAll, E as DocumentMetaCache, J as targetDocumentId, K as submittedActionIds, L as createConsistencyToken, Q as DocumentAlreadyExistsError, R as createDocumentFromAction, S as createKyselyWatermarkProbe, St as ReactorEventTypes, T as snapshotFunctionsFor, Tt as throwIfAborted, X as APPEND_CONDITION_FAILED_PREFIX, Z as AppendConditionFailedError, _ as acquirePurgeLocks, _t as ReservedActionError, a as REACTOR_SCHEMA, at as AuthorizationDeniedError, b as ProbeSettler, bt as UpgradePreconditionFailedError, c as createForwardingPoolInstrumentation, ct as DocumentNotDeletedError, d as KyselyKeyframeStore, dt as ExcessiveReshuffleError, et as DuplicateOperationError, f as EventBus, ft as GroupInUseError, g as PURGE_NS, gt as RelationshipNotFoundError, h as PURGE_LOCK_BUCKETS, ht as PurgeTooLargeError, it as AuthTimestampNotMonotonicError, l as instrumentPgPool, lt as DocumentNotFoundError, m as KyselyOperationIndex, mt as InvalidSignatureError, n as errorToInfo, nt as RevisionMismatchError, o as getMigrationStatus, ot as BatchJobFailedError, p as KyselyWriteCache, pt as InvalidOperationTimestampError, q as summarizeSubmittedActions, r as sanitizeArg, rt as AuthEnforcementDisabledError, s as runMigrations, st as DocumentDeletedError, t as createForwardingLogger, tt as OptimisticLockError, u as KyselyOperationStore, ut as DocumentPurgedError, v as findPurged, vt as UnsupportedProtocolVersionError, w as parseSnapshot, wt as parsePagingOptions, x as SettledWatermark, xt as EventBusAggregateError, y as listPurged, yt as UnsupportedStoredProtocolError, z as createEmptyConsistencyToken } from "./forwarding-logger-ByfBFgMS.js";
|
|
2
|
+
import { _ as buildDecisionModel, a as KyselyExecutionScope, c as appliedDelete, d as DEFAULT_MAX_PURGE_OPERATIONS, f as JobExecutorEventTypes, g as authDecisionModel, h as documentDecisionModel, i as PassthroughSigner, l as resolveFeatureFlags, m as selectDecisionModel, n as SimpleJobExecutor, o as DEFAULT_PURGE_DELETE_BATCH, p as decideAtHead, r as verifyActionSignature, s as KyselyDocumentPurger, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as DEFAULT_DEFERRED_JOB_TTL_MS, v as DriveCollectionId } from "./drive-container-types-Ci1pdv4X.js";
|
|
3
|
+
import { a as DuplicateModuleError, c as ModuleNotFoundError, i as DuplicateManifestError, n as loadDocumentModelSpec, o as InvalidModuleError, r as DocumentModelRegistry, s as ModelNotWorkerImportableError } from "./load-spec-ZLjuyhxz.js";
|
|
3
4
|
import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-BM_ULVWd.js";
|
|
4
|
-
import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-
|
|
5
|
+
import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-DwAMDVRI.js";
|
|
5
6
|
import { DEFAULT_SIGNATURE_POLICY, DOCUMENT_PURGE_PROTOCOL, DowngradeNotSupportedError, MARKER_REFUSAL_FEATURE, MAX_AUTH_GRANTS, PEER_CAPABILITIES, PURGE_DOCUMENT, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, isPurgeMarker, isPurgeMarker as isPurgeMarker$1, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, purgedProtocolVersions, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
6
7
|
import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
|
|
7
8
|
import { v4 } from "uuid";
|
|
8
9
|
import { ConsoleLogger, childLogger } from "document-model";
|
|
9
|
-
import { sql } from "kysely";
|
|
10
|
+
import { CompiledQuery, sql } from "kysely";
|
|
11
|
+
import { PGliteDialect } from "kysely-pglite-dialect";
|
|
10
12
|
import { RelationalDbProcessor, createRelationalDb } from "@powerhousedao/shared/processors";
|
|
11
13
|
//#region src/actions/index.ts
|
|
12
14
|
/**
|
|
@@ -5600,6 +5602,7 @@ var Mailbox = class {
|
|
|
5600
5602
|
itemsMap = /* @__PURE__ */ new Map();
|
|
5601
5603
|
/** Unapplied items carrying a marker, so the held ack reads only these. */
|
|
5602
5604
|
heldMarkers = /* @__PURE__ */ new Set();
|
|
5605
|
+
held = /* @__PURE__ */ new Set();
|
|
5603
5606
|
addedCallbacks = [];
|
|
5604
5607
|
removedCallbacks = [];
|
|
5605
5608
|
paused = false;
|
|
@@ -5620,9 +5623,9 @@ var Mailbox = class {
|
|
|
5620
5623
|
return Array.from(this.itemsMap.values());
|
|
5621
5624
|
}
|
|
5622
5625
|
get ackOrdinal() {
|
|
5623
|
-
if (!this.holdAckBelowMarkers) return this._ack;
|
|
5626
|
+
if (!this.holdAckBelowMarkers && this.held.size === 0) return this._ack;
|
|
5624
5627
|
let floor = Number.POSITIVE_INFINITY;
|
|
5625
|
-
for (const item of this.heldMarkers) for (const op of item.operations) {
|
|
5628
|
+
for (const item of [...this.heldMarkers, ...this.held]) for (const op of item.operations) {
|
|
5626
5629
|
const ordinal = op.context.ordinal;
|
|
5627
5630
|
if (ordinal > 0 && ordinal < floor) floor = ordinal;
|
|
5628
5631
|
}
|
|
@@ -5631,13 +5634,21 @@ var Mailbox = class {
|
|
|
5631
5634
|
get latestOrdinal() {
|
|
5632
5635
|
return this._latestOrdinal;
|
|
5633
5636
|
}
|
|
5637
|
+
hold(item) {
|
|
5638
|
+
if (item.status === 2) return;
|
|
5639
|
+
if (this.itemsMap.get(item.id) !== item) return;
|
|
5640
|
+
this.held.add(item);
|
|
5641
|
+
}
|
|
5634
5642
|
get(id) {
|
|
5635
5643
|
return this.itemsMap.get(id);
|
|
5636
5644
|
}
|
|
5637
5645
|
add(...items) {
|
|
5638
5646
|
for (const item of items) {
|
|
5639
5647
|
const replaced = this.itemsMap.get(item.id);
|
|
5640
|
-
if (replaced !== void 0
|
|
5648
|
+
if (replaced !== void 0 && replaced !== item) {
|
|
5649
|
+
this.heldMarkers.delete(replaced);
|
|
5650
|
+
this.held.delete(replaced);
|
|
5651
|
+
}
|
|
5641
5652
|
this.itemsMap.set(item.id, item);
|
|
5642
5653
|
let marker = false;
|
|
5643
5654
|
for (const op of item.operations) {
|
|
@@ -5648,6 +5659,7 @@ var Mailbox = class {
|
|
|
5648
5659
|
item.on((syncOp, _, next) => {
|
|
5649
5660
|
if (next === 2) {
|
|
5650
5661
|
this.heldMarkers.delete(syncOp);
|
|
5662
|
+
this.held.delete(syncOp);
|
|
5651
5663
|
for (const op of syncOp.operations) this._ack = Math.max(this._ack, op.context.ordinal);
|
|
5652
5664
|
}
|
|
5653
5665
|
});
|
|
@@ -5669,6 +5681,7 @@ var Mailbox = class {
|
|
|
5669
5681
|
for (const item of items) {
|
|
5670
5682
|
this.itemsMap.delete(item.id);
|
|
5671
5683
|
this.heldMarkers.delete(item);
|
|
5684
|
+
this.held.delete(item);
|
|
5672
5685
|
}
|
|
5673
5686
|
if (this.paused) {
|
|
5674
5687
|
this.removedBuffer.push(...items);
|
|
@@ -5945,6 +5958,13 @@ function isDriveAuthError(error) {
|
|
|
5945
5958
|
if (error.category === "graphql") return Object.values(DRIVE_AUTH_ERROR_MESSAGES).some((m) => error.message.includes(m));
|
|
5946
5959
|
return false;
|
|
5947
5960
|
}
|
|
5961
|
+
/** A repair lever declined a request it cannot carry out safely. */
|
|
5962
|
+
var SyncRepairRefusedError = class extends Error {
|
|
5963
|
+
constructor(message) {
|
|
5964
|
+
super(message);
|
|
5965
|
+
this.name = "SyncRepairRefusedError";
|
|
5966
|
+
}
|
|
5967
|
+
};
|
|
5948
5968
|
var PollingChannelError = class extends Error {
|
|
5949
5969
|
constructor(message) {
|
|
5950
5970
|
super(message);
|
|
@@ -6362,6 +6382,7 @@ function classifyJobFailure(errorName) {
|
|
|
6362
6382
|
case "HashMismatchError": return "HASH_MISMATCH";
|
|
6363
6383
|
case "UnsupportedProtocolVersionError": return "UNSUPPORTED_PROTOCOL";
|
|
6364
6384
|
case "DocumentPurgedError": return "DOCUMENT_PURGED";
|
|
6385
|
+
case "DocumentNotFoundError": return "MISSING_OPERATIONS";
|
|
6365
6386
|
case "DocumentNotDeletedError":
|
|
6366
6387
|
case "GroupInUseError":
|
|
6367
6388
|
case "PurgeTooLargeError": return "PURGE_PRECONDITION";
|
|
@@ -6380,47 +6401,57 @@ const NON_QUARANTINING_ERROR_TYPES = /* @__PURE__ */ new Set([
|
|
|
6380
6401
|
"PEER_PROTOCOL_UNSUPPORTED",
|
|
6381
6402
|
"DOCUMENT_PURGED",
|
|
6382
6403
|
"PURGE_PRECONDITION",
|
|
6383
|
-
"MARKER_REFUSED"
|
|
6404
|
+
"MARKER_REFUSED",
|
|
6405
|
+
"MISSING_OPERATIONS"
|
|
6384
6406
|
]);
|
|
6385
6407
|
/**
|
|
6386
6408
|
* A held auth operation must not quarantine: reconciling the two policies needs
|
|
6387
6409
|
* the traffic a quarantine would stop. A protocol refusal concerns one peer,
|
|
6388
|
-
* and quarantine is global to the document.
|
|
6410
|
+
* and quarantine is global to the document. A missing ancestor arrives through
|
|
6411
|
+
* exactly the traffic a quarantine stops.
|
|
6389
6412
|
*/
|
|
6390
6413
|
function quarantinesDocument(errorType) {
|
|
6391
6414
|
return !NON_QUARANTINING_ERROR_TYPES.has(errorType);
|
|
6392
6415
|
}
|
|
6393
6416
|
//#endregion
|
|
6394
6417
|
//#region src/sync/channels/interval-poll-timer.ts
|
|
6418
|
+
/** Lower bound on the derived {@link PollTimerConfig.delegateTimeoutMs}. */
|
|
6419
|
+
const DELEGATE_TIMEOUT_FLOOR_MS = 3e4;
|
|
6395
6420
|
const DEFAULT_CONFIG = {
|
|
6396
6421
|
intervalMs: 2e3,
|
|
6397
6422
|
maxQueueDepth: 100,
|
|
6398
6423
|
backpressureCheckIntervalMs: 500,
|
|
6399
6424
|
retryBaseDelayMs: 1e3,
|
|
6400
6425
|
retryMaxDelayMs: 3e5,
|
|
6401
|
-
startPaused: false
|
|
6426
|
+
startPaused: false,
|
|
6427
|
+
delegateTimeoutMs: DELEGATE_TIMEOUT_FLOOR_MS,
|
|
6428
|
+
queueProbeTimeoutMs: 1e4
|
|
6402
6429
|
};
|
|
6403
6430
|
function calculateBackoffDelay(consecutiveFailures, retryBaseDelayMs, retryMaxDelayMs, random) {
|
|
6404
6431
|
const backoff = Math.min(retryMaxDelayMs, retryBaseDelayMs * Math.pow(2, consecutiveFailures - 1));
|
|
6405
6432
|
return backoff / 2 + random * (backoff / 2);
|
|
6406
6433
|
}
|
|
6407
|
-
/**
|
|
6408
|
-
* Default poll timer using setTimeout.
|
|
6409
|
-
* Waits for delegate completion before scheduling next tick.
|
|
6410
|
-
* Checks queue depth and defers polling when backpressure is detected.
|
|
6411
|
-
*/
|
|
6434
|
+
/** setTimeout poll timer; a stuck tick is cancelled and the next waits for it to settle. */
|
|
6412
6435
|
var IntervalPollTimer = class {
|
|
6413
6436
|
delegate;
|
|
6414
6437
|
timer;
|
|
6438
|
+
watchdog;
|
|
6415
6439
|
running;
|
|
6416
6440
|
paused;
|
|
6417
6441
|
consecutiveFailures;
|
|
6442
|
+
tickToken = 0;
|
|
6443
|
+
ticking = false;
|
|
6444
|
+
tickAbort;
|
|
6445
|
+
cancelledToken;
|
|
6446
|
+
retriggerRequested = false;
|
|
6418
6447
|
queue;
|
|
6419
6448
|
config;
|
|
6420
6449
|
constructor(queue, config = {}) {
|
|
6421
6450
|
this.queue = queue;
|
|
6451
|
+
const intervalMs = config.intervalMs ?? DEFAULT_CONFIG.intervalMs;
|
|
6422
6452
|
this.config = {
|
|
6423
6453
|
...DEFAULT_CONFIG,
|
|
6454
|
+
delegateTimeoutMs: Math.max(DELEGATE_TIMEOUT_FLOOR_MS, intervalMs * 10),
|
|
6424
6455
|
...config
|
|
6425
6456
|
};
|
|
6426
6457
|
this.running = false;
|
|
@@ -6437,54 +6468,25 @@ var IntervalPollTimer = class {
|
|
|
6437
6468
|
}
|
|
6438
6469
|
stop() {
|
|
6439
6470
|
this.running = false;
|
|
6440
|
-
|
|
6441
|
-
|
|
6442
|
-
|
|
6443
|
-
|
|
6444
|
-
|
|
6445
|
-
|
|
6446
|
-
|
|
6447
|
-
const delegate = this.delegate;
|
|
6448
|
-
this.queue.totalSize().then((size) => {
|
|
6449
|
-
if (!this.running) return;
|
|
6450
|
-
if (size > this.config.maxQueueDepth) this.scheduleBackpressureRecheck();
|
|
6451
|
-
else delegate().then(() => {
|
|
6452
|
-
this.consecutiveFailures = 0;
|
|
6453
|
-
this.scheduleNext();
|
|
6454
|
-
}).catch(() => {
|
|
6455
|
-
this.consecutiveFailures++;
|
|
6456
|
-
this.scheduleRetry();
|
|
6457
|
-
});
|
|
6458
|
-
}).catch(() => {
|
|
6459
|
-
this.scheduleNext();
|
|
6460
|
-
});
|
|
6461
|
-
}
|
|
6462
|
-
scheduleNext() {
|
|
6463
|
-
if (!this.running || this.paused) return;
|
|
6464
|
-
this.timer = setTimeout(() => this.tick(), this.config.intervalMs);
|
|
6465
|
-
}
|
|
6466
|
-
scheduleRetry() {
|
|
6467
|
-
if (!this.running || this.paused) return;
|
|
6468
|
-
const delay = calculateBackoffDelay(this.consecutiveFailures, this.config.retryBaseDelayMs, this.config.retryMaxDelayMs, Math.random());
|
|
6469
|
-
this.timer = setTimeout(() => this.tick(), delay);
|
|
6470
|
-
}
|
|
6471
|
-
scheduleBackpressureRecheck() {
|
|
6472
|
-
if (!this.running || this.paused) return;
|
|
6473
|
-
this.timer = setTimeout(() => this.tick(), this.config.backpressureCheckIntervalMs);
|
|
6471
|
+
this.retriggerRequested = false;
|
|
6472
|
+
this.clearTimer();
|
|
6473
|
+
this.clearWatchdog();
|
|
6474
|
+
this.cancelTick(/* @__PURE__ */ new Error("poll timer stopped"));
|
|
6475
|
+
this.tickToken++;
|
|
6476
|
+
this.cancelledToken = void 0;
|
|
6477
|
+
this.ticking = false;
|
|
6474
6478
|
}
|
|
6475
6479
|
pause() {
|
|
6476
6480
|
this.paused = true;
|
|
6477
|
-
|
|
6478
|
-
clearTimeout(this.timer);
|
|
6479
|
-
this.timer = void 0;
|
|
6480
|
-
}
|
|
6481
|
+
this.clearTimer();
|
|
6481
6482
|
}
|
|
6482
6483
|
resume() {
|
|
6483
6484
|
this.paused = false;
|
|
6484
6485
|
if (this.running) this.scheduleNext();
|
|
6485
6486
|
}
|
|
6486
6487
|
triggerNow() {
|
|
6487
|
-
if (this.running
|
|
6488
|
+
if (!this.running || !this.delegate) return;
|
|
6489
|
+
this.tick();
|
|
6488
6490
|
}
|
|
6489
6491
|
isPaused() {
|
|
6490
6492
|
return this.paused;
|
|
@@ -6498,6 +6500,133 @@ var IntervalPollTimer = class {
|
|
|
6498
6500
|
setIntervalMs(ms) {
|
|
6499
6501
|
this.config.intervalMs = ms;
|
|
6500
6502
|
}
|
|
6503
|
+
tick() {
|
|
6504
|
+
if (!this.delegate || !this.running) return;
|
|
6505
|
+
if (this.ticking) {
|
|
6506
|
+
this.retriggerRequested = true;
|
|
6507
|
+
return;
|
|
6508
|
+
}
|
|
6509
|
+
const delegate = this.delegate;
|
|
6510
|
+
const token = ++this.tickToken;
|
|
6511
|
+
this.ticking = true;
|
|
6512
|
+
this.cancelledToken = void 0;
|
|
6513
|
+
const abort = new AbortController();
|
|
6514
|
+
this.tickAbort = abort;
|
|
6515
|
+
this.clearTimer();
|
|
6516
|
+
this.armWatchdog(token);
|
|
6517
|
+
this.measureQueue(abort.signal).then((size) => {
|
|
6518
|
+
if (!this.running) {
|
|
6519
|
+
this.settle(token, "stopped");
|
|
6520
|
+
return;
|
|
6521
|
+
}
|
|
6522
|
+
if (size === "cancelled") {
|
|
6523
|
+
this.settle(token, "failure");
|
|
6524
|
+
return;
|
|
6525
|
+
}
|
|
6526
|
+
if (size === "unknown") {
|
|
6527
|
+
this.runDelegate(delegate, token, abort.signal);
|
|
6528
|
+
return;
|
|
6529
|
+
}
|
|
6530
|
+
if (size > this.config.maxQueueDepth) {
|
|
6531
|
+
this.settle(token, "backpressure");
|
|
6532
|
+
return;
|
|
6533
|
+
}
|
|
6534
|
+
this.runDelegate(delegate, token, abort.signal);
|
|
6535
|
+
});
|
|
6536
|
+
}
|
|
6537
|
+
runDelegate(delegate, token, signal) {
|
|
6538
|
+
delegate(signal).then(() => this.settle(token, "success"), () => this.settle(token, "failure"));
|
|
6539
|
+
}
|
|
6540
|
+
/** `"unknown"` when the probe timed out or rejected; `"cancelled"` when the tick was. */
|
|
6541
|
+
async measureQueue(signal) {
|
|
6542
|
+
const timeoutMs = this.config.queueProbeTimeoutMs;
|
|
6543
|
+
let handle;
|
|
6544
|
+
const expiry = new Promise((resolve) => {
|
|
6545
|
+
if (timeoutMs > 0) handle = setTimeout(() => resolve("unknown"), timeoutMs);
|
|
6546
|
+
});
|
|
6547
|
+
const cancelled = new Promise((resolve) => {
|
|
6548
|
+
if (signal.aborted) {
|
|
6549
|
+
resolve("cancelled");
|
|
6550
|
+
return;
|
|
6551
|
+
}
|
|
6552
|
+
signal.addEventListener("abort", () => resolve("cancelled"), { once: true });
|
|
6553
|
+
});
|
|
6554
|
+
try {
|
|
6555
|
+
return await Promise.race([
|
|
6556
|
+
this.queue.totalSize(),
|
|
6557
|
+
expiry,
|
|
6558
|
+
cancelled
|
|
6559
|
+
]);
|
|
6560
|
+
} catch {
|
|
6561
|
+
return "unknown";
|
|
6562
|
+
} finally {
|
|
6563
|
+
if (handle !== void 0) clearTimeout(handle);
|
|
6564
|
+
}
|
|
6565
|
+
}
|
|
6566
|
+
settle(token, outcome) {
|
|
6567
|
+
if (token !== this.tickToken) return;
|
|
6568
|
+
this.clearWatchdog();
|
|
6569
|
+
this.ticking = false;
|
|
6570
|
+
this.tickAbort = void 0;
|
|
6571
|
+
const cancelled = this.cancelledToken === token;
|
|
6572
|
+
this.cancelledToken = void 0;
|
|
6573
|
+
const effective = cancelled && outcome !== "stopped" ? "failure" : outcome;
|
|
6574
|
+
if (effective === "success") this.consecutiveFailures = 0;
|
|
6575
|
+
else if (effective === "failure") this.consecutiveFailures++;
|
|
6576
|
+
if (effective === "stopped" || !this.running) return;
|
|
6577
|
+
if (this.retriggerRequested) {
|
|
6578
|
+
this.retriggerRequested = false;
|
|
6579
|
+
this.tick();
|
|
6580
|
+
return;
|
|
6581
|
+
}
|
|
6582
|
+
if (effective === "backpressure") this.scheduleBackpressureRecheck();
|
|
6583
|
+
else if (effective === "failure") this.scheduleRetry();
|
|
6584
|
+
else this.scheduleNext();
|
|
6585
|
+
}
|
|
6586
|
+
/** Cancels rather than abandons the tick, so two delegates never overlap. */
|
|
6587
|
+
armWatchdog(token) {
|
|
6588
|
+
const timeoutMs = this.config.delegateTimeoutMs;
|
|
6589
|
+
if (timeoutMs <= 0) return;
|
|
6590
|
+
this.watchdog = setTimeout(() => {
|
|
6591
|
+
this.watchdog = void 0;
|
|
6592
|
+
if (token !== this.tickToken || !this.ticking) return;
|
|
6593
|
+
this.cancelledToken = token;
|
|
6594
|
+
this.cancelTick(/* @__PURE__ */ new Error(`poll delegate exceeded its ${timeoutMs}ms bound`));
|
|
6595
|
+
}, timeoutMs);
|
|
6596
|
+
}
|
|
6597
|
+
cancelTick(reason) {
|
|
6598
|
+
const abort = this.tickAbort;
|
|
6599
|
+
if (abort === void 0 || abort.signal.aborted) return;
|
|
6600
|
+
abort.abort(reason);
|
|
6601
|
+
}
|
|
6602
|
+
clearWatchdog() {
|
|
6603
|
+
if (this.watchdog) {
|
|
6604
|
+
clearTimeout(this.watchdog);
|
|
6605
|
+
this.watchdog = void 0;
|
|
6606
|
+
}
|
|
6607
|
+
}
|
|
6608
|
+
clearTimer() {
|
|
6609
|
+
if (this.timer) {
|
|
6610
|
+
clearTimeout(this.timer);
|
|
6611
|
+
this.timer = void 0;
|
|
6612
|
+
}
|
|
6613
|
+
}
|
|
6614
|
+
scheduleNext() {
|
|
6615
|
+
if (!this.running || this.paused) return;
|
|
6616
|
+
this.clearTimer();
|
|
6617
|
+
this.timer = setTimeout(() => this.tick(), this.config.intervalMs);
|
|
6618
|
+
}
|
|
6619
|
+
scheduleRetry() {
|
|
6620
|
+
if (!this.running || this.paused) return;
|
|
6621
|
+
const delay = calculateBackoffDelay(this.consecutiveFailures, this.config.retryBaseDelayMs, this.config.retryMaxDelayMs, Math.random());
|
|
6622
|
+
this.clearTimer();
|
|
6623
|
+
this.timer = setTimeout(() => this.tick(), delay);
|
|
6624
|
+
}
|
|
6625
|
+
scheduleBackpressureRecheck() {
|
|
6626
|
+
if (!this.running || this.paused) return;
|
|
6627
|
+
this.clearTimer();
|
|
6628
|
+
this.timer = setTimeout(() => this.tick(), this.config.backpressureCheckIntervalMs);
|
|
6629
|
+
}
|
|
6501
6630
|
};
|
|
6502
6631
|
//#endregion
|
|
6503
6632
|
//#region src/sync/purge-refusals.ts
|
|
@@ -6620,6 +6749,7 @@ const getLatestAppliedOrdinal = (syncOps) => {
|
|
|
6620
6749
|
};
|
|
6621
6750
|
//#endregion
|
|
6622
6751
|
//#region src/sync/channels/gql-req-channel.ts
|
|
6752
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 6e4;
|
|
6623
6753
|
/**
|
|
6624
6754
|
* Fields the auth projection added to the sync schema. A remote that predates
|
|
6625
6755
|
* them rejects the whole query for naming one, so they are selected only while
|
|
@@ -6670,8 +6800,18 @@ var GqlRequestChannel = class {
|
|
|
6670
6800
|
failureCount;
|
|
6671
6801
|
lastSuccessUtcMs;
|
|
6672
6802
|
lastFailureUtcMs;
|
|
6673
|
-
|
|
6674
|
-
|
|
6803
|
+
cursorWriters = {
|
|
6804
|
+
inbox: {
|
|
6805
|
+
persisted: 0,
|
|
6806
|
+
requested: 0,
|
|
6807
|
+
tail: Promise.resolve()
|
|
6808
|
+
},
|
|
6809
|
+
outbox: {
|
|
6810
|
+
persisted: 0,
|
|
6811
|
+
requested: 0,
|
|
6812
|
+
tail: Promise.resolve()
|
|
6813
|
+
}
|
|
6814
|
+
};
|
|
6675
6815
|
pushFailureCount = 0;
|
|
6676
6816
|
pushRetryTimer = null;
|
|
6677
6817
|
pushBlocked = false;
|
|
@@ -6698,6 +6838,8 @@ var GqlRequestChannel = class {
|
|
|
6698
6838
|
connectionState = "connecting";
|
|
6699
6839
|
/** Latest unrecoverable error was an auth rejection; cleared on connect. */
|
|
6700
6840
|
requiresAuth = false;
|
|
6841
|
+
polling = false;
|
|
6842
|
+
polledSinceStart = false;
|
|
6701
6843
|
connectionStateCallbacks = /* @__PURE__ */ new Set();
|
|
6702
6844
|
constructor(logger, channelId, remoteName, cursorStorage, config, operationIndex, pollTimer) {
|
|
6703
6845
|
this.logger = logger;
|
|
@@ -6713,7 +6855,8 @@ var GqlRequestChannel = class {
|
|
|
6713
6855
|
collectionId: config.collectionId,
|
|
6714
6856
|
filter: config.filter,
|
|
6715
6857
|
retryBaseDelayMs: config.retryBaseDelayMs,
|
|
6716
|
-
retryMaxDelayMs: config.retryMaxDelayMs
|
|
6858
|
+
retryMaxDelayMs: config.retryMaxDelayMs,
|
|
6859
|
+
requestTimeoutMs: config.requestTimeoutMs ?? 6e4
|
|
6717
6860
|
};
|
|
6718
6861
|
this.isShutdown = false;
|
|
6719
6862
|
this.failureCount = 0;
|
|
@@ -6755,33 +6898,38 @@ var GqlRequestChannel = class {
|
|
|
6755
6898
|
this.outbox.onRemoved((syncOps) => {
|
|
6756
6899
|
for (const syncOp of syncOps) this.markerPushedAt.delete(syncOp.id);
|
|
6757
6900
|
const ordinal = Math.min(getLatestAppliedOrdinal(syncOps), this.unappliedFloor() - 1);
|
|
6758
|
-
|
|
6759
|
-
this.lastPersistedOutboxOrdinal = ordinal;
|
|
6760
|
-
this.cursorStorage.upsert({
|
|
6761
|
-
remoteName: this.remoteName,
|
|
6762
|
-
cursorType: "outbox",
|
|
6763
|
-
cursorOrdinal: ordinal,
|
|
6764
|
-
lastSyncedAtUtcMs: Date.now()
|
|
6765
|
-
}).catch((error) => {
|
|
6766
|
-
this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
|
|
6767
|
-
});
|
|
6768
|
-
}
|
|
6901
|
+
this.persistCursor("outbox", ordinal);
|
|
6769
6902
|
});
|
|
6770
6903
|
this.inbox.onRemoved(() => {
|
|
6771
|
-
|
|
6772
|
-
if (maxOrdinal > this.lastPersistedInboxOrdinal) {
|
|
6773
|
-
this.lastPersistedInboxOrdinal = maxOrdinal;
|
|
6774
|
-
this.cursorStorage.upsert({
|
|
6775
|
-
remoteName: this.remoteName,
|
|
6776
|
-
cursorType: "inbox",
|
|
6777
|
-
cursorOrdinal: maxOrdinal,
|
|
6778
|
-
lastSyncedAtUtcMs: Date.now()
|
|
6779
|
-
}).catch((error) => {
|
|
6780
|
-
this.logger.error("Failed to update inbox cursor for @ChannelId! This is unlikely to cause a problem, but not-optimal: @Error", this.channelId, error);
|
|
6781
|
-
});
|
|
6782
|
-
}
|
|
6904
|
+
this.persistCursor("inbox", this.inbox.ackOrdinal);
|
|
6783
6905
|
});
|
|
6784
6906
|
}
|
|
6907
|
+
/** Advances the watermark only once the write lands; writes per row run one at a time, coalesced. */
|
|
6908
|
+
persistCursor(cursorType, ordinal) {
|
|
6909
|
+
const writer = this.cursorWriters[cursorType];
|
|
6910
|
+
if (ordinal <= Math.max(writer.persisted, writer.requested)) return;
|
|
6911
|
+
writer.requested = ordinal;
|
|
6912
|
+
writer.tail = writer.tail.then(() => this.writeCursor(cursorType));
|
|
6913
|
+
}
|
|
6914
|
+
/** One link of a cursor write chain; never rejects, so the chain survives. */
|
|
6915
|
+
async writeCursor(cursorType) {
|
|
6916
|
+
const writer = this.cursorWriters[cursorType];
|
|
6917
|
+
const ordinal = writer.requested;
|
|
6918
|
+
if (ordinal <= writer.persisted) return;
|
|
6919
|
+
writer.requested = 0;
|
|
6920
|
+
try {
|
|
6921
|
+
await this.cursorStorage.upsert({
|
|
6922
|
+
remoteName: this.remoteName,
|
|
6923
|
+
cursorType,
|
|
6924
|
+
cursorOrdinal: ordinal,
|
|
6925
|
+
lastSyncedAtUtcMs: Date.now()
|
|
6926
|
+
});
|
|
6927
|
+
} catch (error) {
|
|
6928
|
+
this.logger.error("Failed to update @CursorType cursor for @ChannelId at ordinal @Ordinal; the watermark stays put so the next advance retries it: @Error", cursorType, this.channelId, ordinal, error);
|
|
6929
|
+
return;
|
|
6930
|
+
}
|
|
6931
|
+
writer.persisted = Math.max(writer.persisted, ordinal);
|
|
6932
|
+
}
|
|
6785
6933
|
/**
|
|
6786
6934
|
* Shuts down the channel and prevents further operations.
|
|
6787
6935
|
*/
|
|
@@ -6789,7 +6937,7 @@ var GqlRequestChannel = class {
|
|
|
6789
6937
|
this.abortController.abort();
|
|
6790
6938
|
this.bufferedOutbox.flush();
|
|
6791
6939
|
this.isShutdown = true;
|
|
6792
|
-
this.
|
|
6940
|
+
this.stopPolling();
|
|
6793
6941
|
if (this.pushRetryTimer) {
|
|
6794
6942
|
clearTimeout(this.pushRetryTimer);
|
|
6795
6943
|
this.pushRetryTimer = null;
|
|
@@ -6848,31 +6996,31 @@ var GqlRequestChannel = class {
|
|
|
6848
6996
|
}
|
|
6849
6997
|
/**
|
|
6850
6998
|
* Re-touches once when either side's manifest moved; touching is idempotent.
|
|
6851
|
-
* False when
|
|
6999
|
+
* False only when shutting down; a failed refresh raises as a recoverable
|
|
7000
|
+
* poll failure, so the next tick retries it.
|
|
6852
7001
|
*/
|
|
6853
|
-
async refreshManifestsIfStale(manifestRevision, peerManifestRevision) {
|
|
7002
|
+
async refreshManifestsIfStale(manifestRevision, peerManifestRevision, signal) {
|
|
6854
7003
|
if (!this.peerServesAgreement || typeof manifestRevision !== "string") return true;
|
|
6855
7004
|
const local = this.localManifestProvider?.().revision ?? null;
|
|
6856
7005
|
if (manifestRevision === (this.peerManifest?.revision ?? null) && (peerManifestRevision ?? null) === local) return true;
|
|
6857
7006
|
if (this.isShutdown) return false;
|
|
6858
|
-
this.manifestRefresh ??= this.touchRemoteChannel().then(({ ackOrdinal }) => {
|
|
7007
|
+
this.manifestRefresh ??= this.touchRemoteChannel(signal).then(({ ackOrdinal }) => {
|
|
6859
7008
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
6860
7009
|
}).finally(() => {
|
|
6861
7010
|
this.manifestRefresh = void 0;
|
|
6862
7011
|
});
|
|
6863
7012
|
try {
|
|
6864
7013
|
await this.manifestRefresh;
|
|
6865
|
-
return true;
|
|
6866
7014
|
} catch (error) {
|
|
6867
|
-
|
|
6868
|
-
return false;
|
|
7015
|
+
throw new Error(`Manifest refresh failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
6869
7016
|
}
|
|
7017
|
+
return true;
|
|
6870
7018
|
}
|
|
6871
7019
|
/** Re-touches a silent remote in case it was upgraded. */
|
|
6872
|
-
async probeAgreement() {
|
|
7020
|
+
async probeAgreement(signal) {
|
|
6873
7021
|
this.agreementStoppedUtcMs = Date.now();
|
|
6874
7022
|
try {
|
|
6875
|
-
const { ackOrdinal } = await this.touchRemoteChannel();
|
|
7023
|
+
const { ackOrdinal } = await this.touchRemoteChannel(signal);
|
|
6876
7024
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
6877
7025
|
} catch (error) {
|
|
6878
7026
|
this.logger.error("GqlChannel @ChannelId agreement probe failed: @Error", this.channelId, error);
|
|
@@ -6882,21 +7030,38 @@ var GqlRequestChannel = class {
|
|
|
6882
7030
|
* Initializes the channel by registering it on the remote server and starting polling.
|
|
6883
7031
|
*/
|
|
6884
7032
|
async init() {
|
|
6885
|
-
const { ackOrdinal } = await this.touchRemoteChannel();
|
|
7033
|
+
const { ackOrdinal } = await this.touchRemoteChannel(void 0);
|
|
6886
7034
|
const cursors = await this.cursorStorage.list(this.remoteName);
|
|
6887
7035
|
const inboxOrdinal = cursors.find((c) => c.cursorType === "inbox")?.cursorOrdinal ?? 0;
|
|
6888
7036
|
const outboxOrdinal = cursors.find((c) => c.cursorType === "outbox")?.cursorOrdinal ?? 0;
|
|
6889
7037
|
this.inbox.init(inboxOrdinal);
|
|
6890
7038
|
this.outbox.init(outboxOrdinal);
|
|
6891
|
-
this.
|
|
6892
|
-
this.
|
|
7039
|
+
this.cursorWriters.inbox.persisted = inboxOrdinal;
|
|
7040
|
+
this.cursorWriters.outbox.persisted = outboxOrdinal;
|
|
6893
7041
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
6894
|
-
this.pollTimer.setDelegate(() => this.poll());
|
|
7042
|
+
this.pollTimer.setDelegate((signal) => this.poll(signal));
|
|
7043
|
+
this.startPolling();
|
|
7044
|
+
this.transitionConnectionState(this.reachedState());
|
|
7045
|
+
}
|
|
7046
|
+
startPolling() {
|
|
7047
|
+
this.polling = true;
|
|
7048
|
+
this.polledSinceStart = false;
|
|
6895
7049
|
this.pollTimer.start();
|
|
6896
|
-
|
|
7050
|
+
}
|
|
7051
|
+
stopPolling() {
|
|
7052
|
+
this.polling = false;
|
|
7053
|
+
this.polledSinceStart = false;
|
|
7054
|
+
this.pollTimer.stop();
|
|
7055
|
+
}
|
|
7056
|
+
/**
|
|
7057
|
+
* `"connected"` is earned by a poll completed since the loop last started; a
|
|
7058
|
+
* timer that polls only on demand earns it with the touch that started it.
|
|
7059
|
+
*/
|
|
7060
|
+
reachedState() {
|
|
7061
|
+
return this.polledSinceStart || this.pollTimer.isPaused?.() === true ? "connected" : "connecting";
|
|
6897
7062
|
}
|
|
6898
7063
|
transitionConnectionState(next) {
|
|
6899
|
-
if (next === "connected") this.requiresAuth = false;
|
|
7064
|
+
if (next === "connected" || next === "connecting") this.requiresAuth = false;
|
|
6900
7065
|
if (this.connectionState === next) return;
|
|
6901
7066
|
this.connectionState = next;
|
|
6902
7067
|
const snapshot = this.getConnectionState();
|
|
@@ -6907,23 +7072,26 @@ var GqlRequestChannel = class {
|
|
|
6907
7072
|
}
|
|
6908
7073
|
}
|
|
6909
7074
|
/**
|
|
6910
|
-
* Polls the remote for new sync envelopes.
|
|
7075
|
+
* Polls the remote for new sync envelopes; any failure of the body is recorded.
|
|
6911
7076
|
*/
|
|
6912
|
-
async poll() {
|
|
7077
|
+
async poll(signal) {
|
|
6913
7078
|
if (this.isShutdown) return;
|
|
6914
|
-
if (!this.peerServesAgreement && Date.now() - this.agreementStoppedUtcMs >= AGREEMENT_PROBE_INTERVAL_MS) await this.probeAgreement();
|
|
6915
|
-
let response;
|
|
6916
7079
|
try {
|
|
6917
|
-
|
|
7080
|
+
await this.pollOnce(signal);
|
|
6918
7081
|
} catch (error) {
|
|
6919
7082
|
if (!this.handlePollError(error)) throw error;
|
|
6920
|
-
return;
|
|
6921
7083
|
}
|
|
7084
|
+
}
|
|
7085
|
+
async pollOnce(signal) {
|
|
7086
|
+
if (!this.peerServesAgreement && Date.now() - this.agreementStoppedUtcMs >= AGREEMENT_PROBE_INTERVAL_MS) await this.probeAgreement(signal);
|
|
7087
|
+
const response = await this.pollSyncEnvelopes(this.inbox.ackOrdinal, this.inbox.latestOrdinal, signal);
|
|
7088
|
+
this.throwIfPollCancelled(signal);
|
|
6922
7089
|
const { envelopes, ackOrdinal, deadLetters, hasMore, manifestRevision, peerManifestRevision } = response;
|
|
6923
7090
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
6924
7091
|
this.retireRefusedMarkers(deadLetters);
|
|
6925
7092
|
this.repushUnackedMarkers();
|
|
6926
|
-
if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision)) return;
|
|
7093
|
+
if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision, signal)) return;
|
|
7094
|
+
this.throwIfPollCancelled(signal);
|
|
6927
7095
|
const allSyncOps = [];
|
|
6928
7096
|
for (const envelope of envelopes) if (envelope.type.toLowerCase() === "operations" && envelope.operations) {
|
|
6929
7097
|
const syncOps = envelopesToSyncOperations(envelope, this.remoteName);
|
|
@@ -6940,6 +7108,7 @@ var GqlRequestChannel = class {
|
|
|
6940
7108
|
}
|
|
6941
7109
|
this.lastSuccessUtcMs = Date.now();
|
|
6942
7110
|
this.failureCount = 0;
|
|
7111
|
+
this.polledSinceStart = true;
|
|
6943
7112
|
this.transitionConnectionState("connected");
|
|
6944
7113
|
}
|
|
6945
7114
|
/**
|
|
@@ -6982,7 +7151,7 @@ var GqlRequestChannel = class {
|
|
|
6982
7151
|
const channelError = new ChannelError("inbox", err);
|
|
6983
7152
|
this.logger.error("GqlChannel poll error (@FailureCount, @Classification): @Error", this.failureCount, classification, channelError);
|
|
6984
7153
|
if (classification === "unrecoverable") {
|
|
6985
|
-
this.
|
|
7154
|
+
this.stopPolling();
|
|
6986
7155
|
this.requiresAuth = isDriveAuthError(err);
|
|
6987
7156
|
this.transitionConnectionState("error");
|
|
6988
7157
|
return true;
|
|
@@ -6998,19 +7167,19 @@ var GqlRequestChannel = class {
|
|
|
6998
7167
|
if (this.isRecovering) return;
|
|
6999
7168
|
this.isRecovering = true;
|
|
7000
7169
|
this.logger.info("GqlChannel @ChannelId not found on remote, re-registering...", this.channelId);
|
|
7001
|
-
this.
|
|
7170
|
+
this.stopPolling();
|
|
7002
7171
|
const attemptRecovery = (attempt) => {
|
|
7003
7172
|
if (this.isShutdown) {
|
|
7004
7173
|
this.isRecovering = false;
|
|
7005
7174
|
return;
|
|
7006
7175
|
}
|
|
7007
|
-
this.touchRemoteChannel().then(({ ackOrdinal }) => {
|
|
7176
|
+
this.touchRemoteChannel(void 0).then(({ ackOrdinal }) => {
|
|
7008
7177
|
this.logger.info("GqlChannel @ChannelId re-registered successfully", this.channelId);
|
|
7009
7178
|
this.isRecovering = false;
|
|
7010
7179
|
this.failureCount = 0;
|
|
7011
7180
|
if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
|
|
7012
|
-
this.
|
|
7013
|
-
this.transitionConnectionState(
|
|
7181
|
+
this.startPolling();
|
|
7182
|
+
this.transitionConnectionState(this.reachedState());
|
|
7014
7183
|
this.resumePushAfterRecovery();
|
|
7015
7184
|
}).catch((recoveryError) => {
|
|
7016
7185
|
const err = recoveryError instanceof Error ? recoveryError : new Error(String(recoveryError));
|
|
@@ -7050,7 +7219,7 @@ var GqlRequestChannel = class {
|
|
|
7050
7219
|
/**
|
|
7051
7220
|
* Queries the remote GraphQL endpoint for sync envelopes.
|
|
7052
7221
|
*/
|
|
7053
|
-
async pollSyncEnvelopes(ackOrdinal, latestOrdinal) {
|
|
7222
|
+
async pollSyncEnvelopes(ackOrdinal, latestOrdinal, signal) {
|
|
7054
7223
|
const variables = {
|
|
7055
7224
|
channelId: this.channelId,
|
|
7056
7225
|
outboxAck: ackOrdinal,
|
|
@@ -7066,7 +7235,7 @@ var GqlRequestChannel = class {
|
|
|
7066
7235
|
...variables,
|
|
7067
7236
|
manifestRevision: revision,
|
|
7068
7237
|
refusals
|
|
7069
|
-
} : variables);
|
|
7238
|
+
} : variables, signal);
|
|
7070
7239
|
break;
|
|
7071
7240
|
} catch (error) {
|
|
7072
7241
|
if (this.rejectsAgreementFields(error)) {
|
|
@@ -7197,7 +7366,7 @@ var GqlRequestChannel = class {
|
|
|
7197
7366
|
* Registers or updates this channel on the remote server via GraphQL mutation.
|
|
7198
7367
|
* Returns the remote's ack ordinal so the client can trim its outbox.
|
|
7199
7368
|
*/
|
|
7200
|
-
async touchRemoteChannel() {
|
|
7369
|
+
async touchRemoteChannel(signal) {
|
|
7201
7370
|
let sinceTimestampUtcMs = "0";
|
|
7202
7371
|
try {
|
|
7203
7372
|
const result = await this.operationIndex.getLatestTimestampForCollection(this.config.collectionId.key);
|
|
@@ -7224,7 +7393,7 @@ var GqlRequestChannel = class {
|
|
|
7224
7393
|
},
|
|
7225
7394
|
sinceTimestampUtcMs,
|
|
7226
7395
|
...manifest ? { manifest } : {}
|
|
7227
|
-
} });
|
|
7396
|
+
} }, signal);
|
|
7228
7397
|
};
|
|
7229
7398
|
let data;
|
|
7230
7399
|
try {
|
|
@@ -7251,7 +7420,7 @@ var GqlRequestChannel = class {
|
|
|
7251
7420
|
this.isPushing = false;
|
|
7252
7421
|
this.pushBlocked = false;
|
|
7253
7422
|
this.pushFailureCount = 0;
|
|
7254
|
-
if (this.connectionState === "reconnecting" || this.connectionState === "error") this.transitionConnectionState(
|
|
7423
|
+
if (this.polling && (this.connectionState === "reconnecting" || this.connectionState === "error")) this.transitionConnectionState(this.reachedState());
|
|
7255
7424
|
this.drainOutbox();
|
|
7256
7425
|
}).catch((error) => {
|
|
7257
7426
|
this.isPushing = false;
|
|
@@ -7268,7 +7437,7 @@ var GqlRequestChannel = class {
|
|
|
7268
7437
|
this.pushFailureCount++;
|
|
7269
7438
|
this.pushBlocked = true;
|
|
7270
7439
|
this.logger.error("GqlChannel push failed (attempt @FailureCount), will retry: @Error", this.pushFailureCount, err);
|
|
7271
|
-
this.transitionConnectionState("reconnecting");
|
|
7440
|
+
if (this.polling) this.transitionConnectionState("reconnecting");
|
|
7272
7441
|
this.schedulePushRetry();
|
|
7273
7442
|
} else {
|
|
7274
7443
|
const channelError = new ChannelError("outbox", err);
|
|
@@ -7342,6 +7511,7 @@ var GqlRequestChannel = class {
|
|
|
7342
7511
|
if (error.statusCode !== void 0 && error.statusCode >= 500) return "recoverable";
|
|
7343
7512
|
return "unrecoverable";
|
|
7344
7513
|
case "parse": return "recoverable";
|
|
7514
|
+
case "timeout": return "recoverable";
|
|
7345
7515
|
case "graphql": return isRecoverableGraphQLError(error) ? "recoverable" : "unrecoverable";
|
|
7346
7516
|
case "missing-data": return "unrecoverable";
|
|
7347
7517
|
}
|
|
@@ -7361,7 +7531,7 @@ var GqlRequestChannel = class {
|
|
|
7361
7531
|
await this.executeGraphQL(pushMutation(gatedUnder !== void 0), {
|
|
7362
7532
|
envelopes: this.envelopesFor(syncOps),
|
|
7363
7533
|
...gatedUnder !== void 0 ? { peerManifestRevision: gatedUnder } : {}
|
|
7364
|
-
});
|
|
7534
|
+
}, void 0);
|
|
7365
7535
|
return;
|
|
7366
7536
|
} catch (error) {
|
|
7367
7537
|
if (gatedUnder === void 0 || !this.rejectsAgreementFields(error)) throw error;
|
|
@@ -7370,7 +7540,7 @@ var GqlRequestChannel = class {
|
|
|
7370
7540
|
const unsent = new Set(this.outbox.items);
|
|
7371
7541
|
const remaining = syncOps.filter((syncOp) => unsent.has(syncOp));
|
|
7372
7542
|
if (remaining.length === 0) return;
|
|
7373
|
-
await this.executeGraphQL(pushMutation(false), { envelopes: this.envelopesFor(remaining) });
|
|
7543
|
+
await this.executeGraphQL(pushMutation(false), { envelopes: this.envelopesFor(remaining) }, void 0);
|
|
7374
7544
|
}
|
|
7375
7545
|
/** One envelope per SyncOperation, with key/dependsOn for batch ordering. */
|
|
7376
7546
|
envelopesFor(syncOps) {
|
|
@@ -7411,36 +7581,95 @@ var GqlRequestChannel = class {
|
|
|
7411
7581
|
this.logger.error("JWT handler failed: @Error", error);
|
|
7412
7582
|
}
|
|
7413
7583
|
}
|
|
7414
|
-
/**
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
|
|
7584
|
+
/** Aborted by shutdown, tick cancellation or the deadline; `expired` bounds what ignores it. */
|
|
7585
|
+
requestDeadline(timeoutMs, tick) {
|
|
7586
|
+
const controller = new AbortController();
|
|
7587
|
+
const onShutdown = () => controller.abort(this.abortController.signal.reason);
|
|
7588
|
+
if (this.abortController.signal.aborted) onShutdown();
|
|
7589
|
+
else this.abortController.signal.addEventListener("abort", onShutdown, { once: true });
|
|
7590
|
+
let expireTick = () => void 0;
|
|
7591
|
+
const onTickCancelled = () => {
|
|
7592
|
+
controller.abort(tick?.reason);
|
|
7593
|
+
expireTick();
|
|
7594
|
+
};
|
|
7595
|
+
if (tick?.aborted) onTickCancelled();
|
|
7596
|
+
else tick?.addEventListener("abort", onTickCancelled, { once: true });
|
|
7597
|
+
let timer;
|
|
7598
|
+
const expired = new Promise((resolve) => {
|
|
7599
|
+
expireTick = () => resolve();
|
|
7600
|
+
if (tick?.aborted) resolve();
|
|
7601
|
+
if (timeoutMs > 0) timer = setTimeout(() => {
|
|
7602
|
+
controller.abort(/* @__PURE__ */ new Error("request deadline exceeded"));
|
|
7603
|
+
resolve();
|
|
7604
|
+
}, timeoutMs);
|
|
7605
|
+
});
|
|
7606
|
+
return {
|
|
7607
|
+
signal: controller.signal,
|
|
7608
|
+
expired,
|
|
7609
|
+
timeoutMs,
|
|
7610
|
+
dispose: () => {
|
|
7611
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
7612
|
+
this.abortController.signal.removeEventListener("abort", onShutdown);
|
|
7613
|
+
tick?.removeEventListener("abort", onTickCancelled);
|
|
7614
|
+
}
|
|
7615
|
+
};
|
|
7616
|
+
}
|
|
7617
|
+
/** A cancelled poll must not touch mailboxes or cursors the next poll will. */
|
|
7618
|
+
throwIfPollCancelled(signal) {
|
|
7619
|
+
if (signal?.aborted !== true) return;
|
|
7620
|
+
const reason = signal.reason;
|
|
7621
|
+
throw reason instanceof Error ? reason : /* @__PURE__ */ new Error(`poll cancelled: ${String(reason)}`);
|
|
7622
|
+
}
|
|
7623
|
+
async withDeadline(work, deadline, what) {
|
|
7624
|
+
const timedOut = Symbol("timed-out");
|
|
7625
|
+
const outcome = await Promise.race([work, deadline.expired.then(() => timedOut)]);
|
|
7626
|
+
if (outcome === timedOut) throw new GraphQLRequestError(`GraphQL ${what} timed out after ${deadline.timeoutMs}ms`, "timeout");
|
|
7627
|
+
return outcome;
|
|
7628
|
+
}
|
|
7629
|
+
/** Executes a GraphQL query or mutation against the remote endpoint. */
|
|
7630
|
+
async executeGraphQL(query, variables, signal) {
|
|
7418
7631
|
const headers = { "Content-Type": "application/json" };
|
|
7419
|
-
const authHeader = await this.getAuthorizationHeader();
|
|
7420
|
-
if (authHeader) headers["Authorization"] = authHeader;
|
|
7421
7632
|
const operationName = query.match(/(?:query|mutation)\s+(\w+)/)?.[1] ?? "unknown";
|
|
7633
|
+
const timeoutMs = this.config.requestTimeoutMs ?? 6e4;
|
|
7634
|
+
const deadline = this.requestDeadline(timeoutMs, signal);
|
|
7635
|
+
let authHeader;
|
|
7636
|
+
try {
|
|
7637
|
+
authHeader = await this.withDeadline(this.getAuthorizationHeader(), deadline, `${operationName} authorization`);
|
|
7638
|
+
} catch (error) {
|
|
7639
|
+
deadline.dispose();
|
|
7640
|
+
throw error;
|
|
7641
|
+
}
|
|
7642
|
+
if (authHeader) headers["Authorization"] = authHeader;
|
|
7422
7643
|
this.logger.verbose("GQL request @channelId @operation @url vars=@variables", this.channelId, operationName, this.config.url, JSON.stringify(variables));
|
|
7423
7644
|
const fetchFn = this.config.fetchFn ?? fetch;
|
|
7424
7645
|
let response;
|
|
7425
7646
|
try {
|
|
7426
|
-
response = await fetchFn(this.config.url, {
|
|
7647
|
+
response = await this.withDeadline(fetchFn(this.config.url, {
|
|
7427
7648
|
method: "POST",
|
|
7428
7649
|
headers,
|
|
7429
7650
|
body: JSON.stringify({
|
|
7430
7651
|
query,
|
|
7431
7652
|
variables
|
|
7432
7653
|
}),
|
|
7433
|
-
signal:
|
|
7434
|
-
});
|
|
7654
|
+
signal: deadline.signal
|
|
7655
|
+
}), deadline, `${operationName} request`);
|
|
7435
7656
|
} catch (error) {
|
|
7657
|
+
deadline.dispose();
|
|
7658
|
+
if (error instanceof GraphQLRequestError) throw error;
|
|
7436
7659
|
throw new GraphQLRequestError(`GraphQL request failed: ${error instanceof Error ? error.message : String(error)}`, "network");
|
|
7437
7660
|
}
|
|
7438
|
-
if (!response.ok)
|
|
7661
|
+
if (!response.ok) {
|
|
7662
|
+
deadline.dispose();
|
|
7663
|
+
throw new GraphQLRequestError(`GraphQL request failed: ${response.status} ${response.statusText}`, "http", response.status);
|
|
7664
|
+
}
|
|
7439
7665
|
let result;
|
|
7440
7666
|
try {
|
|
7441
|
-
result = await response.json();
|
|
7667
|
+
result = await this.withDeadline(response.json(), deadline, `${operationName} response body`);
|
|
7442
7668
|
} catch (error) {
|
|
7669
|
+
if (error instanceof GraphQLRequestError) throw error;
|
|
7443
7670
|
throw new GraphQLRequestError(`Failed to parse GraphQL response: ${error instanceof Error ? error.message : String(error)}`, "parse");
|
|
7671
|
+
} finally {
|
|
7672
|
+
deadline.dispose();
|
|
7444
7673
|
}
|
|
7445
7674
|
this.logger.verbose("GQL response @channelId @operation status=@status data=@data errors=@errors", this.channelId, operationName, response.status, JSON.stringify(result.data), result.errors ? JSON.stringify(result.errors) : "none");
|
|
7446
7675
|
if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql", void 0, result.errors.map((error) => error.extensions?.code));
|
|
@@ -7524,6 +7753,12 @@ var GqlRequestChannelFactory = class {
|
|
|
7524
7753
|
}
|
|
7525
7754
|
if (retryBaseDelayMs !== void 0) gqlConfig.retryBaseDelayMs = retryBaseDelayMs;
|
|
7526
7755
|
if (retryMaxDelayMs !== void 0) gqlConfig.retryMaxDelayMs = retryMaxDelayMs;
|
|
7756
|
+
let requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS;
|
|
7757
|
+
if (config.parameters.requestTimeoutMs !== void 0) {
|
|
7758
|
+
if (typeof config.parameters.requestTimeoutMs !== "number") throw new Error("\"requestTimeoutMs\" parameter must be a number");
|
|
7759
|
+
requestTimeoutMs = config.parameters.requestTimeoutMs;
|
|
7760
|
+
}
|
|
7761
|
+
gqlConfig.requestTimeoutMs = requestTimeoutMs;
|
|
7527
7762
|
let maxQueueDepth;
|
|
7528
7763
|
if (config.parameters.maxQueueDepth !== void 0) {
|
|
7529
7764
|
if (typeof config.parameters.maxQueueDepth !== "number") throw new Error("\"maxQueueDepth\" parameter must be a number");
|
|
@@ -7540,6 +7775,7 @@ var GqlRequestChannelFactory = class {
|
|
|
7540
7775
|
...retryMaxDelayMs !== void 0 && { retryMaxDelayMs },
|
|
7541
7776
|
...maxQueueDepth !== void 0 && { maxQueueDepth },
|
|
7542
7777
|
...backpressureCheckIntervalMs !== void 0 && { backpressureCheckIntervalMs },
|
|
7778
|
+
delegateTimeoutMs: requestTimeoutMs > 0 ? Math.max(DELEGATE_TIMEOUT_FLOOR_MS, requestTimeoutMs * 6) : 0,
|
|
7543
7779
|
startPaused: options?.pollBehavior === "manual"
|
|
7544
7780
|
});
|
|
7545
7781
|
return new GqlRequestChannel(this.logger, remoteId, remoteName, cursorStorage, gqlConfig, operationIndex, pollTimer);
|
|
@@ -7901,7 +8137,11 @@ var KyselySyncDeadLetterStorage = class {
|
|
|
7901
8137
|
await acquirePurgeLocks(trx, [deadLetter.documentId], "shared");
|
|
7902
8138
|
if ((await findPurged(trx, [deadLetter.documentId])).size > 0) throw new DocumentPurgedError(deadLetter.documentId);
|
|
7903
8139
|
const insertable = deadLetterRecordToRow(deadLetter);
|
|
7904
|
-
await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").
|
|
8140
|
+
await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").doUpdateSet((eb) => ({
|
|
8141
|
+
error_source: eb.ref("excluded.error_source"),
|
|
8142
|
+
error_message: eb.ref("excluded.error_message"),
|
|
8143
|
+
error_type: eb.ref("excluded.error_type")
|
|
8144
|
+
}))).execute();
|
|
7905
8145
|
});
|
|
7906
8146
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
7907
8147
|
}
|
|
@@ -7919,9 +8159,11 @@ var KyselySyncDeadLetterStorage = class {
|
|
|
7919
8159
|
});
|
|
7920
8160
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
7921
8161
|
}
|
|
7922
|
-
async listQuarantinedDocumentIds(signal) {
|
|
8162
|
+
async listQuarantinedDocumentIds(signal, exceptIds = []) {
|
|
7923
8163
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
7924
|
-
|
|
8164
|
+
let query = this.db.selectFrom("sync_dead_letters").select(["document_id", "error_type"]).distinct();
|
|
8165
|
+
if (exceptIds.length > 0) query = query.where("id", "not in", [...exceptIds]);
|
|
8166
|
+
const rows = await query.execute();
|
|
7925
8167
|
if (signal?.aborted) throw new Error("Operation aborted");
|
|
7926
8168
|
const quarantined = /* @__PURE__ */ new Set();
|
|
7927
8169
|
for (const row of rows) if (quarantinesDocument(row.error_type ?? "UNCLASSIFIED")) quarantined.add(row.document_id);
|
|
@@ -8672,7 +8914,7 @@ const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
|
|
|
8672
8914
|
*/
|
|
8673
8915
|
function isCredentialOrNetworkError(error) {
|
|
8674
8916
|
if (isDriveAuthError(error)) return true;
|
|
8675
|
-
return error instanceof GraphQLRequestError && error.category === "network";
|
|
8917
|
+
return error instanceof GraphQLRequestError && (error.category === "network" || error.category === "timeout");
|
|
8676
8918
|
}
|
|
8677
8919
|
const holdKey = (documentId, branch) => `${documentId}\u0000${branch}`;
|
|
8678
8920
|
const VERSION_CACHE_CAP = 1e4;
|
|
@@ -8719,7 +8961,10 @@ var SyncManager = class {
|
|
|
8719
8961
|
config;
|
|
8720
8962
|
connectionStateUnsubscribes = /* @__PURE__ */ new Map();
|
|
8721
8963
|
quarantinedDocumentIds = /* @__PURE__ */ new Set();
|
|
8964
|
+
parkedByQuarantine = /* @__PURE__ */ new WeakSet();
|
|
8722
8965
|
purgedDocumentIds = /* @__PURE__ */ new Set();
|
|
8966
|
+
requeuedDeadLetterIds = /* @__PURE__ */ new Map();
|
|
8967
|
+
requeuesInFlight = /* @__PURE__ */ new Set();
|
|
8723
8968
|
purges;
|
|
8724
8969
|
delivery;
|
|
8725
8970
|
forgetDocument;
|
|
@@ -8731,6 +8976,8 @@ var SyncManager = class {
|
|
|
8731
8976
|
derivingOutboxes = 0;
|
|
8732
8977
|
pruneDrainDeferred = false;
|
|
8733
8978
|
removing = /* @__PURE__ */ new Set();
|
|
8979
|
+
resets = /* @__PURE__ */ new Map();
|
|
8980
|
+
queuedResets = /* @__PURE__ */ new Map();
|
|
8734
8981
|
lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
|
|
8735
8982
|
watermark;
|
|
8736
8983
|
owed = /* @__PURE__ */ new Map();
|
|
@@ -9042,11 +9289,18 @@ var SyncManager = class {
|
|
|
9042
9289
|
if (stale.length > 0) remote.channel.outbox.remove(...stale);
|
|
9043
9290
|
const dead = remote.channel.deadLetter.items.filter((item) => item.documentId === documentId);
|
|
9044
9291
|
if (dead.length > 0) remote.channel.deadLetter.remove(...dead);
|
|
9292
|
+
const parked = remote.channel.inbox.items.filter((item) => item.documentId === documentId && this.parkedByQuarantine.has(item));
|
|
9293
|
+
for (const item of parked) {
|
|
9294
|
+
this.parkedByQuarantine.delete(item);
|
|
9295
|
+
item.executed();
|
|
9296
|
+
}
|
|
9297
|
+
if (parked.length > 0) remote.channel.inbox.remove(...parked);
|
|
9045
9298
|
}
|
|
9046
9299
|
}
|
|
9047
9300
|
async forgetRemote(name) {
|
|
9048
9301
|
this.records.delete(name);
|
|
9049
9302
|
this.heldKeys.delete(name);
|
|
9303
|
+
for (const [id, pending] of [...this.requeuedDeadLetterIds]) if (pending.remoteName === name) this.requeuedDeadLetterIds.delete(id);
|
|
9050
9304
|
await this.holds.removeRemote(name);
|
|
9051
9305
|
await Promise.allSettled(this.markerWritesOf([name]));
|
|
9052
9306
|
await this.markerStorage.removeRemote(name);
|
|
@@ -9300,28 +9554,14 @@ var SyncManager = class {
|
|
|
9300
9554
|
};
|
|
9301
9555
|
this.remotes.set(name, remote);
|
|
9302
9556
|
this.records.set(name, meta);
|
|
9303
|
-
await this.
|
|
9304
|
-
await this.restoreReceivedMarkers(remote);
|
|
9305
|
-
const unheard = [...remote.channel.inbox.items];
|
|
9306
|
-
this.wireChannelCallbacks(remote);
|
|
9557
|
+
const unheard = await this.wireRemote(remote);
|
|
9307
9558
|
try {
|
|
9308
9559
|
await channel.init();
|
|
9309
9560
|
} catch (error) {
|
|
9310
9561
|
await this.dropRemoteAfterFailedInit(remote, !isCredentialOrNetworkError(error));
|
|
9311
9562
|
throw error;
|
|
9312
9563
|
}
|
|
9313
|
-
|
|
9314
|
-
await this.peerUpdates.get(name);
|
|
9315
|
-
this.owe(name, await this.watermarkHead());
|
|
9316
|
-
const backfillController = new AbortController();
|
|
9317
|
-
this.backfillAbortControllers.set(name, backfillController);
|
|
9318
|
-
this.updateOutbox(remote, 0, "backfill", backfillController.signal).catch((error) => {
|
|
9319
|
-
if (backfillController.signal.aborted) return;
|
|
9320
|
-
this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.meta.name, error instanceof Error ? error : new Error(String(error)));
|
|
9321
|
-
}).finally(() => {
|
|
9322
|
-
this.backfillAbortControllers.delete(name);
|
|
9323
|
-
this.drainPrunes();
|
|
9324
|
-
});
|
|
9564
|
+
await this.activateRemote(remote, unheard, 0);
|
|
9325
9565
|
return remote;
|
|
9326
9566
|
}
|
|
9327
9567
|
triggerPull(name) {
|
|
@@ -9332,6 +9572,7 @@ var SyncManager = class {
|
|
|
9332
9572
|
async remove(name) {
|
|
9333
9573
|
const remote = this.remotes.get(name);
|
|
9334
9574
|
if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
|
|
9575
|
+
if (this.resets.has(name) || this.queuedResets.has(name)) throw new Error(`Remote with name '${name}' is being reset`);
|
|
9335
9576
|
this.removing.add(name);
|
|
9336
9577
|
this.owed.delete(name);
|
|
9337
9578
|
this.derivedThrough.delete(name);
|
|
@@ -9382,6 +9623,11 @@ var SyncManager = class {
|
|
|
9382
9623
|
backfillController.abort();
|
|
9383
9624
|
this.backfillAbortControllers.delete(name);
|
|
9384
9625
|
}
|
|
9626
|
+
for (const [id, pending] of [...this.requeuedDeadLetterIds]) {
|
|
9627
|
+
if (pending.remoteName !== name) continue;
|
|
9628
|
+
const item = remote.channel.inbox.get(id);
|
|
9629
|
+
if (item && this.parkedByQuarantine.has(item)) this.requeuedDeadLetterIds.delete(id);
|
|
9630
|
+
}
|
|
9385
9631
|
try {
|
|
9386
9632
|
await remote.channel.shutdown();
|
|
9387
9633
|
} finally {
|
|
@@ -9421,6 +9667,255 @@ var SyncManager = class {
|
|
|
9421
9667
|
async receiptsStored(remoteNames) {
|
|
9422
9668
|
await Promise.all(this.markerWritesOf(remoteNames));
|
|
9423
9669
|
}
|
|
9670
|
+
/** Never joins a running rebuild: it may have read state older than the request. */
|
|
9671
|
+
resetChannel(remoteName) {
|
|
9672
|
+
const queued = this.queuedResets.get(remoteName);
|
|
9673
|
+
if (queued) return queued;
|
|
9674
|
+
const running = this.resets.get(remoteName);
|
|
9675
|
+
if (!running) return this.startReset(remoteName);
|
|
9676
|
+
const next = running.catch(() => void 0).then(() => {
|
|
9677
|
+
this.queuedResets.delete(remoteName);
|
|
9678
|
+
return this.startReset(remoteName);
|
|
9679
|
+
});
|
|
9680
|
+
this.queuedResets.set(remoteName, next);
|
|
9681
|
+
return next;
|
|
9682
|
+
}
|
|
9683
|
+
startReset(name) {
|
|
9684
|
+
const reset = this.rebuildChannel(name).finally(() => {
|
|
9685
|
+
this.resets.delete(name);
|
|
9686
|
+
});
|
|
9687
|
+
this.resets.set(name, reset);
|
|
9688
|
+
return reset;
|
|
9689
|
+
}
|
|
9690
|
+
/** On failure the remote is unregistered, record kept, and can be reset again. */
|
|
9691
|
+
async rebuildChannel(name) {
|
|
9692
|
+
if (this.isShutdown) throw new Error("SyncManager is shutdown and cannot reset remotes");
|
|
9693
|
+
if (this.removing.has(name)) throw new Error(`Remote with name '${name}' is being removed`);
|
|
9694
|
+
const live = this.remotes.get(name);
|
|
9695
|
+
const meta = live?.meta ?? this.records.get(name);
|
|
9696
|
+
if (!meta) throw new Error(`Remote with name '${name}' does not exist`);
|
|
9697
|
+
this.removing.add(name);
|
|
9698
|
+
const requeued = new Set([...this.requeuedDeadLetterIds].filter(([, pending]) => pending.remoteName === name).map(([id]) => id));
|
|
9699
|
+
if (live) try {
|
|
9700
|
+
await this.teardownRemoteResources(live);
|
|
9701
|
+
} catch (error) {
|
|
9702
|
+
this.abandonReset(name, error);
|
|
9703
|
+
throw error;
|
|
9704
|
+
}
|
|
9705
|
+
if (this.isShutdown) {
|
|
9706
|
+
const error = /* @__PURE__ */ new Error("SyncManager shut down during the reset");
|
|
9707
|
+
this.abandonReset(name, error);
|
|
9708
|
+
throw error;
|
|
9709
|
+
}
|
|
9710
|
+
let channel;
|
|
9711
|
+
try {
|
|
9712
|
+
channel = this.channelFactory.instance(meta.id, meta.name, meta.channelConfig, this.cursorStorage, meta.collectionId, meta.filter, this.operationIndex, meta.options);
|
|
9713
|
+
} catch (error) {
|
|
9714
|
+
this.abandonReset(name, error);
|
|
9715
|
+
throw error;
|
|
9716
|
+
}
|
|
9717
|
+
const fresh = {
|
|
9718
|
+
meta,
|
|
9719
|
+
channel
|
|
9720
|
+
};
|
|
9721
|
+
this.remotes.set(name, fresh);
|
|
9722
|
+
if (live) this.carryParked(live, fresh, requeued);
|
|
9723
|
+
let unheard;
|
|
9724
|
+
try {
|
|
9725
|
+
unheard = await this.wireRemote(fresh, true);
|
|
9726
|
+
} catch (error) {
|
|
9727
|
+
this.removing.delete(name);
|
|
9728
|
+
await this.failReset(fresh, error);
|
|
9729
|
+
throw error;
|
|
9730
|
+
}
|
|
9731
|
+
this.removing.delete(name);
|
|
9732
|
+
try {
|
|
9733
|
+
await channel.init();
|
|
9734
|
+
} catch (error) {
|
|
9735
|
+
await this.failReset(fresh, error);
|
|
9736
|
+
throw error;
|
|
9737
|
+
}
|
|
9738
|
+
await this.activateRemote(fresh, unheard, channel.outbox.ackOrdinal);
|
|
9739
|
+
}
|
|
9740
|
+
/** A pushing remote never resends them, so the fresh inbox parks them again. */
|
|
9741
|
+
carryParked(live, fresh, requeued) {
|
|
9742
|
+
for (const item of live.channel.inbox.items) {
|
|
9743
|
+
if (!this.parkedByQuarantine.has(item) || requeued.has(item.id)) continue;
|
|
9744
|
+
const copy = new SyncOperation(item.id, item.jobId, item.jobDependencies, item.remoteName, item.documentId, item.scopes, item.branch, item.operations);
|
|
9745
|
+
fresh.channel.inbox.add(copy);
|
|
9746
|
+
fresh.channel.inbox.hold?.(copy);
|
|
9747
|
+
}
|
|
9748
|
+
}
|
|
9749
|
+
/** Before a fresh channel is registered: nothing to tear down. */
|
|
9750
|
+
abandonReset(name, error) {
|
|
9751
|
+
this.logResetFailure(name, error);
|
|
9752
|
+
this.remotes.delete(name);
|
|
9753
|
+
this.removing.delete(name);
|
|
9754
|
+
}
|
|
9755
|
+
async failReset(fresh, error) {
|
|
9756
|
+
this.logResetFailure(fresh.meta.name, error);
|
|
9757
|
+
await this.dropRemoteAfterFailedInit(fresh, false);
|
|
9758
|
+
}
|
|
9759
|
+
logResetFailure(name, error) {
|
|
9760
|
+
this.logger.error("Resetting remote @name failed; it stays down until reset again or restarted: @error", name, error instanceof Error ? error.message : String(error));
|
|
9761
|
+
}
|
|
9762
|
+
/**
|
|
9763
|
+
* Restores and wires the mailboxes; returns the inbox items not yet heard.
|
|
9764
|
+
* `strict` fails on a storage read rather than wiring a partial channel.
|
|
9765
|
+
*/
|
|
9766
|
+
async wireRemote(remote, strict = false) {
|
|
9767
|
+
await this.loadDeadLetters(remote, strict);
|
|
9768
|
+
await this.restoreReceivedMarkers(remote, strict);
|
|
9769
|
+
const unheard = [...remote.channel.inbox.items];
|
|
9770
|
+
this.wireChannelCallbacks(remote);
|
|
9771
|
+
return unheard;
|
|
9772
|
+
}
|
|
9773
|
+
/** After init: hands over unheard items and starts the outbox backfill. */
|
|
9774
|
+
async activateRemote(remote, unheard, backfillFrom) {
|
|
9775
|
+
const name = remote.meta.name;
|
|
9776
|
+
if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
|
|
9777
|
+
await this.peerUpdates.get(name);
|
|
9778
|
+
this.owe(name, await this.watermarkHead());
|
|
9779
|
+
const backfillController = new AbortController();
|
|
9780
|
+
this.backfillAbortControllers.set(name, backfillController);
|
|
9781
|
+
this.updateOutbox(remote, backfillFrom, "backfill", backfillController.signal).catch((error) => {
|
|
9782
|
+
if (backfillController.signal.aborted) return;
|
|
9783
|
+
this.logger.error("Backfill failed for remote @RemoteName: @Error", name, error instanceof Error ? error : new Error(String(error)));
|
|
9784
|
+
}).finally(() => {
|
|
9785
|
+
this.backfillAbortControllers.delete(name);
|
|
9786
|
+
this.drainPrunes();
|
|
9787
|
+
});
|
|
9788
|
+
}
|
|
9789
|
+
/** Keeps the row until the retry succeeds, so a crash cannot lose the op. */
|
|
9790
|
+
async requeueDeadLetter(remoteName, id) {
|
|
9791
|
+
const remote = this.getByName(remoteName);
|
|
9792
|
+
if (this.requeuesInFlight.has(id) || this.requeuedDeadLetterIds.has(id) || remote.channel.inbox.get(id)) return;
|
|
9793
|
+
this.requeuesInFlight.add(id);
|
|
9794
|
+
try {
|
|
9795
|
+
await this.requeue(remote, remoteName, id);
|
|
9796
|
+
} finally {
|
|
9797
|
+
this.requeuesInFlight.delete(id);
|
|
9798
|
+
}
|
|
9799
|
+
}
|
|
9800
|
+
async requeue(remote, remoteName, id) {
|
|
9801
|
+
const source = await this.findDeadLetter(remote, id);
|
|
9802
|
+
if (!source) return;
|
|
9803
|
+
this.assertServing(remote, id);
|
|
9804
|
+
const errorSource = source instanceof SyncOperation ? source.error?.source : source.errorSource;
|
|
9805
|
+
if (errorSource !== "inbox") throw new SyncRepairRefusedError(`Cannot requeue dead letter '${id}' of '${remoteName}': it failed on the ${errorSource ?? "unknown"} side, not while applying here.`);
|
|
9806
|
+
const item = remote.channel.deadLetter.get(id);
|
|
9807
|
+
if (item) remote.channel.deadLetter.remove(item);
|
|
9808
|
+
this.requeuedDeadLetterIds.set(source.id, {
|
|
9809
|
+
remoteName: remote.meta.name,
|
|
9810
|
+
documentId: source.documentId
|
|
9811
|
+
});
|
|
9812
|
+
const lift = await this.mayLiftQuarantine(source.documentId);
|
|
9813
|
+
try {
|
|
9814
|
+
this.assertServing(remote, id);
|
|
9815
|
+
} catch (error) {
|
|
9816
|
+
this.requeuedDeadLetterIds.delete(source.id);
|
|
9817
|
+
throw error;
|
|
9818
|
+
}
|
|
9819
|
+
const requeued = new SyncOperation(source.id, source.jobId, source.jobDependencies, source.remoteName, source.documentId, source.scopes, source.branch, source.operations);
|
|
9820
|
+
remote.channel.inbox.add(requeued);
|
|
9821
|
+
if (lift) this.liftQuarantine(source.documentId);
|
|
9822
|
+
}
|
|
9823
|
+
/** A requeue into a channel torn down meanwhile would never load. */
|
|
9824
|
+
assertServing(remote, id) {
|
|
9825
|
+
const name = remote.meta.name;
|
|
9826
|
+
if (this.remotes.get(name) === remote && !this.removing.has(name)) return;
|
|
9827
|
+
throw new SyncRepairRefusedError(`Remote '${name}' was reset or removed while requeueing dead letter '${id}'; requeue it again`);
|
|
9828
|
+
}
|
|
9829
|
+
async liftQuarantineIfClear(documentId) {
|
|
9830
|
+
if (await this.mayLiftQuarantine(documentId)) this.liftQuarantine(documentId);
|
|
9831
|
+
}
|
|
9832
|
+
/** Whether the restart rule would lift it: no quarantining dead letter left. */
|
|
9833
|
+
async mayLiftQuarantine(documentId) {
|
|
9834
|
+
if (!this.quarantinedDocumentIds.has(documentId)) return false;
|
|
9835
|
+
if (this.holdsQuarantiningDeadLetter(documentId)) return false;
|
|
9836
|
+
let stored;
|
|
9837
|
+
try {
|
|
9838
|
+
stored = await this.storesQuarantiningDeadLetter(documentId);
|
|
9839
|
+
} catch (error) {
|
|
9840
|
+
this.logger.warn("Could not read dead letters for @documentId; its quarantine stays: @error", documentId, error instanceof Error ? error.message : String(error));
|
|
9841
|
+
return false;
|
|
9842
|
+
}
|
|
9843
|
+
return !stored && this.quarantinedDocumentIds.has(documentId) && !this.holdsQuarantiningDeadLetter(documentId);
|
|
9844
|
+
}
|
|
9845
|
+
holdsQuarantiningDeadLetter(documentId) {
|
|
9846
|
+
for (const remote of this.remotes.values()) for (const item of remote.channel.deadLetter.items) if (item.documentId === documentId && !this.requeuedDeadLetterIds.has(item.id) && quarantinesDocument(syncOperationErrorType(item.error))) return true;
|
|
9847
|
+
return false;
|
|
9848
|
+
}
|
|
9849
|
+
/** The restart rule, less the rows kept only for a pending requeue. */
|
|
9850
|
+
async storesQuarantiningDeadLetter(documentId) {
|
|
9851
|
+
return (await this.deadLetterStorage.listQuarantinedDocumentIds(void 0, [...this.requeuedDeadLetterIds.keys()])).includes(documentId);
|
|
9852
|
+
}
|
|
9853
|
+
/** Hands the document's parked inbox items to the apply path again. */
|
|
9854
|
+
liftQuarantine(documentId) {
|
|
9855
|
+
if (!this.quarantinedDocumentIds.delete(documentId)) return;
|
|
9856
|
+
for (const remote of this.remotes.values()) {
|
|
9857
|
+
if (this.removing.has(remote.meta.name)) continue;
|
|
9858
|
+
const parked = remote.channel.inbox.items.filter((item) => item.documentId === documentId && this.parkedByQuarantine.has(item)).sort((a, b) => firstOrdinalOf(a) - firstOrdinalOf(b));
|
|
9859
|
+
if (parked.length === 0) continue;
|
|
9860
|
+
for (const item of parked) this.parkedByQuarantine.delete(item);
|
|
9861
|
+
const rounds = [];
|
|
9862
|
+
const seen = /* @__PURE__ */ new Map();
|
|
9863
|
+
for (const item of parked) {
|
|
9864
|
+
const round = item.jobId ? seen.get(item.jobId) ?? 0 : 0;
|
|
9865
|
+
if (item.jobId) seen.set(item.jobId, round + 1);
|
|
9866
|
+
(rounds[round] ??= []).push(item);
|
|
9867
|
+
}
|
|
9868
|
+
for (const round of rounds) this.handleInboxAdded(remote, round);
|
|
9869
|
+
}
|
|
9870
|
+
}
|
|
9871
|
+
/** A failed remove leaves a row for a later clear; the op itself is durable. */
|
|
9872
|
+
async dropRequeuedDeadLetter(id, remoteName) {
|
|
9873
|
+
if (this.requeuedDeadLetterIds.get(id)?.remoteName !== remoteName) return;
|
|
9874
|
+
this.requeuedDeadLetterIds.delete(id);
|
|
9875
|
+
const current = this.remotes.get(remoteName);
|
|
9876
|
+
const reloaded = current?.channel.deadLetter.get(id);
|
|
9877
|
+
if (current && reloaded) current.channel.deadLetter.remove(reloaded);
|
|
9878
|
+
try {
|
|
9879
|
+
await this.deadLetterStorage.remove(id);
|
|
9880
|
+
} catch (error) {
|
|
9881
|
+
this.logger.error("Failed to remove a requeued dead letter after its retry succeeded (@id, @error)", id, error instanceof Error ? error.message : String(error));
|
|
9882
|
+
}
|
|
9883
|
+
}
|
|
9884
|
+
async clearDeadLetter(remoteName, id) {
|
|
9885
|
+
const remote = this.getByName(remoteName);
|
|
9886
|
+
const source = await this.findDeadLetter(remote, id);
|
|
9887
|
+
if (!source) return;
|
|
9888
|
+
await this.deadLetterStorage.remove(id);
|
|
9889
|
+
for (const holder of /* @__PURE__ */ new Set([remote, this.remotes.get(remoteName)])) {
|
|
9890
|
+
if (!holder) continue;
|
|
9891
|
+
const item = holder.channel.deadLetter.get(id);
|
|
9892
|
+
if (item) holder.channel.deadLetter.remove(item);
|
|
9893
|
+
const parked = holder.channel.inbox.get(id);
|
|
9894
|
+
if (parked && this.parkedByQuarantine.has(parked)) {
|
|
9895
|
+
this.parkedByQuarantine.delete(parked);
|
|
9896
|
+
parked.executed();
|
|
9897
|
+
holder.channel.inbox.remove(parked);
|
|
9898
|
+
}
|
|
9899
|
+
}
|
|
9900
|
+
this.requeuedDeadLetterIds.delete(id);
|
|
9901
|
+
await this.liftQuarantineIfClear(source.documentId);
|
|
9902
|
+
}
|
|
9903
|
+
/** Live item first; else scans storage for one the capped mailbox evicted. */
|
|
9904
|
+
async findDeadLetter(remote, id) {
|
|
9905
|
+
const item = remote.channel.deadLetter.get(id);
|
|
9906
|
+
if (item) return item;
|
|
9907
|
+
let cursor = "0";
|
|
9908
|
+
for (;;) {
|
|
9909
|
+
const page = await this.deadLetterStorage.list(remote.meta.name, {
|
|
9910
|
+
cursor,
|
|
9911
|
+
limit: this.config.maxDeadLettersPerRemote
|
|
9912
|
+
});
|
|
9913
|
+
const match = page.results.find((record) => record.id === id);
|
|
9914
|
+
if (match) return match;
|
|
9915
|
+
if (!page.nextCursor) return;
|
|
9916
|
+
cursor = page.nextCursor;
|
|
9917
|
+
}
|
|
9918
|
+
}
|
|
9424
9919
|
recordPlanKeyMapping(planKey, jobId) {
|
|
9425
9920
|
if (!this.planKeyToJobUuid.has(planKey) && this.planKeyToJobUuid.size >= PLAN_KEY_TO_JOB_UUID_CAP) {
|
|
9426
9921
|
const oldest = this.planKeyToJobUuid.keys().next().value;
|
|
@@ -9471,6 +9966,7 @@ var SyncManager = class {
|
|
|
9471
9966
|
if (purged.length > 0) remote.channel.deadLetter.remove(...purged);
|
|
9472
9967
|
const syncOps = remaining.filter((syncOp) => !purged.includes(syncOp));
|
|
9473
9968
|
for (const syncOp of syncOps) {
|
|
9969
|
+
this.requeuedDeadLetterIds.delete(syncOp.id);
|
|
9474
9970
|
this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
|
|
9475
9971
|
const errorType = syncOperationErrorType(syncOp.error);
|
|
9476
9972
|
if (quarantinesDocument(errorType)) this.quarantinedDocumentIds.add(syncOp.documentId);
|
|
@@ -9511,7 +10007,7 @@ var SyncManager = class {
|
|
|
9511
10007
|
}
|
|
9512
10008
|
});
|
|
9513
10009
|
}
|
|
9514
|
-
async loadDeadLetters(remote) {
|
|
10010
|
+
async loadDeadLetters(remote, strict = false) {
|
|
9515
10011
|
let records;
|
|
9516
10012
|
try {
|
|
9517
10013
|
records = (await this.deadLetterStorage.list(remote.meta.name, {
|
|
@@ -9519,6 +10015,7 @@ var SyncManager = class {
|
|
|
9519
10015
|
limit: this.config.maxDeadLettersPerRemote
|
|
9520
10016
|
})).results;
|
|
9521
10017
|
} catch (error) {
|
|
10018
|
+
if (strict) throw error;
|
|
9522
10019
|
this.logger.error("Failed to load dead letters for remote (@name, @error)", remote.meta.name, error instanceof Error ? error.message : String(error));
|
|
9523
10020
|
return;
|
|
9524
10021
|
}
|
|
@@ -9624,10 +10121,17 @@ var SyncManager = class {
|
|
|
9624
10121
|
this.storeReceivedMarker(remote, syncOp);
|
|
9625
10122
|
eligible.push(syncOp);
|
|
9626
10123
|
}
|
|
9627
|
-
} else if (this.purgedDocumentIds.has(syncOp.documentId))
|
|
9628
|
-
|
|
10124
|
+
} else if (this.purgedDocumentIds.has(syncOp.documentId)) {
|
|
10125
|
+
dropped.push(syncOp);
|
|
10126
|
+
this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
|
|
10127
|
+
} else if (!this.quarantinedDocumentIds.has(syncOp.documentId)) eligible.push(syncOp);
|
|
10128
|
+
else if (!this.requeuedDeadLetterIds.has(syncOp.id) && this.parkedCopyOf(remote, syncOp)) dropped.push(syncOp);
|
|
10129
|
+
else {
|
|
10130
|
+
this.parkedByQuarantine.add(syncOp);
|
|
10131
|
+
remote.channel.inbox.hold?.(syncOp);
|
|
10132
|
+
}
|
|
9629
10133
|
for (const syncOp of dropped) {
|
|
9630
|
-
this.logger.debug("Dropping received operations of a purged or already-
|
|
10134
|
+
this.logger.debug("Dropping received operations of a purged, already-loading or already-parked document (@remote, @documentId)", remote.meta.name, syncOp.documentId);
|
|
9631
10135
|
syncOp.executed();
|
|
9632
10136
|
}
|
|
9633
10137
|
if (dropped.length > 0) remote.channel.inbox.remove(...dropped);
|
|
@@ -9645,6 +10149,11 @@ var SyncManager = class {
|
|
|
9645
10149
|
this.processInboxChunks(chunks);
|
|
9646
10150
|
}
|
|
9647
10151
|
}
|
|
10152
|
+
parkedCopyOf(remote, syncOp) {
|
|
10153
|
+
const ops = syncOp.operations.map((op) => op.operation);
|
|
10154
|
+
if (ops.length === 0) return false;
|
|
10155
|
+
return remote.channel.inbox.items.some((item) => item !== syncOp && this.parkedByQuarantine.has(item) && item.documentId === syncOp.documentId && item.branch === syncOp.branch && item.jobId === syncOp.jobId && ops.every((op) => item.operations.some(({ operation }) => operation.id === op.id && operation.index === op.index)));
|
|
10156
|
+
}
|
|
9648
10157
|
/** Forgets a marker once the item loading it leaves the inbox. */
|
|
9649
10158
|
handleInboxRemoved(remote, syncOps) {
|
|
9650
10159
|
const received = this.receivedMarkers.get(remote.meta.name);
|
|
@@ -9692,12 +10201,13 @@ var SyncManager = class {
|
|
|
9692
10201
|
});
|
|
9693
10202
|
}
|
|
9694
10203
|
/** Queued before init resets latestOrdinal, so a puller is re-served above it. */
|
|
9695
|
-
async restoreReceivedMarkers(remote) {
|
|
10204
|
+
async restoreReceivedMarkers(remote, strict = false) {
|
|
9696
10205
|
const name = remote.meta.name;
|
|
9697
10206
|
let records;
|
|
9698
10207
|
try {
|
|
9699
10208
|
records = await this.markerStorage.list(name);
|
|
9700
10209
|
} catch (error) {
|
|
10210
|
+
if (strict) throw error;
|
|
9701
10211
|
this.logger.error("Failed to load received markers for remote (@name, @error)", name, error instanceof Error ? error.message : String(error));
|
|
9702
10212
|
return;
|
|
9703
10213
|
}
|
|
@@ -9769,12 +10279,15 @@ var SyncManager = class {
|
|
|
9769
10279
|
return;
|
|
9770
10280
|
}
|
|
9771
10281
|
if (this.isShutdown) return;
|
|
10282
|
+
let resolved = false;
|
|
9772
10283
|
if (completedJobInfo.status !== "FAILED") {
|
|
9773
10284
|
syncOp.executed();
|
|
9774
10285
|
if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
|
|
10286
|
+
resolved = true;
|
|
9775
10287
|
} else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
|
|
9776
10288
|
this.tombstone(syncOp.documentId);
|
|
9777
10289
|
syncOp.executed();
|
|
10290
|
+
resolved = true;
|
|
9778
10291
|
} else {
|
|
9779
10292
|
const errorMessage = completedJobInfo.error?.message || "Unknown error";
|
|
9780
10293
|
this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
|
|
@@ -9787,6 +10300,7 @@ var SyncManager = class {
|
|
|
9787
10300
|
}
|
|
9788
10301
|
this.markerRetries.delete(syncOp.id);
|
|
9789
10302
|
remote.channel.inbox.remove(syncOp);
|
|
10303
|
+
if (resolved) await this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
|
|
9790
10304
|
}
|
|
9791
10305
|
/** Reloads a marker with backoff; it stays in the inbox, not dead-lettered. */
|
|
9792
10306
|
retryMarker(remote, syncOp, reason) {
|
|
@@ -9816,6 +10330,7 @@ var SyncManager = class {
|
|
|
9816
10330
|
this.markerRetries.delete(syncOp.id);
|
|
9817
10331
|
syncOp.executed();
|
|
9818
10332
|
remote.channel.inbox.remove(syncOp);
|
|
10333
|
+
await this.dropRequeuedDeadLetter(syncOp.id, name);
|
|
9819
10334
|
return;
|
|
9820
10335
|
}
|
|
9821
10336
|
await this.applyInboxJob(remote, syncOp);
|
|
@@ -9906,12 +10421,15 @@ var SyncManager = class {
|
|
|
9906
10421
|
continue;
|
|
9907
10422
|
}
|
|
9908
10423
|
if (this.isShutdown) return;
|
|
10424
|
+
let resolved = false;
|
|
9909
10425
|
if (completedJobInfo.status !== "FAILED") {
|
|
9910
10426
|
syncOp.executed();
|
|
9911
10427
|
if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
|
|
10428
|
+
resolved = true;
|
|
9912
10429
|
} else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
|
|
9913
10430
|
this.tombstone(syncOp.documentId);
|
|
9914
10431
|
syncOp.executed();
|
|
10432
|
+
resolved = true;
|
|
9915
10433
|
} else if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
|
|
9916
10434
|
this.retryMarker(remote, syncOp, completedJobInfo.error?.message || "Unknown error");
|
|
9917
10435
|
continue;
|
|
@@ -9920,6 +10438,7 @@ var SyncManager = class {
|
|
|
9920
10438
|
remote.channel.deadLetter.add(syncOp);
|
|
9921
10439
|
}
|
|
9922
10440
|
remote.channel.inbox.remove(syncOp);
|
|
10441
|
+
if (resolved) await this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
|
|
9923
10442
|
}
|
|
9924
10443
|
}
|
|
9925
10444
|
/**
|
|
@@ -10215,7 +10734,8 @@ var SyncBuilder = class {
|
|
|
10215
10734
|
deadLetterStorage,
|
|
10216
10735
|
holdStorage,
|
|
10217
10736
|
channelFactory: this.channelFactory,
|
|
10218
|
-
syncManager
|
|
10737
|
+
syncManager,
|
|
10738
|
+
syncAdmin: syncManager
|
|
10219
10739
|
};
|
|
10220
10740
|
}
|
|
10221
10741
|
};
|
|
@@ -10224,8 +10744,8 @@ var SyncBuilder = class {
|
|
|
10224
10744
|
async function createDefaultDatabase() {
|
|
10225
10745
|
const { Kysely } = await import("kysely");
|
|
10226
10746
|
const { PGlite } = await import("@electric-sql/pglite");
|
|
10227
|
-
const {
|
|
10228
|
-
return new Kysely({ dialect: new
|
|
10747
|
+
const { HardenedPGliteDialect } = await Promise.resolve().then(() => pglite_dialect_exports);
|
|
10748
|
+
return new Kysely({ dialect: new HardenedPGliteDialect(new PGlite()) });
|
|
10229
10749
|
}
|
|
10230
10750
|
//#endregion
|
|
10231
10751
|
//#region src/shared/factories.ts
|
|
@@ -11016,6 +11536,7 @@ var ReactorBuilder = class {
|
|
|
11016
11536
|
this.unsupportedStoredDocuments = mode;
|
|
11017
11537
|
return this;
|
|
11018
11538
|
}
|
|
11539
|
+
/** Mutually exclusive with {@link withChannelScheme}; setting both fails the build. */
|
|
11019
11540
|
withSync(syncBuilder) {
|
|
11020
11541
|
this.syncBuilder = syncBuilder;
|
|
11021
11542
|
return this;
|
|
@@ -11079,6 +11600,7 @@ var ReactorBuilder = class {
|
|
|
11079
11600
|
this.queueInstance = queue;
|
|
11080
11601
|
return this;
|
|
11081
11602
|
}
|
|
11603
|
+
/** Mutually exclusive with {@link withSync}; setting both fails the build. */
|
|
11082
11604
|
withChannelScheme(scheme) {
|
|
11083
11605
|
this.channelScheme = scheme;
|
|
11084
11606
|
return this;
|
|
@@ -11164,6 +11686,7 @@ var ReactorBuilder = class {
|
|
|
11164
11686
|
...this.executorConfig,
|
|
11165
11687
|
protocolSupport: localSupports(this.getPeerCapabilities(), featureFlags).protocols
|
|
11166
11688
|
};
|
|
11689
|
+
if (this.channelScheme !== void 0 && this.syncBuilder !== void 0) throw new Error("withChannelScheme and withSync are mutually exclusive; use one (the scheme builds its own SyncBuilder)");
|
|
11167
11690
|
if (this.readModelCoordinator !== void 0 && this.readModelCoordinatorFactory !== void 0) throw new Error("withReadModelCoordinator and withReadModelCoordinatorFactory are mutually exclusive; register one coordinator source");
|
|
11168
11691
|
if (this.projectionShardConfig !== void 0 && this.readModelCoordinatorFactory !== void 0) throw new Error("withProjectionShards and withReadModelCoordinatorFactory are mutually exclusive; the factory owns the projection worker through its createProjectionShardManager dependency");
|
|
11169
11692
|
if (this.projectionShardConfig !== void 0 && this.readModelFactories.length > 0) throw new Error("withProjectionShards does not support read models registered through withReadModelFactory; projection workers cannot receive host-only factory dependencies");
|
|
@@ -11453,7 +11976,7 @@ var ReactorBuilder = class {
|
|
|
11453
11976
|
poolInstrumentations.push(forwarder);
|
|
11454
11977
|
this.instrumentedPools.push(forwarder);
|
|
11455
11978
|
}
|
|
11456
|
-
const { ProjectionShardManager } = await import("./projection-shard-manager-
|
|
11979
|
+
const { ProjectionShardManager } = await import("./projection-shard-manager-CQ5FwQmv.js");
|
|
11457
11980
|
const manager = new ProjectionShardManager({
|
|
11458
11981
|
shardCount: config.shardCount,
|
|
11459
11982
|
db,
|
|
@@ -11498,7 +12021,7 @@ var ReactorBuilder = class {
|
|
|
11498
12021
|
*/
|
|
11499
12022
|
async createDefaultWorkerFactory(numWorkers, db) {
|
|
11500
12023
|
const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
|
|
11501
|
-
import("./worker-handle-
|
|
12024
|
+
import("./worker-handle-CN2-oRv-.js"),
|
|
11502
12025
|
import("./transport-CZDHzfvy.js"),
|
|
11503
12026
|
Promise.resolve().then(() => worker_exports)
|
|
11504
12027
|
]);
|
|
@@ -12095,6 +12618,391 @@ function firstOutOfOrderPair(operations, options) {
|
|
|
12095
12618
|
}
|
|
12096
12619
|
}
|
|
12097
12620
|
//#endregion
|
|
12621
|
+
//#region src/storage/kysely/pglite-dialect.ts
|
|
12622
|
+
var pglite_dialect_exports = /* @__PURE__ */ __exportAll({
|
|
12623
|
+
DEFAULT_LONG_STATEMENT_TIMEOUT_MS: () => DEFAULT_LONG_STATEMENT_TIMEOUT_MS,
|
|
12624
|
+
DEFAULT_RECOVERY_TIMEOUT_MS: () => DEFAULT_RECOVERY_TIMEOUT_MS,
|
|
12625
|
+
DEFAULT_STATEMENT_TIMEOUT_MS: () => DEFAULT_STATEMENT_TIMEOUT_MS,
|
|
12626
|
+
HardenedPGliteDialect: () => HardenedPGliteDialect,
|
|
12627
|
+
PGliteAbortedTransactionError: () => PGliteAbortedTransactionError,
|
|
12628
|
+
PGliteAcquireTimeoutError: () => PGliteAcquireTimeoutError,
|
|
12629
|
+
PGliteSessionError: () => PGliteSessionError,
|
|
12630
|
+
PGliteSessionPoisonedError: () => PGliteSessionPoisonedError,
|
|
12631
|
+
PGliteStatementTimeoutError: () => PGliteStatementTimeoutError,
|
|
12632
|
+
isLongRunningStatement: () => isLongRunningStatement,
|
|
12633
|
+
queryThroughDialect: () => queryThroughDialect
|
|
12634
|
+
});
|
|
12635
|
+
const DEFAULT_STATEMENT_TIMEOUT_MS = 12e4;
|
|
12636
|
+
const DEFAULT_LONG_STATEMENT_TIMEOUT_MS = 9e5;
|
|
12637
|
+
const DEFAULT_RECOVERY_TIMEOUT_MS = 15e3;
|
|
12638
|
+
const LONG_STATEMENT_PREFIXES = [
|
|
12639
|
+
"vacuum",
|
|
12640
|
+
"analyze",
|
|
12641
|
+
"reindex",
|
|
12642
|
+
"cluster",
|
|
12643
|
+
"checkpoint",
|
|
12644
|
+
"copy",
|
|
12645
|
+
"alter ",
|
|
12646
|
+
"create ",
|
|
12647
|
+
"drop ",
|
|
12648
|
+
"truncate",
|
|
12649
|
+
"refresh "
|
|
12650
|
+
];
|
|
12651
|
+
/** Maintenance and DDL, matched on the leading keyword of the compiled SQL. */
|
|
12652
|
+
function isLongRunningStatement(statement) {
|
|
12653
|
+
const normalized = statement.trimStart().toLowerCase();
|
|
12654
|
+
return LONG_STATEMENT_PREFIXES.some((prefix) => normalized.startsWith(prefix));
|
|
12655
|
+
}
|
|
12656
|
+
function summarize(statement) {
|
|
12657
|
+
const collapsed = statement.replace(/\s+/g, " ").trim();
|
|
12658
|
+
return collapsed.length > 160 ? `${collapsed.slice(0, 160)}...` : collapsed;
|
|
12659
|
+
}
|
|
12660
|
+
/** COMMIT in an aborted transaction silently rolls back; the guard raises first. */
|
|
12661
|
+
const COMMIT_WITH_GUARD = "select 1 as __commit_guard; commit";
|
|
12662
|
+
const PROBE = "select 1 as __session_probe";
|
|
12663
|
+
const ABORTED_TRANSACTION_CODE = "25P02";
|
|
12664
|
+
var PGliteSessionError = class extends Error {
|
|
12665
|
+
cause;
|
|
12666
|
+
constructor(message, cause) {
|
|
12667
|
+
super(message);
|
|
12668
|
+
this.cause = cause;
|
|
12669
|
+
this.name = "PGliteSessionError";
|
|
12670
|
+
}
|
|
12671
|
+
};
|
|
12672
|
+
var PGliteAcquireTimeoutError = class extends PGliteSessionError {
|
|
12673
|
+
constructor(timeoutMs) {
|
|
12674
|
+
super(`Timed out after ${timeoutMs}ms waiting for the PGlite connection. The single session is held by a statement that never completed: a transaction awaiting a nested query on the base handle, or an abandoned stream iterator.`);
|
|
12675
|
+
this.name = "PGliteAcquireTimeoutError";
|
|
12676
|
+
}
|
|
12677
|
+
};
|
|
12678
|
+
/** The call cannot be aborted, so it is abandoned and the session treated as poisoned. */
|
|
12679
|
+
var PGliteStatementTimeoutError = class extends PGliteSessionError {
|
|
12680
|
+
constructor(timeoutMs, statement) {
|
|
12681
|
+
super(`A PGlite statement neither resolved nor rejected within ${timeoutMs}ms, so its wasm call is presumed dead and the session poisoned: ${summarize(statement)}`);
|
|
12682
|
+
this.name = "PGliteStatementTimeoutError";
|
|
12683
|
+
}
|
|
12684
|
+
};
|
|
12685
|
+
var PGliteAbortedTransactionError = class extends PGliteSessionError {
|
|
12686
|
+
constructor(cause) {
|
|
12687
|
+
super(`Refusing to report a commit for an aborted transaction: COMMIT would have been answered with a ROLLBACK tag and nothing written. Underlying error: ${errorOf(cause).message}`, cause);
|
|
12688
|
+
this.name = "PGliteAbortedTransactionError";
|
|
12689
|
+
}
|
|
12690
|
+
};
|
|
12691
|
+
/** No statement can clear the session (e.g. a stuck active portal). */
|
|
12692
|
+
var PGliteSessionPoisonedError = class extends PGliteSessionError {
|
|
12693
|
+
constructor(cause) {
|
|
12694
|
+
super(`The PGlite session is unrecoverable and cannot be reused; the component owning it must be restarted. Original error: ${errorOf(cause).message}`, cause);
|
|
12695
|
+
this.name = "PGliteSessionPoisonedError";
|
|
12696
|
+
}
|
|
12697
|
+
};
|
|
12698
|
+
function errorOf(error) {
|
|
12699
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
12700
|
+
}
|
|
12701
|
+
const TIMED_OUT = Symbol("pglite-deadline-expired");
|
|
12702
|
+
/** Races a call that cannot be cancelled; on {@link TIMED_OUT} the call may still settle later. */
|
|
12703
|
+
async function withDeadline(pending, timeoutMs) {
|
|
12704
|
+
let handle;
|
|
12705
|
+
const expiry = new Promise((resolve) => {
|
|
12706
|
+
handle = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
|
|
12707
|
+
});
|
|
12708
|
+
try {
|
|
12709
|
+
return await Promise.race([pending, expiry]);
|
|
12710
|
+
} finally {
|
|
12711
|
+
clearTimeout(handle);
|
|
12712
|
+
}
|
|
12713
|
+
}
|
|
12714
|
+
function isAbortedTransactionError(error) {
|
|
12715
|
+
if (error?.code === ABORTED_TRANSACTION_CODE) return true;
|
|
12716
|
+
return errorOf(error).message.includes("current transaction is aborted");
|
|
12717
|
+
}
|
|
12718
|
+
/** `kysely-pglite-dialect` hardened for one shared session that a fault must not brick. */
|
|
12719
|
+
var HardenedPGliteDialect = class {
|
|
12720
|
+
client;
|
|
12721
|
+
inner;
|
|
12722
|
+
options;
|
|
12723
|
+
constructor(client, options = {}) {
|
|
12724
|
+
this.client = client;
|
|
12725
|
+
this.inner = new PGliteDialect(client);
|
|
12726
|
+
this.options = {
|
|
12727
|
+
acquireTimeoutMs: options.acquireTimeoutMs ?? 0,
|
|
12728
|
+
statementTimeoutMs: options.statementTimeoutMs ?? 12e4,
|
|
12729
|
+
longStatementTimeoutMs: options.longStatementTimeoutMs ?? 9e5,
|
|
12730
|
+
recoveryTimeoutMs: options.recoveryTimeoutMs ?? 15e3,
|
|
12731
|
+
onDiagnostic: options.onDiagnostic ?? ((message, error) => {
|
|
12732
|
+
console.error(`[pglite-dialect] ${message}`, error);
|
|
12733
|
+
}),
|
|
12734
|
+
onPoisoned: options.onPoisoned ?? (() => void 0)
|
|
12735
|
+
};
|
|
12736
|
+
}
|
|
12737
|
+
createAdapter() {
|
|
12738
|
+
return this.inner.createAdapter();
|
|
12739
|
+
}
|
|
12740
|
+
createDriver() {
|
|
12741
|
+
return new HardenedPGliteDriver(this.inner.createDriver(), this.client, this.options);
|
|
12742
|
+
}
|
|
12743
|
+
createQueryCompiler() {
|
|
12744
|
+
return this.inner.createQueryCompiler();
|
|
12745
|
+
}
|
|
12746
|
+
createIntrospector(db) {
|
|
12747
|
+
return this.inner.createIntrospector(db);
|
|
12748
|
+
}
|
|
12749
|
+
};
|
|
12750
|
+
var HardenedPGliteConnection = class {
|
|
12751
|
+
inner;
|
|
12752
|
+
guard;
|
|
12753
|
+
failure = void 0;
|
|
12754
|
+
rollbackFailure = void 0;
|
|
12755
|
+
transactionOpen = false;
|
|
12756
|
+
constructor(inner, guard) {
|
|
12757
|
+
this.inner = inner;
|
|
12758
|
+
this.guard = guard;
|
|
12759
|
+
}
|
|
12760
|
+
/** Never retried: a replay could commit alone a write meant for another consumer's transaction. */
|
|
12761
|
+
async executeQuery(compiledQuery) {
|
|
12762
|
+
const generation = this.guard.generation;
|
|
12763
|
+
try {
|
|
12764
|
+
return await this.guard.runStatement(compiledQuery.sql, () => this.inner.executeQuery(compiledQuery));
|
|
12765
|
+
} catch (error) {
|
|
12766
|
+
this.guard.recordFailure(this, generation, error);
|
|
12767
|
+
throw error;
|
|
12768
|
+
}
|
|
12769
|
+
}
|
|
12770
|
+
streamQuery(compiledQuery, chunkSize) {
|
|
12771
|
+
const inner = this.inner.streamQuery(compiledQuery, chunkSize);
|
|
12772
|
+
return this.deadlinedStream(compiledQuery, inner);
|
|
12773
|
+
}
|
|
12774
|
+
/** Upstream issues the whole query on the first pull, so each pull is bounded like a statement. */
|
|
12775
|
+
async *deadlinedStream(compiledQuery, inner) {
|
|
12776
|
+
for (;;) {
|
|
12777
|
+
const generation = this.guard.generation;
|
|
12778
|
+
let next;
|
|
12779
|
+
try {
|
|
12780
|
+
next = await this.guard.runStatement(compiledQuery.sql, () => inner.next());
|
|
12781
|
+
} catch (error) {
|
|
12782
|
+
this.guard.recordFailure(this, generation, error);
|
|
12783
|
+
throw error;
|
|
12784
|
+
}
|
|
12785
|
+
if (next.done === true) return;
|
|
12786
|
+
yield next.value;
|
|
12787
|
+
}
|
|
12788
|
+
}
|
|
12789
|
+
get suspect() {
|
|
12790
|
+
return this.failure !== void 0 || this.rollbackFailure !== void 0 || this.transactionOpen;
|
|
12791
|
+
}
|
|
12792
|
+
};
|
|
12793
|
+
var HardenedPGliteDriver = class {
|
|
12794
|
+
inner;
|
|
12795
|
+
client;
|
|
12796
|
+
options;
|
|
12797
|
+
/** Set when release could not reset the session; each acquire retries the reset. */
|
|
12798
|
+
sessionFault = void 0;
|
|
12799
|
+
/** Lets an abandoned call's late settlement be told apart from current failures. */
|
|
12800
|
+
statementGeneration = 0;
|
|
12801
|
+
/** A call abandoned at its deadline; PGlite would queue every later call behind it, so none is issued. */
|
|
12802
|
+
deadCall = void 0;
|
|
12803
|
+
poisonReported = false;
|
|
12804
|
+
constructor(inner, client, options) {
|
|
12805
|
+
this.inner = inner;
|
|
12806
|
+
this.client = client;
|
|
12807
|
+
this.options = options;
|
|
12808
|
+
}
|
|
12809
|
+
get generation() {
|
|
12810
|
+
return this.statementGeneration;
|
|
12811
|
+
}
|
|
12812
|
+
/** Bounds one statement; an expiry is escalated as a poisoned session. */
|
|
12813
|
+
async runStatement(statement, execute) {
|
|
12814
|
+
if (this.deadCall !== void 0) throw new PGliteSessionPoisonedError(this.deadCall);
|
|
12815
|
+
const timeoutMs = this.timeoutFor(statement);
|
|
12816
|
+
if (timeoutMs <= 0) return execute();
|
|
12817
|
+
const generation = this.statementGeneration;
|
|
12818
|
+
const outcome = await withDeadline(execute(), timeoutMs);
|
|
12819
|
+
if (outcome !== TIMED_OUT) return outcome;
|
|
12820
|
+
throw this.escalateHungStatement(new PGliteStatementTimeoutError(timeoutMs, statement), generation);
|
|
12821
|
+
}
|
|
12822
|
+
recordFailure(connection, generation, error) {
|
|
12823
|
+
if (generation === this.statementGeneration) connection.failure = errorOf(error);
|
|
12824
|
+
}
|
|
12825
|
+
async init() {
|
|
12826
|
+
await this.inner.init();
|
|
12827
|
+
}
|
|
12828
|
+
async acquireConnection() {
|
|
12829
|
+
const innerConnection = await this.acquireWithTimeout();
|
|
12830
|
+
if (this.deadCall !== void 0) {
|
|
12831
|
+
await this.inner.releaseConnection(innerConnection);
|
|
12832
|
+
throw new PGliteSessionPoisonedError(this.deadCall);
|
|
12833
|
+
}
|
|
12834
|
+
if (this.sessionFault !== void 0) {
|
|
12835
|
+
const fault = this.sessionFault;
|
|
12836
|
+
let recovered = false;
|
|
12837
|
+
try {
|
|
12838
|
+
recovered = await this.tryRecoverSession(false);
|
|
12839
|
+
} finally {
|
|
12840
|
+
if (!recovered) await this.inner.releaseConnection(innerConnection);
|
|
12841
|
+
}
|
|
12842
|
+
if (!recovered) {
|
|
12843
|
+
this.reportPoisoned(fault);
|
|
12844
|
+
throw new PGliteSessionPoisonedError(fault);
|
|
12845
|
+
}
|
|
12846
|
+
this.sessionFault = void 0;
|
|
12847
|
+
}
|
|
12848
|
+
return new HardenedPGliteConnection(innerConnection, this);
|
|
12849
|
+
}
|
|
12850
|
+
/** Kysely never rolls back after a failed BEGIN, so an aborted session is reset here. */
|
|
12851
|
+
async beginTransaction(connection, settings) {
|
|
12852
|
+
const wrapper = asWrapper(connection);
|
|
12853
|
+
const generation = this.statementGeneration;
|
|
12854
|
+
try {
|
|
12855
|
+
await this.runStatement("begin", () => this.inner.beginTransaction(wrapper.inner, settings));
|
|
12856
|
+
} catch (error) {
|
|
12857
|
+
this.recordFailure(wrapper, generation, error);
|
|
12858
|
+
if (!isAbortedTransactionError(error)) throw error;
|
|
12859
|
+
if (!await this.tryRecoverSession(false)) {
|
|
12860
|
+
this.sessionFault = errorOf(error);
|
|
12861
|
+
throw error;
|
|
12862
|
+
}
|
|
12863
|
+
const retryGeneration = this.statementGeneration;
|
|
12864
|
+
try {
|
|
12865
|
+
await this.runStatement("begin", () => this.inner.beginTransaction(wrapper.inner, settings));
|
|
12866
|
+
} catch (retryError) {
|
|
12867
|
+
this.recordFailure(wrapper, retryGeneration, retryError);
|
|
12868
|
+
throw retryError;
|
|
12869
|
+
}
|
|
12870
|
+
wrapper.failure = void 0;
|
|
12871
|
+
}
|
|
12872
|
+
wrapper.transactionOpen = true;
|
|
12873
|
+
}
|
|
12874
|
+
async commitTransaction(connection) {
|
|
12875
|
+
const wrapper = asWrapper(connection);
|
|
12876
|
+
const generation = this.statementGeneration;
|
|
12877
|
+
try {
|
|
12878
|
+
await this.runStatement(COMMIT_WITH_GUARD, () => this.client.exec(COMMIT_WITH_GUARD));
|
|
12879
|
+
} catch (error) {
|
|
12880
|
+
this.recordFailure(wrapper, generation, error);
|
|
12881
|
+
if (isAbortedTransactionError(error)) throw new PGliteAbortedTransactionError(error);
|
|
12882
|
+
throw error;
|
|
12883
|
+
}
|
|
12884
|
+
wrapper.transactionOpen = false;
|
|
12885
|
+
wrapper.failure = void 0;
|
|
12886
|
+
}
|
|
12887
|
+
/** Swallowed so the original failure propagates; release resets the session. */
|
|
12888
|
+
async rollbackTransaction(connection) {
|
|
12889
|
+
const wrapper = asWrapper(connection);
|
|
12890
|
+
const generation = this.statementGeneration;
|
|
12891
|
+
try {
|
|
12892
|
+
await this.runStatement("rollback", () => this.inner.rollbackTransaction(wrapper.inner));
|
|
12893
|
+
wrapper.transactionOpen = false;
|
|
12894
|
+
} catch (error) {
|
|
12895
|
+
if (generation === this.statementGeneration) wrapper.rollbackFailure = errorOf(error);
|
|
12896
|
+
this.options.onDiagnostic("rollback failed; preserving the original failure and recovering the session on release", error);
|
|
12897
|
+
}
|
|
12898
|
+
}
|
|
12899
|
+
async releaseConnection(connection) {
|
|
12900
|
+
const wrapper = asWrapper(connection);
|
|
12901
|
+
try {
|
|
12902
|
+
if (wrapper.suspect && this.deadCall === void 0) {
|
|
12903
|
+
if (!await this.tryRecoverSession(wrapper.transactionOpen)) {
|
|
12904
|
+
this.sessionFault = wrapper.rollbackFailure ?? wrapper.failure ?? /* @__PURE__ */ new Error("session left in a transaction");
|
|
12905
|
+
this.options.onDiagnostic("the PGlite session could not be reset and is being marked unrecoverable", this.sessionFault);
|
|
12906
|
+
}
|
|
12907
|
+
}
|
|
12908
|
+
} finally {
|
|
12909
|
+
await this.inner.releaseConnection(wrapper.inner);
|
|
12910
|
+
}
|
|
12911
|
+
}
|
|
12912
|
+
async destroy() {
|
|
12913
|
+
await this.inner.destroy();
|
|
12914
|
+
}
|
|
12915
|
+
reportPoisoned(cause) {
|
|
12916
|
+
if (this.poisonReported) return;
|
|
12917
|
+
this.poisonReported = true;
|
|
12918
|
+
try {
|
|
12919
|
+
this.options.onPoisoned(cause);
|
|
12920
|
+
} catch (error) {
|
|
12921
|
+
this.options.onDiagnostic("onPoisoned threw", error);
|
|
12922
|
+
}
|
|
12923
|
+
}
|
|
12924
|
+
timeoutFor(statement) {
|
|
12925
|
+
return isLongRunningStatement(statement) ? this.options.longStatementTimeoutMs : this.options.statementTimeoutMs;
|
|
12926
|
+
}
|
|
12927
|
+
/** Marks the session faulted once per generation; a stale expiry reports only itself. */
|
|
12928
|
+
escalateHungStatement(cause, generation) {
|
|
12929
|
+
if (generation !== this.statementGeneration) return cause;
|
|
12930
|
+
this.statementGeneration += 1;
|
|
12931
|
+
this.deadCall = cause;
|
|
12932
|
+
this.reportPoisoned(cause);
|
|
12933
|
+
this.options.onDiagnostic("a PGlite statement never settled within its deadline; treating the session as poisoned", cause);
|
|
12934
|
+
return new PGliteSessionPoisonedError(cause);
|
|
12935
|
+
}
|
|
12936
|
+
/** A waiter that gave up still passes the lease on, so it cannot wedge the queue. */
|
|
12937
|
+
async acquireWithTimeout() {
|
|
12938
|
+
const timeoutMs = this.options.acquireTimeoutMs;
|
|
12939
|
+
if (timeoutMs <= 0) return this.inner.acquireConnection();
|
|
12940
|
+
let abandoned = false;
|
|
12941
|
+
const pending = this.inner.acquireConnection().then((connection) => {
|
|
12942
|
+
if (!abandoned) return connection;
|
|
12943
|
+
this.inner.releaseConnection(connection).catch(() => void 0);
|
|
12944
|
+
throw new PGliteAcquireTimeoutError(timeoutMs);
|
|
12945
|
+
});
|
|
12946
|
+
let handle;
|
|
12947
|
+
const expiry = new Promise((_resolve, reject) => {
|
|
12948
|
+
handle = setTimeout(() => {
|
|
12949
|
+
abandoned = true;
|
|
12950
|
+
reject(new PGliteAcquireTimeoutError(timeoutMs));
|
|
12951
|
+
}, timeoutMs);
|
|
12952
|
+
});
|
|
12953
|
+
try {
|
|
12954
|
+
return await Promise.race([pending, expiry]);
|
|
12955
|
+
} finally {
|
|
12956
|
+
clearTimeout(handle);
|
|
12957
|
+
}
|
|
12958
|
+
}
|
|
12959
|
+
/** {@link recoverSession}, with a recovery that throws counted as failed. */
|
|
12960
|
+
async tryRecoverSession(mustEndTransaction) {
|
|
12961
|
+
try {
|
|
12962
|
+
return await this.recoverSession(mustEndTransaction);
|
|
12963
|
+
} catch (error) {
|
|
12964
|
+
this.options.onDiagnostic("session recovery threw", error);
|
|
12965
|
+
return false;
|
|
12966
|
+
}
|
|
12967
|
+
}
|
|
12968
|
+
/** Rolls back only when told to or when the probe fails, sparing another consumer's live transaction. */
|
|
12969
|
+
async recoverSession(mustEndTransaction) {
|
|
12970
|
+
if (!mustEndTransaction && !this.client.isInTransaction() && await this.probeSession()) return true;
|
|
12971
|
+
if (!await this.recoveryExec("rollback")) this.options.onDiagnostic("session rollback did not complete; the session stays suspect");
|
|
12972
|
+
if (!await this.probeSession()) return false;
|
|
12973
|
+
return !this.client.isInTransaction();
|
|
12974
|
+
}
|
|
12975
|
+
async probeSession() {
|
|
12976
|
+
return this.recoveryExec(PROBE);
|
|
12977
|
+
}
|
|
12978
|
+
/** Kysely awaits release, so an unbounded recovery statement would hold the lease forever. */
|
|
12979
|
+
async recoveryExec(statement) {
|
|
12980
|
+
if (this.deadCall !== void 0) return false;
|
|
12981
|
+
const timeoutMs = this.options.recoveryTimeoutMs;
|
|
12982
|
+
const pending = this.client.exec(statement).then(() => true, (error) => {
|
|
12983
|
+
this.options.onDiagnostic("recovery statement failed", error);
|
|
12984
|
+
return false;
|
|
12985
|
+
});
|
|
12986
|
+
if (timeoutMs <= 0) return pending;
|
|
12987
|
+
const outcome = await withDeadline(pending, timeoutMs);
|
|
12988
|
+
if (outcome === TIMED_OUT) {
|
|
12989
|
+
this.deadCall ??= new PGliteStatementTimeoutError(timeoutMs, statement);
|
|
12990
|
+
this.reportPoisoned(this.deadCall);
|
|
12991
|
+
this.options.onDiagnostic(`a PGlite recovery statement did not settle within ${timeoutMs}ms; the session is unrecoverable from SQL`);
|
|
12992
|
+
return false;
|
|
12993
|
+
}
|
|
12994
|
+
return outcome;
|
|
12995
|
+
}
|
|
12996
|
+
};
|
|
12997
|
+
function asWrapper(connection) {
|
|
12998
|
+
if (!(connection instanceof HardenedPGliteConnection)) throw new PGliteSessionError("Connection was not created by HardenedPGliteDialect");
|
|
12999
|
+
return connection;
|
|
13000
|
+
}
|
|
13001
|
+
/** Raw SQL through the dialect's queue, instead of into whatever transaction holds the session. */
|
|
13002
|
+
async function queryThroughDialect(db, sql, params) {
|
|
13003
|
+
return [...(await db.executeQuery(CompiledQuery.raw(sql, params ?? []))).rows];
|
|
13004
|
+
}
|
|
13005
|
+
//#endregion
|
|
12098
13006
|
//#region src/read-models/interfaces.ts
|
|
12099
13007
|
function supportsLiveReadModelRegistration(coordinator) {
|
|
12100
13008
|
return "addReadModel" in coordinator && typeof coordinator.addReadModel === "function";
|
|
@@ -12248,6 +13156,6 @@ var DocumentIntegrityService = class {
|
|
|
12248
13156
|
}
|
|
12249
13157
|
};
|
|
12250
13158
|
//#endregion
|
|
12251
|
-
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, BatchJobFailedError, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelNotWorkerImportableError, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isPurgeMarker, isRecoverableGraphQLError, listPurged, loadDocumentModelSpec, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
|
|
13159
|
+
export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, BatchJobFailedError, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_LONG_STATEMENT_TIMEOUT_MS, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DEFAULT_RECOVERY_TIMEOUT_MS, DEFAULT_STATEMENT_TIMEOUT_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HardenedPGliteDialect, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelNotWorkerImportableError, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PGliteAbortedTransactionError, PGliteAcquireTimeoutError, PGliteSessionError, PGliteSessionPoisonedError, PGliteStatementTimeoutError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncRepairRefusedError, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isLongRunningStatement, isPurgeMarker, isRecoverableGraphQLError, listPurged, loadDocumentModelSpec, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, queryThroughDialect, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
|
|
12252
13160
|
|
|
12253
13161
|
//# sourceMappingURL=index.js.map
|