@powerhousedao/reactor 6.2.3-dev.49 → 6.2.3-dev.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/dist/{document-indexer-Dc_Sv6yi.js → document-indexer-DwAMDVRI.js} +2 -2
  2. package/dist/{document-indexer-Dc_Sv6yi.js.map → document-indexer-DwAMDVRI.js.map} +1 -1
  3. package/dist/{drive-container-types-D380mcB3.js → drive-container-types-Ci1pdv4X.js} +9 -9
  4. package/dist/{drive-container-types-D380mcB3.js.map → drive-container-types-Ci1pdv4X.js.map} +1 -1
  5. package/dist/entry.js +3 -2
  6. package/dist/entry.js.map +1 -1
  7. package/dist/{load-spec-D9lnje-H.js → forwarding-logger-ByfBFgMS.js} +9 -248
  8. package/dist/forwarding-logger-ByfBFgMS.js.map +1 -0
  9. package/dist/{index-Dq2f7oAC.d.ts → index-CNhT69De.d.ts} +34 -5
  10. package/dist/index-CNhT69De.d.ts.map +1 -0
  11. package/dist/index.d.ts +126 -26
  12. package/dist/index.d.ts.map +1 -1
  13. package/dist/index.js +1065 -157
  14. package/dist/index.js.map +1 -1
  15. package/dist/load-spec-ZLjuyhxz.js +246 -0
  16. package/dist/load-spec-ZLjuyhxz.js.map +1 -0
  17. package/dist/projection-entry.js +3 -2
  18. package/dist/projection-entry.js.map +1 -1
  19. package/dist/{projection-shard-manager-MRJ9BjVM.js → projection-shard-manager-CQ5FwQmv.js} +2 -2
  20. package/dist/{projection-shard-manager-MRJ9BjVM.js.map → projection-shard-manager-CQ5FwQmv.js.map} +1 -1
  21. package/dist/rpc.d.ts +2 -2
  22. package/dist/rpc.js +3 -2
  23. package/dist/rpc.js.map +1 -1
  24. package/dist/{worker-handle-B48V1hDm.js → worker-handle-CN2-oRv-.js} +2 -2
  25. package/dist/{worker-handle-B48V1hDm.js.map → worker-handle-CN2-oRv-.js.map} +1 -1
  26. package/package.json +13 -13
  27. package/dist/index-Dq2f7oAC.d.ts.map +0 -1
  28. package/dist/load-spec-D9lnje-H.js.map +0 -1
package/dist/index.js CHANGED
@@ -1,12 +1,14 @@
1
- import { A as CollectionMembershipCache, At as ReactorEventTypes, C as ProbeSettler, Ct as PurgeTooLargeError, D as parseSnapshot, Dt as UnsupportedStoredProtocolError, E as describeWaitingSessions, Et as UnsupportedProtocolVersionError, F as ModuleNotFoundError, Ft as __exportAll, G as createConsistencyToken, K as createDocumentFromAction, M as DuplicateModuleError, Mt as parsePagingOptions, N as InvalidModuleError, Nt as throwIfAborted, O as snapshotFunctionsFor, Ot as UpgradePreconditionFailedError, P as ModelNotWorkerImportableError, R as GATED_DOCUMENT_ACTIONS, S as listPurged, St as InvalidSignatureError, T as createKyselyWatermarkProbe, Tt as ReservedActionError, _ as KyselyOperationIndex, _t as DocumentNotFoundError, a as sanitizeArg, at as AppendConditionFailedError, b as acquirePurgeLocks, bt as GroupInUseError, c as getMigrationStatus, ct as DuplicateOperationError, d as instrumentPgPool, dt as AuthEnforcementDisabledError, et as submittedActionIds, f as KyselyOperationStore, ft as AuthTimestampNotMonotonicError, g as KyselyWriteCache, gt as DocumentNotDeletedError, h as EventBus, ht as DocumentDeletedError, i as errorToInfo, it as APPEND_CONDITION_FAILED_PREFIX, j as DuplicateManifestError, jt as matchesScope, k as DocumentMetaCache, kt as EventBusAggregateError, l as runMigrations, lt as OptimisticLockError, m as DocumentModelRegistry, mt as BatchJobFailedError, n as loadDocumentModelSpec, nt as targetDocumentId, ot as DocumentAlreadyExistsError, p as KyselyKeyframeStore, pt as AuthorizationDeniedError, q as createEmptyConsistencyToken, r as createForwardingLogger, s as REACTOR_SCHEMA, st as DocumentExistence, tt as summarizeSubmittedActions, u as createForwardingPoolInstrumentation, ut as RevisionMismatchError, v as PURGE_LOCK_BUCKETS, vt as DocumentPurgedError, w as SettledWatermark, wt as RelationshipNotFoundError, x as findPurged, xt as InvalidOperationTimestampError, y as PURGE_NS, yt as ExcessiveReshuffleError } from "./load-spec-D9lnje-H.js";
2
- import { _ as buildDecisionModel, a as KyselyExecutionScope, c as appliedDelete, d as DEFAULT_MAX_PURGE_OPERATIONS, f as JobExecutorEventTypes, g as authDecisionModel, h as documentDecisionModel, i as PassthroughSigner, l as resolveFeatureFlags, m as selectDecisionModel, n as SimpleJobExecutor, o as DEFAULT_PURGE_DELETE_BATCH, p as decideAtHead, r as verifyActionSignature, s as KyselyDocumentPurger, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as DEFAULT_DEFERRED_JOB_TTL_MS, v as DriveCollectionId } from "./drive-container-types-D380mcB3.js";
1
+ import { $ as DocumentExistence, A as GATED_DOCUMENT_ACTIONS, C as describeWaitingSessions, Ct as matchesScope, D as CollectionMembershipCache, Dt as __exportAll, E as DocumentMetaCache, J as targetDocumentId, K as submittedActionIds, L as createConsistencyToken, Q as DocumentAlreadyExistsError, R as createDocumentFromAction, S as createKyselyWatermarkProbe, St as ReactorEventTypes, T as snapshotFunctionsFor, Tt as throwIfAborted, X as APPEND_CONDITION_FAILED_PREFIX, Z as AppendConditionFailedError, _ as acquirePurgeLocks, _t as ReservedActionError, a as REACTOR_SCHEMA, at as AuthorizationDeniedError, b as ProbeSettler, bt as UpgradePreconditionFailedError, c as createForwardingPoolInstrumentation, ct as DocumentNotDeletedError, d as KyselyKeyframeStore, dt as ExcessiveReshuffleError, et as DuplicateOperationError, f as EventBus, ft as GroupInUseError, g as PURGE_NS, gt as RelationshipNotFoundError, h as PURGE_LOCK_BUCKETS, ht as PurgeTooLargeError, it as AuthTimestampNotMonotonicError, l as instrumentPgPool, lt as DocumentNotFoundError, m as KyselyOperationIndex, mt as InvalidSignatureError, n as errorToInfo, nt as RevisionMismatchError, o as getMigrationStatus, ot as BatchJobFailedError, p as KyselyWriteCache, pt as InvalidOperationTimestampError, q as summarizeSubmittedActions, r as sanitizeArg, rt as AuthEnforcementDisabledError, s as runMigrations, st as DocumentDeletedError, t as createForwardingLogger, tt as OptimisticLockError, u as KyselyOperationStore, ut as DocumentPurgedError, v as findPurged, vt as UnsupportedProtocolVersionError, w as parseSnapshot, wt as parsePagingOptions, x as SettledWatermark, xt as EventBusAggregateError, y as listPurged, yt as UnsupportedStoredProtocolError, z as createEmptyConsistencyToken } from "./forwarding-logger-ByfBFgMS.js";
2
+ import { _ as buildDecisionModel, a as KyselyExecutionScope, c as appliedDelete, d as DEFAULT_MAX_PURGE_OPERATIONS, f as JobExecutorEventTypes, g as authDecisionModel, h as documentDecisionModel, i as PassthroughSigner, l as resolveFeatureFlags, m as selectDecisionModel, n as SimpleJobExecutor, o as DEFAULT_PURGE_DELETE_BATCH, p as decideAtHead, r as verifyActionSignature, s as KyselyDocumentPurger, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as DEFAULT_DEFERRED_JOB_TTL_MS, v as DriveCollectionId } from "./drive-container-types-Ci1pdv4X.js";
3
+ import { a as DuplicateModuleError, c as ModuleNotFoundError, i as DuplicateManifestError, n as loadDocumentModelSpec, o as InvalidModuleError, r as DocumentModelRegistry, s as ModelNotWorkerImportableError } from "./load-spec-ZLjuyhxz.js";
3
4
  import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-BM_ULVWd.js";
4
- import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-Dc_Sv6yi.js";
5
+ import { _ as ContiguousCursor, a as KyselyDocumentView, b as isCatchUpConsumer, c as ReadModelCoordinator, d as reserveBatch, f as BaseReadModel, g as unchunkedReadModelIndexingConfig, h as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as indexReserved, m as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as DEFAULT_COMMIT_CHUNK_SIZE, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as releaseBatch, v as defaultCatchUpConfig, y as CatchUpScheduler } from "./document-indexer-DwAMDVRI.js";
5
6
  import { DEFAULT_SIGNATURE_POLICY, DOCUMENT_PURGE_PROTOCOL, DowngradeNotSupportedError, MARKER_REFUSAL_FEATURE, MAX_AUTH_GRANTS, PEER_CAPABILITIES, PURGE_DOCUMENT, UnsupportedDocumentModelVersionError, actionSignerIdentity, actionSigningTarget, actions, actions as documentActions, canonicalJson, coversLocal, createCopyHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, holdReason, isOlderManifest, isPurgeMarker, isPurgeMarker as isPurgeMarker$1, legacySupports, localPeerManifest, localSupports, mergePeerCapabilities, normalizeDocumentModelVersion, peerSupports, purgedProtocolVersions, readPeerManifest, requestedSignaturePolicy, selectProtocolVersions, sortOperations, toTransportAction, withSignaturePolicy } from "@powerhousedao/shared/document-model";
6
7
  import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
7
8
  import { v4 } from "uuid";
8
9
  import { ConsoleLogger, childLogger } from "document-model";
9
- import { sql } from "kysely";
10
+ import { CompiledQuery, sql } from "kysely";
11
+ import { PGliteDialect } from "kysely-pglite-dialect";
10
12
  import { RelationalDbProcessor, createRelationalDb } from "@powerhousedao/shared/processors";
11
13
  //#region src/actions/index.ts
12
14
  /**
@@ -5600,6 +5602,7 @@ var Mailbox = class {
5600
5602
  itemsMap = /* @__PURE__ */ new Map();
5601
5603
  /** Unapplied items carrying a marker, so the held ack reads only these. */
5602
5604
  heldMarkers = /* @__PURE__ */ new Set();
5605
+ held = /* @__PURE__ */ new Set();
5603
5606
  addedCallbacks = [];
5604
5607
  removedCallbacks = [];
5605
5608
  paused = false;
@@ -5620,9 +5623,9 @@ var Mailbox = class {
5620
5623
  return Array.from(this.itemsMap.values());
5621
5624
  }
5622
5625
  get ackOrdinal() {
5623
- if (!this.holdAckBelowMarkers) return this._ack;
5626
+ if (!this.holdAckBelowMarkers && this.held.size === 0) return this._ack;
5624
5627
  let floor = Number.POSITIVE_INFINITY;
5625
- for (const item of this.heldMarkers) for (const op of item.operations) {
5628
+ for (const item of [...this.heldMarkers, ...this.held]) for (const op of item.operations) {
5626
5629
  const ordinal = op.context.ordinal;
5627
5630
  if (ordinal > 0 && ordinal < floor) floor = ordinal;
5628
5631
  }
@@ -5631,13 +5634,21 @@ var Mailbox = class {
5631
5634
  get latestOrdinal() {
5632
5635
  return this._latestOrdinal;
5633
5636
  }
5637
+ hold(item) {
5638
+ if (item.status === 2) return;
5639
+ if (this.itemsMap.get(item.id) !== item) return;
5640
+ this.held.add(item);
5641
+ }
5634
5642
  get(id) {
5635
5643
  return this.itemsMap.get(id);
5636
5644
  }
5637
5645
  add(...items) {
5638
5646
  for (const item of items) {
5639
5647
  const replaced = this.itemsMap.get(item.id);
5640
- if (replaced !== void 0) this.heldMarkers.delete(replaced);
5648
+ if (replaced !== void 0 && replaced !== item) {
5649
+ this.heldMarkers.delete(replaced);
5650
+ this.held.delete(replaced);
5651
+ }
5641
5652
  this.itemsMap.set(item.id, item);
5642
5653
  let marker = false;
5643
5654
  for (const op of item.operations) {
@@ -5648,6 +5659,7 @@ var Mailbox = class {
5648
5659
  item.on((syncOp, _, next) => {
5649
5660
  if (next === 2) {
5650
5661
  this.heldMarkers.delete(syncOp);
5662
+ this.held.delete(syncOp);
5651
5663
  for (const op of syncOp.operations) this._ack = Math.max(this._ack, op.context.ordinal);
5652
5664
  }
5653
5665
  });
@@ -5669,6 +5681,7 @@ var Mailbox = class {
5669
5681
  for (const item of items) {
5670
5682
  this.itemsMap.delete(item.id);
5671
5683
  this.heldMarkers.delete(item);
5684
+ this.held.delete(item);
5672
5685
  }
5673
5686
  if (this.paused) {
5674
5687
  this.removedBuffer.push(...items);
@@ -5945,6 +5958,13 @@ function isDriveAuthError(error) {
5945
5958
  if (error.category === "graphql") return Object.values(DRIVE_AUTH_ERROR_MESSAGES).some((m) => error.message.includes(m));
5946
5959
  return false;
5947
5960
  }
5961
+ /** A repair lever declined a request it cannot carry out safely. */
5962
+ var SyncRepairRefusedError = class extends Error {
5963
+ constructor(message) {
5964
+ super(message);
5965
+ this.name = "SyncRepairRefusedError";
5966
+ }
5967
+ };
5948
5968
  var PollingChannelError = class extends Error {
5949
5969
  constructor(message) {
5950
5970
  super(message);
@@ -6362,6 +6382,7 @@ function classifyJobFailure(errorName) {
6362
6382
  case "HashMismatchError": return "HASH_MISMATCH";
6363
6383
  case "UnsupportedProtocolVersionError": return "UNSUPPORTED_PROTOCOL";
6364
6384
  case "DocumentPurgedError": return "DOCUMENT_PURGED";
6385
+ case "DocumentNotFoundError": return "MISSING_OPERATIONS";
6365
6386
  case "DocumentNotDeletedError":
6366
6387
  case "GroupInUseError":
6367
6388
  case "PurgeTooLargeError": return "PURGE_PRECONDITION";
@@ -6380,47 +6401,57 @@ const NON_QUARANTINING_ERROR_TYPES = /* @__PURE__ */ new Set([
6380
6401
  "PEER_PROTOCOL_UNSUPPORTED",
6381
6402
  "DOCUMENT_PURGED",
6382
6403
  "PURGE_PRECONDITION",
6383
- "MARKER_REFUSED"
6404
+ "MARKER_REFUSED",
6405
+ "MISSING_OPERATIONS"
6384
6406
  ]);
6385
6407
  /**
6386
6408
  * A held auth operation must not quarantine: reconciling the two policies needs
6387
6409
  * the traffic a quarantine would stop. A protocol refusal concerns one peer,
6388
- * and quarantine is global to the document.
6410
+ * and quarantine is global to the document. A missing ancestor arrives through
6411
+ * exactly the traffic a quarantine stops.
6389
6412
  */
6390
6413
  function quarantinesDocument(errorType) {
6391
6414
  return !NON_QUARANTINING_ERROR_TYPES.has(errorType);
6392
6415
  }
6393
6416
  //#endregion
6394
6417
  //#region src/sync/channels/interval-poll-timer.ts
6418
+ /** Lower bound on the derived {@link PollTimerConfig.delegateTimeoutMs}. */
6419
+ const DELEGATE_TIMEOUT_FLOOR_MS = 3e4;
6395
6420
  const DEFAULT_CONFIG = {
6396
6421
  intervalMs: 2e3,
6397
6422
  maxQueueDepth: 100,
6398
6423
  backpressureCheckIntervalMs: 500,
6399
6424
  retryBaseDelayMs: 1e3,
6400
6425
  retryMaxDelayMs: 3e5,
6401
- startPaused: false
6426
+ startPaused: false,
6427
+ delegateTimeoutMs: DELEGATE_TIMEOUT_FLOOR_MS,
6428
+ queueProbeTimeoutMs: 1e4
6402
6429
  };
6403
6430
  function calculateBackoffDelay(consecutiveFailures, retryBaseDelayMs, retryMaxDelayMs, random) {
6404
6431
  const backoff = Math.min(retryMaxDelayMs, retryBaseDelayMs * Math.pow(2, consecutiveFailures - 1));
6405
6432
  return backoff / 2 + random * (backoff / 2);
6406
6433
  }
6407
- /**
6408
- * Default poll timer using setTimeout.
6409
- * Waits for delegate completion before scheduling next tick.
6410
- * Checks queue depth and defers polling when backpressure is detected.
6411
- */
6434
+ /** setTimeout poll timer; a stuck tick is cancelled and the next waits for it to settle. */
6412
6435
  var IntervalPollTimer = class {
6413
6436
  delegate;
6414
6437
  timer;
6438
+ watchdog;
6415
6439
  running;
6416
6440
  paused;
6417
6441
  consecutiveFailures;
6442
+ tickToken = 0;
6443
+ ticking = false;
6444
+ tickAbort;
6445
+ cancelledToken;
6446
+ retriggerRequested = false;
6418
6447
  queue;
6419
6448
  config;
6420
6449
  constructor(queue, config = {}) {
6421
6450
  this.queue = queue;
6451
+ const intervalMs = config.intervalMs ?? DEFAULT_CONFIG.intervalMs;
6422
6452
  this.config = {
6423
6453
  ...DEFAULT_CONFIG,
6454
+ delegateTimeoutMs: Math.max(DELEGATE_TIMEOUT_FLOOR_MS, intervalMs * 10),
6424
6455
  ...config
6425
6456
  };
6426
6457
  this.running = false;
@@ -6437,54 +6468,25 @@ var IntervalPollTimer = class {
6437
6468
  }
6438
6469
  stop() {
6439
6470
  this.running = false;
6440
- if (this.timer) {
6441
- clearTimeout(this.timer);
6442
- this.timer = void 0;
6443
- }
6444
- }
6445
- tick() {
6446
- if (!this.delegate || !this.running) return;
6447
- const delegate = this.delegate;
6448
- this.queue.totalSize().then((size) => {
6449
- if (!this.running) return;
6450
- if (size > this.config.maxQueueDepth) this.scheduleBackpressureRecheck();
6451
- else delegate().then(() => {
6452
- this.consecutiveFailures = 0;
6453
- this.scheduleNext();
6454
- }).catch(() => {
6455
- this.consecutiveFailures++;
6456
- this.scheduleRetry();
6457
- });
6458
- }).catch(() => {
6459
- this.scheduleNext();
6460
- });
6461
- }
6462
- scheduleNext() {
6463
- if (!this.running || this.paused) return;
6464
- this.timer = setTimeout(() => this.tick(), this.config.intervalMs);
6465
- }
6466
- scheduleRetry() {
6467
- if (!this.running || this.paused) return;
6468
- const delay = calculateBackoffDelay(this.consecutiveFailures, this.config.retryBaseDelayMs, this.config.retryMaxDelayMs, Math.random());
6469
- this.timer = setTimeout(() => this.tick(), delay);
6470
- }
6471
- scheduleBackpressureRecheck() {
6472
- if (!this.running || this.paused) return;
6473
- this.timer = setTimeout(() => this.tick(), this.config.backpressureCheckIntervalMs);
6471
+ this.retriggerRequested = false;
6472
+ this.clearTimer();
6473
+ this.clearWatchdog();
6474
+ this.cancelTick(/* @__PURE__ */ new Error("poll timer stopped"));
6475
+ this.tickToken++;
6476
+ this.cancelledToken = void 0;
6477
+ this.ticking = false;
6474
6478
  }
6475
6479
  pause() {
6476
6480
  this.paused = true;
6477
- if (this.timer) {
6478
- clearTimeout(this.timer);
6479
- this.timer = void 0;
6480
- }
6481
+ this.clearTimer();
6481
6482
  }
6482
6483
  resume() {
6483
6484
  this.paused = false;
6484
6485
  if (this.running) this.scheduleNext();
6485
6486
  }
6486
6487
  triggerNow() {
6487
- if (this.running && this.delegate) this.tick();
6488
+ if (!this.running || !this.delegate) return;
6489
+ this.tick();
6488
6490
  }
6489
6491
  isPaused() {
6490
6492
  return this.paused;
@@ -6498,6 +6500,133 @@ var IntervalPollTimer = class {
6498
6500
  setIntervalMs(ms) {
6499
6501
  this.config.intervalMs = ms;
6500
6502
  }
6503
+ tick() {
6504
+ if (!this.delegate || !this.running) return;
6505
+ if (this.ticking) {
6506
+ this.retriggerRequested = true;
6507
+ return;
6508
+ }
6509
+ const delegate = this.delegate;
6510
+ const token = ++this.tickToken;
6511
+ this.ticking = true;
6512
+ this.cancelledToken = void 0;
6513
+ const abort = new AbortController();
6514
+ this.tickAbort = abort;
6515
+ this.clearTimer();
6516
+ this.armWatchdog(token);
6517
+ this.measureQueue(abort.signal).then((size) => {
6518
+ if (!this.running) {
6519
+ this.settle(token, "stopped");
6520
+ return;
6521
+ }
6522
+ if (size === "cancelled") {
6523
+ this.settle(token, "failure");
6524
+ return;
6525
+ }
6526
+ if (size === "unknown") {
6527
+ this.runDelegate(delegate, token, abort.signal);
6528
+ return;
6529
+ }
6530
+ if (size > this.config.maxQueueDepth) {
6531
+ this.settle(token, "backpressure");
6532
+ return;
6533
+ }
6534
+ this.runDelegate(delegate, token, abort.signal);
6535
+ });
6536
+ }
6537
+ runDelegate(delegate, token, signal) {
6538
+ delegate(signal).then(() => this.settle(token, "success"), () => this.settle(token, "failure"));
6539
+ }
6540
+ /** `"unknown"` when the probe timed out or rejected; `"cancelled"` when the tick was. */
6541
+ async measureQueue(signal) {
6542
+ const timeoutMs = this.config.queueProbeTimeoutMs;
6543
+ let handle;
6544
+ const expiry = new Promise((resolve) => {
6545
+ if (timeoutMs > 0) handle = setTimeout(() => resolve("unknown"), timeoutMs);
6546
+ });
6547
+ const cancelled = new Promise((resolve) => {
6548
+ if (signal.aborted) {
6549
+ resolve("cancelled");
6550
+ return;
6551
+ }
6552
+ signal.addEventListener("abort", () => resolve("cancelled"), { once: true });
6553
+ });
6554
+ try {
6555
+ return await Promise.race([
6556
+ this.queue.totalSize(),
6557
+ expiry,
6558
+ cancelled
6559
+ ]);
6560
+ } catch {
6561
+ return "unknown";
6562
+ } finally {
6563
+ if (handle !== void 0) clearTimeout(handle);
6564
+ }
6565
+ }
6566
+ settle(token, outcome) {
6567
+ if (token !== this.tickToken) return;
6568
+ this.clearWatchdog();
6569
+ this.ticking = false;
6570
+ this.tickAbort = void 0;
6571
+ const cancelled = this.cancelledToken === token;
6572
+ this.cancelledToken = void 0;
6573
+ const effective = cancelled && outcome !== "stopped" ? "failure" : outcome;
6574
+ if (effective === "success") this.consecutiveFailures = 0;
6575
+ else if (effective === "failure") this.consecutiveFailures++;
6576
+ if (effective === "stopped" || !this.running) return;
6577
+ if (this.retriggerRequested) {
6578
+ this.retriggerRequested = false;
6579
+ this.tick();
6580
+ return;
6581
+ }
6582
+ if (effective === "backpressure") this.scheduleBackpressureRecheck();
6583
+ else if (effective === "failure") this.scheduleRetry();
6584
+ else this.scheduleNext();
6585
+ }
6586
+ /** Cancels rather than abandons the tick, so two delegates never overlap. */
6587
+ armWatchdog(token) {
6588
+ const timeoutMs = this.config.delegateTimeoutMs;
6589
+ if (timeoutMs <= 0) return;
6590
+ this.watchdog = setTimeout(() => {
6591
+ this.watchdog = void 0;
6592
+ if (token !== this.tickToken || !this.ticking) return;
6593
+ this.cancelledToken = token;
6594
+ this.cancelTick(/* @__PURE__ */ new Error(`poll delegate exceeded its ${timeoutMs}ms bound`));
6595
+ }, timeoutMs);
6596
+ }
6597
+ cancelTick(reason) {
6598
+ const abort = this.tickAbort;
6599
+ if (abort === void 0 || abort.signal.aborted) return;
6600
+ abort.abort(reason);
6601
+ }
6602
+ clearWatchdog() {
6603
+ if (this.watchdog) {
6604
+ clearTimeout(this.watchdog);
6605
+ this.watchdog = void 0;
6606
+ }
6607
+ }
6608
+ clearTimer() {
6609
+ if (this.timer) {
6610
+ clearTimeout(this.timer);
6611
+ this.timer = void 0;
6612
+ }
6613
+ }
6614
+ scheduleNext() {
6615
+ if (!this.running || this.paused) return;
6616
+ this.clearTimer();
6617
+ this.timer = setTimeout(() => this.tick(), this.config.intervalMs);
6618
+ }
6619
+ scheduleRetry() {
6620
+ if (!this.running || this.paused) return;
6621
+ const delay = calculateBackoffDelay(this.consecutiveFailures, this.config.retryBaseDelayMs, this.config.retryMaxDelayMs, Math.random());
6622
+ this.clearTimer();
6623
+ this.timer = setTimeout(() => this.tick(), delay);
6624
+ }
6625
+ scheduleBackpressureRecheck() {
6626
+ if (!this.running || this.paused) return;
6627
+ this.clearTimer();
6628
+ this.timer = setTimeout(() => this.tick(), this.config.backpressureCheckIntervalMs);
6629
+ }
6501
6630
  };
6502
6631
  //#endregion
6503
6632
  //#region src/sync/purge-refusals.ts
@@ -6620,6 +6749,7 @@ const getLatestAppliedOrdinal = (syncOps) => {
6620
6749
  };
6621
6750
  //#endregion
6622
6751
  //#region src/sync/channels/gql-req-channel.ts
6752
+ const DEFAULT_REQUEST_TIMEOUT_MS = 6e4;
6623
6753
  /**
6624
6754
  * Fields the auth projection added to the sync schema. A remote that predates
6625
6755
  * them rejects the whole query for naming one, so they are selected only while
@@ -6670,8 +6800,18 @@ var GqlRequestChannel = class {
6670
6800
  failureCount;
6671
6801
  lastSuccessUtcMs;
6672
6802
  lastFailureUtcMs;
6673
- lastPersistedInboxOrdinal = 0;
6674
- lastPersistedOutboxOrdinal = 0;
6803
+ cursorWriters = {
6804
+ inbox: {
6805
+ persisted: 0,
6806
+ requested: 0,
6807
+ tail: Promise.resolve()
6808
+ },
6809
+ outbox: {
6810
+ persisted: 0,
6811
+ requested: 0,
6812
+ tail: Promise.resolve()
6813
+ }
6814
+ };
6675
6815
  pushFailureCount = 0;
6676
6816
  pushRetryTimer = null;
6677
6817
  pushBlocked = false;
@@ -6698,6 +6838,8 @@ var GqlRequestChannel = class {
6698
6838
  connectionState = "connecting";
6699
6839
  /** Latest unrecoverable error was an auth rejection; cleared on connect. */
6700
6840
  requiresAuth = false;
6841
+ polling = false;
6842
+ polledSinceStart = false;
6701
6843
  connectionStateCallbacks = /* @__PURE__ */ new Set();
6702
6844
  constructor(logger, channelId, remoteName, cursorStorage, config, operationIndex, pollTimer) {
6703
6845
  this.logger = logger;
@@ -6713,7 +6855,8 @@ var GqlRequestChannel = class {
6713
6855
  collectionId: config.collectionId,
6714
6856
  filter: config.filter,
6715
6857
  retryBaseDelayMs: config.retryBaseDelayMs,
6716
- retryMaxDelayMs: config.retryMaxDelayMs
6858
+ retryMaxDelayMs: config.retryMaxDelayMs,
6859
+ requestTimeoutMs: config.requestTimeoutMs ?? 6e4
6717
6860
  };
6718
6861
  this.isShutdown = false;
6719
6862
  this.failureCount = 0;
@@ -6755,33 +6898,38 @@ var GqlRequestChannel = class {
6755
6898
  this.outbox.onRemoved((syncOps) => {
6756
6899
  for (const syncOp of syncOps) this.markerPushedAt.delete(syncOp.id);
6757
6900
  const ordinal = Math.min(getLatestAppliedOrdinal(syncOps), this.unappliedFloor() - 1);
6758
- if (ordinal > this.lastPersistedOutboxOrdinal) {
6759
- this.lastPersistedOutboxOrdinal = ordinal;
6760
- this.cursorStorage.upsert({
6761
- remoteName: this.remoteName,
6762
- cursorType: "outbox",
6763
- cursorOrdinal: ordinal,
6764
- lastSyncedAtUtcMs: Date.now()
6765
- }).catch((error) => {
6766
- this.logger.error("Failed to update outbox cursor for @ChannelId! This means that future application runs may resend duplicate operations. This is recoverable (with deduplication protection), but not-optimal: @Error", this.channelId, error);
6767
- });
6768
- }
6901
+ this.persistCursor("outbox", ordinal);
6769
6902
  });
6770
6903
  this.inbox.onRemoved(() => {
6771
- const maxOrdinal = this.inbox.ackOrdinal;
6772
- if (maxOrdinal > this.lastPersistedInboxOrdinal) {
6773
- this.lastPersistedInboxOrdinal = maxOrdinal;
6774
- this.cursorStorage.upsert({
6775
- remoteName: this.remoteName,
6776
- cursorType: "inbox",
6777
- cursorOrdinal: maxOrdinal,
6778
- lastSyncedAtUtcMs: Date.now()
6779
- }).catch((error) => {
6780
- this.logger.error("Failed to update inbox cursor for @ChannelId! This is unlikely to cause a problem, but not-optimal: @Error", this.channelId, error);
6781
- });
6782
- }
6904
+ this.persistCursor("inbox", this.inbox.ackOrdinal);
6783
6905
  });
6784
6906
  }
6907
+ /** Advances the watermark only once the write lands; writes per row run one at a time, coalesced. */
6908
+ persistCursor(cursorType, ordinal) {
6909
+ const writer = this.cursorWriters[cursorType];
6910
+ if (ordinal <= Math.max(writer.persisted, writer.requested)) return;
6911
+ writer.requested = ordinal;
6912
+ writer.tail = writer.tail.then(() => this.writeCursor(cursorType));
6913
+ }
6914
+ /** One link of a cursor write chain; never rejects, so the chain survives. */
6915
+ async writeCursor(cursorType) {
6916
+ const writer = this.cursorWriters[cursorType];
6917
+ const ordinal = writer.requested;
6918
+ if (ordinal <= writer.persisted) return;
6919
+ writer.requested = 0;
6920
+ try {
6921
+ await this.cursorStorage.upsert({
6922
+ remoteName: this.remoteName,
6923
+ cursorType,
6924
+ cursorOrdinal: ordinal,
6925
+ lastSyncedAtUtcMs: Date.now()
6926
+ });
6927
+ } catch (error) {
6928
+ this.logger.error("Failed to update @CursorType cursor for @ChannelId at ordinal @Ordinal; the watermark stays put so the next advance retries it: @Error", cursorType, this.channelId, ordinal, error);
6929
+ return;
6930
+ }
6931
+ writer.persisted = Math.max(writer.persisted, ordinal);
6932
+ }
6785
6933
  /**
6786
6934
  * Shuts down the channel and prevents further operations.
6787
6935
  */
@@ -6789,7 +6937,7 @@ var GqlRequestChannel = class {
6789
6937
  this.abortController.abort();
6790
6938
  this.bufferedOutbox.flush();
6791
6939
  this.isShutdown = true;
6792
- this.pollTimer.stop();
6940
+ this.stopPolling();
6793
6941
  if (this.pushRetryTimer) {
6794
6942
  clearTimeout(this.pushRetryTimer);
6795
6943
  this.pushRetryTimer = null;
@@ -6848,31 +6996,31 @@ var GqlRequestChannel = class {
6848
6996
  }
6849
6997
  /**
6850
6998
  * Re-touches once when either side's manifest moved; touching is idempotent.
6851
- * False when the refresh failed, so polled rows must not be judged yet.
6999
+ * False only when shutting down; a failed refresh raises as a recoverable
7000
+ * poll failure, so the next tick retries it.
6852
7001
  */
6853
- async refreshManifestsIfStale(manifestRevision, peerManifestRevision) {
7002
+ async refreshManifestsIfStale(manifestRevision, peerManifestRevision, signal) {
6854
7003
  if (!this.peerServesAgreement || typeof manifestRevision !== "string") return true;
6855
7004
  const local = this.localManifestProvider?.().revision ?? null;
6856
7005
  if (manifestRevision === (this.peerManifest?.revision ?? null) && (peerManifestRevision ?? null) === local) return true;
6857
7006
  if (this.isShutdown) return false;
6858
- this.manifestRefresh ??= this.touchRemoteChannel().then(({ ackOrdinal }) => {
7007
+ this.manifestRefresh ??= this.touchRemoteChannel(signal).then(({ ackOrdinal }) => {
6859
7008
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
6860
7009
  }).finally(() => {
6861
7010
  this.manifestRefresh = void 0;
6862
7011
  });
6863
7012
  try {
6864
7013
  await this.manifestRefresh;
6865
- return true;
6866
7014
  } catch (error) {
6867
- this.logger.error("GqlChannel @ChannelId manifest refresh failed: @Error", this.channelId, error);
6868
- return false;
7015
+ throw new Error(`Manifest refresh failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
6869
7016
  }
7017
+ return true;
6870
7018
  }
6871
7019
  /** Re-touches a silent remote in case it was upgraded. */
6872
- async probeAgreement() {
7020
+ async probeAgreement(signal) {
6873
7021
  this.agreementStoppedUtcMs = Date.now();
6874
7022
  try {
6875
- const { ackOrdinal } = await this.touchRemoteChannel();
7023
+ const { ackOrdinal } = await this.touchRemoteChannel(signal);
6876
7024
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
6877
7025
  } catch (error) {
6878
7026
  this.logger.error("GqlChannel @ChannelId agreement probe failed: @Error", this.channelId, error);
@@ -6882,21 +7030,38 @@ var GqlRequestChannel = class {
6882
7030
  * Initializes the channel by registering it on the remote server and starting polling.
6883
7031
  */
6884
7032
  async init() {
6885
- const { ackOrdinal } = await this.touchRemoteChannel();
7033
+ const { ackOrdinal } = await this.touchRemoteChannel(void 0);
6886
7034
  const cursors = await this.cursorStorage.list(this.remoteName);
6887
7035
  const inboxOrdinal = cursors.find((c) => c.cursorType === "inbox")?.cursorOrdinal ?? 0;
6888
7036
  const outboxOrdinal = cursors.find((c) => c.cursorType === "outbox")?.cursorOrdinal ?? 0;
6889
7037
  this.inbox.init(inboxOrdinal);
6890
7038
  this.outbox.init(outboxOrdinal);
6891
- this.lastPersistedInboxOrdinal = inboxOrdinal;
6892
- this.lastPersistedOutboxOrdinal = outboxOrdinal;
7039
+ this.cursorWriters.inbox.persisted = inboxOrdinal;
7040
+ this.cursorWriters.outbox.persisted = outboxOrdinal;
6893
7041
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
6894
- this.pollTimer.setDelegate(() => this.poll());
7042
+ this.pollTimer.setDelegate((signal) => this.poll(signal));
7043
+ this.startPolling();
7044
+ this.transitionConnectionState(this.reachedState());
7045
+ }
7046
+ startPolling() {
7047
+ this.polling = true;
7048
+ this.polledSinceStart = false;
6895
7049
  this.pollTimer.start();
6896
- this.transitionConnectionState("connected");
7050
+ }
7051
+ stopPolling() {
7052
+ this.polling = false;
7053
+ this.polledSinceStart = false;
7054
+ this.pollTimer.stop();
7055
+ }
7056
+ /**
7057
+ * `"connected"` is earned by a poll completed since the loop last started; a
7058
+ * timer that polls only on demand earns it with the touch that started it.
7059
+ */
7060
+ reachedState() {
7061
+ return this.polledSinceStart || this.pollTimer.isPaused?.() === true ? "connected" : "connecting";
6897
7062
  }
6898
7063
  transitionConnectionState(next) {
6899
- if (next === "connected") this.requiresAuth = false;
7064
+ if (next === "connected" || next === "connecting") this.requiresAuth = false;
6900
7065
  if (this.connectionState === next) return;
6901
7066
  this.connectionState = next;
6902
7067
  const snapshot = this.getConnectionState();
@@ -6907,23 +7072,26 @@ var GqlRequestChannel = class {
6907
7072
  }
6908
7073
  }
6909
7074
  /**
6910
- * Polls the remote for new sync envelopes.
7075
+ * Polls the remote for new sync envelopes; any failure of the body is recorded.
6911
7076
  */
6912
- async poll() {
7077
+ async poll(signal) {
6913
7078
  if (this.isShutdown) return;
6914
- if (!this.peerServesAgreement && Date.now() - this.agreementStoppedUtcMs >= AGREEMENT_PROBE_INTERVAL_MS) await this.probeAgreement();
6915
- let response;
6916
7079
  try {
6917
- response = await this.pollSyncEnvelopes(this.inbox.ackOrdinal, this.inbox.latestOrdinal);
7080
+ await this.pollOnce(signal);
6918
7081
  } catch (error) {
6919
7082
  if (!this.handlePollError(error)) throw error;
6920
- return;
6921
7083
  }
7084
+ }
7085
+ async pollOnce(signal) {
7086
+ if (!this.peerServesAgreement && Date.now() - this.agreementStoppedUtcMs >= AGREEMENT_PROBE_INTERVAL_MS) await this.probeAgreement(signal);
7087
+ const response = await this.pollSyncEnvelopes(this.inbox.ackOrdinal, this.inbox.latestOrdinal, signal);
7088
+ this.throwIfPollCancelled(signal);
6922
7089
  const { envelopes, ackOrdinal, deadLetters, hasMore, manifestRevision, peerManifestRevision } = response;
6923
7090
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
6924
7091
  this.retireRefusedMarkers(deadLetters);
6925
7092
  this.repushUnackedMarkers();
6926
- if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision)) return;
7093
+ if (!await this.refreshManifestsIfStale(manifestRevision, peerManifestRevision, signal)) return;
7094
+ this.throwIfPollCancelled(signal);
6927
7095
  const allSyncOps = [];
6928
7096
  for (const envelope of envelopes) if (envelope.type.toLowerCase() === "operations" && envelope.operations) {
6929
7097
  const syncOps = envelopesToSyncOperations(envelope, this.remoteName);
@@ -6940,6 +7108,7 @@ var GqlRequestChannel = class {
6940
7108
  }
6941
7109
  this.lastSuccessUtcMs = Date.now();
6942
7110
  this.failureCount = 0;
7111
+ this.polledSinceStart = true;
6943
7112
  this.transitionConnectionState("connected");
6944
7113
  }
6945
7114
  /**
@@ -6982,7 +7151,7 @@ var GqlRequestChannel = class {
6982
7151
  const channelError = new ChannelError("inbox", err);
6983
7152
  this.logger.error("GqlChannel poll error (@FailureCount, @Classification): @Error", this.failureCount, classification, channelError);
6984
7153
  if (classification === "unrecoverable") {
6985
- this.pollTimer.stop();
7154
+ this.stopPolling();
6986
7155
  this.requiresAuth = isDriveAuthError(err);
6987
7156
  this.transitionConnectionState("error");
6988
7157
  return true;
@@ -6998,19 +7167,19 @@ var GqlRequestChannel = class {
6998
7167
  if (this.isRecovering) return;
6999
7168
  this.isRecovering = true;
7000
7169
  this.logger.info("GqlChannel @ChannelId not found on remote, re-registering...", this.channelId);
7001
- this.pollTimer.stop();
7170
+ this.stopPolling();
7002
7171
  const attemptRecovery = (attempt) => {
7003
7172
  if (this.isShutdown) {
7004
7173
  this.isRecovering = false;
7005
7174
  return;
7006
7175
  }
7007
- this.touchRemoteChannel().then(({ ackOrdinal }) => {
7176
+ this.touchRemoteChannel(void 0).then(({ ackOrdinal }) => {
7008
7177
  this.logger.info("GqlChannel @ChannelId re-registered successfully", this.channelId);
7009
7178
  this.isRecovering = false;
7010
7179
  this.failureCount = 0;
7011
7180
  if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
7012
- this.pollTimer.start();
7013
- this.transitionConnectionState("connected");
7181
+ this.startPolling();
7182
+ this.transitionConnectionState(this.reachedState());
7014
7183
  this.resumePushAfterRecovery();
7015
7184
  }).catch((recoveryError) => {
7016
7185
  const err = recoveryError instanceof Error ? recoveryError : new Error(String(recoveryError));
@@ -7050,7 +7219,7 @@ var GqlRequestChannel = class {
7050
7219
  /**
7051
7220
  * Queries the remote GraphQL endpoint for sync envelopes.
7052
7221
  */
7053
- async pollSyncEnvelopes(ackOrdinal, latestOrdinal) {
7222
+ async pollSyncEnvelopes(ackOrdinal, latestOrdinal, signal) {
7054
7223
  const variables = {
7055
7224
  channelId: this.channelId,
7056
7225
  outboxAck: ackOrdinal,
@@ -7066,7 +7235,7 @@ var GqlRequestChannel = class {
7066
7235
  ...variables,
7067
7236
  manifestRevision: revision,
7068
7237
  refusals
7069
- } : variables);
7238
+ } : variables, signal);
7070
7239
  break;
7071
7240
  } catch (error) {
7072
7241
  if (this.rejectsAgreementFields(error)) {
@@ -7197,7 +7366,7 @@ var GqlRequestChannel = class {
7197
7366
  * Registers or updates this channel on the remote server via GraphQL mutation.
7198
7367
  * Returns the remote's ack ordinal so the client can trim its outbox.
7199
7368
  */
7200
- async touchRemoteChannel() {
7369
+ async touchRemoteChannel(signal) {
7201
7370
  let sinceTimestampUtcMs = "0";
7202
7371
  try {
7203
7372
  const result = await this.operationIndex.getLatestTimestampForCollection(this.config.collectionId.key);
@@ -7224,7 +7393,7 @@ var GqlRequestChannel = class {
7224
7393
  },
7225
7394
  sinceTimestampUtcMs,
7226
7395
  ...manifest ? { manifest } : {}
7227
- } });
7396
+ } }, signal);
7228
7397
  };
7229
7398
  let data;
7230
7399
  try {
@@ -7251,7 +7420,7 @@ var GqlRequestChannel = class {
7251
7420
  this.isPushing = false;
7252
7421
  this.pushBlocked = false;
7253
7422
  this.pushFailureCount = 0;
7254
- if (this.connectionState === "reconnecting" || this.connectionState === "error") this.transitionConnectionState("connected");
7423
+ if (this.polling && (this.connectionState === "reconnecting" || this.connectionState === "error")) this.transitionConnectionState(this.reachedState());
7255
7424
  this.drainOutbox();
7256
7425
  }).catch((error) => {
7257
7426
  this.isPushing = false;
@@ -7268,7 +7437,7 @@ var GqlRequestChannel = class {
7268
7437
  this.pushFailureCount++;
7269
7438
  this.pushBlocked = true;
7270
7439
  this.logger.error("GqlChannel push failed (attempt @FailureCount), will retry: @Error", this.pushFailureCount, err);
7271
- this.transitionConnectionState("reconnecting");
7440
+ if (this.polling) this.transitionConnectionState("reconnecting");
7272
7441
  this.schedulePushRetry();
7273
7442
  } else {
7274
7443
  const channelError = new ChannelError("outbox", err);
@@ -7342,6 +7511,7 @@ var GqlRequestChannel = class {
7342
7511
  if (error.statusCode !== void 0 && error.statusCode >= 500) return "recoverable";
7343
7512
  return "unrecoverable";
7344
7513
  case "parse": return "recoverable";
7514
+ case "timeout": return "recoverable";
7345
7515
  case "graphql": return isRecoverableGraphQLError(error) ? "recoverable" : "unrecoverable";
7346
7516
  case "missing-data": return "unrecoverable";
7347
7517
  }
@@ -7361,7 +7531,7 @@ var GqlRequestChannel = class {
7361
7531
  await this.executeGraphQL(pushMutation(gatedUnder !== void 0), {
7362
7532
  envelopes: this.envelopesFor(syncOps),
7363
7533
  ...gatedUnder !== void 0 ? { peerManifestRevision: gatedUnder } : {}
7364
- });
7534
+ }, void 0);
7365
7535
  return;
7366
7536
  } catch (error) {
7367
7537
  if (gatedUnder === void 0 || !this.rejectsAgreementFields(error)) throw error;
@@ -7370,7 +7540,7 @@ var GqlRequestChannel = class {
7370
7540
  const unsent = new Set(this.outbox.items);
7371
7541
  const remaining = syncOps.filter((syncOp) => unsent.has(syncOp));
7372
7542
  if (remaining.length === 0) return;
7373
- await this.executeGraphQL(pushMutation(false), { envelopes: this.envelopesFor(remaining) });
7543
+ await this.executeGraphQL(pushMutation(false), { envelopes: this.envelopesFor(remaining) }, void 0);
7374
7544
  }
7375
7545
  /** One envelope per SyncOperation, with key/dependsOn for batch ordering. */
7376
7546
  envelopesFor(syncOps) {
@@ -7411,36 +7581,95 @@ var GqlRequestChannel = class {
7411
7581
  this.logger.error("JWT handler failed: @Error", error);
7412
7582
  }
7413
7583
  }
7414
- /**
7415
- * Executes a GraphQL query or mutation against the remote endpoint.
7416
- */
7417
- async executeGraphQL(query, variables) {
7584
+ /** Aborted by shutdown, tick cancellation or the deadline; `expired` bounds what ignores it. */
7585
+ requestDeadline(timeoutMs, tick) {
7586
+ const controller = new AbortController();
7587
+ const onShutdown = () => controller.abort(this.abortController.signal.reason);
7588
+ if (this.abortController.signal.aborted) onShutdown();
7589
+ else this.abortController.signal.addEventListener("abort", onShutdown, { once: true });
7590
+ let expireTick = () => void 0;
7591
+ const onTickCancelled = () => {
7592
+ controller.abort(tick?.reason);
7593
+ expireTick();
7594
+ };
7595
+ if (tick?.aborted) onTickCancelled();
7596
+ else tick?.addEventListener("abort", onTickCancelled, { once: true });
7597
+ let timer;
7598
+ const expired = new Promise((resolve) => {
7599
+ expireTick = () => resolve();
7600
+ if (tick?.aborted) resolve();
7601
+ if (timeoutMs > 0) timer = setTimeout(() => {
7602
+ controller.abort(/* @__PURE__ */ new Error("request deadline exceeded"));
7603
+ resolve();
7604
+ }, timeoutMs);
7605
+ });
7606
+ return {
7607
+ signal: controller.signal,
7608
+ expired,
7609
+ timeoutMs,
7610
+ dispose: () => {
7611
+ if (timer !== void 0) clearTimeout(timer);
7612
+ this.abortController.signal.removeEventListener("abort", onShutdown);
7613
+ tick?.removeEventListener("abort", onTickCancelled);
7614
+ }
7615
+ };
7616
+ }
7617
+ /** A cancelled poll must not touch mailboxes or cursors the next poll will. */
7618
+ throwIfPollCancelled(signal) {
7619
+ if (signal?.aborted !== true) return;
7620
+ const reason = signal.reason;
7621
+ throw reason instanceof Error ? reason : /* @__PURE__ */ new Error(`poll cancelled: ${String(reason)}`);
7622
+ }
7623
+ async withDeadline(work, deadline, what) {
7624
+ const timedOut = Symbol("timed-out");
7625
+ const outcome = await Promise.race([work, deadline.expired.then(() => timedOut)]);
7626
+ if (outcome === timedOut) throw new GraphQLRequestError(`GraphQL ${what} timed out after ${deadline.timeoutMs}ms`, "timeout");
7627
+ return outcome;
7628
+ }
7629
+ /** Executes a GraphQL query or mutation against the remote endpoint. */
7630
+ async executeGraphQL(query, variables, signal) {
7418
7631
  const headers = { "Content-Type": "application/json" };
7419
- const authHeader = await this.getAuthorizationHeader();
7420
- if (authHeader) headers["Authorization"] = authHeader;
7421
7632
  const operationName = query.match(/(?:query|mutation)\s+(\w+)/)?.[1] ?? "unknown";
7633
+ const timeoutMs = this.config.requestTimeoutMs ?? 6e4;
7634
+ const deadline = this.requestDeadline(timeoutMs, signal);
7635
+ let authHeader;
7636
+ try {
7637
+ authHeader = await this.withDeadline(this.getAuthorizationHeader(), deadline, `${operationName} authorization`);
7638
+ } catch (error) {
7639
+ deadline.dispose();
7640
+ throw error;
7641
+ }
7642
+ if (authHeader) headers["Authorization"] = authHeader;
7422
7643
  this.logger.verbose("GQL request @channelId @operation @url vars=@variables", this.channelId, operationName, this.config.url, JSON.stringify(variables));
7423
7644
  const fetchFn = this.config.fetchFn ?? fetch;
7424
7645
  let response;
7425
7646
  try {
7426
- response = await fetchFn(this.config.url, {
7647
+ response = await this.withDeadline(fetchFn(this.config.url, {
7427
7648
  method: "POST",
7428
7649
  headers,
7429
7650
  body: JSON.stringify({
7430
7651
  query,
7431
7652
  variables
7432
7653
  }),
7433
- signal: this.abortController.signal
7434
- });
7654
+ signal: deadline.signal
7655
+ }), deadline, `${operationName} request`);
7435
7656
  } catch (error) {
7657
+ deadline.dispose();
7658
+ if (error instanceof GraphQLRequestError) throw error;
7436
7659
  throw new GraphQLRequestError(`GraphQL request failed: ${error instanceof Error ? error.message : String(error)}`, "network");
7437
7660
  }
7438
- if (!response.ok) throw new GraphQLRequestError(`GraphQL request failed: ${response.status} ${response.statusText}`, "http", response.status);
7661
+ if (!response.ok) {
7662
+ deadline.dispose();
7663
+ throw new GraphQLRequestError(`GraphQL request failed: ${response.status} ${response.statusText}`, "http", response.status);
7664
+ }
7439
7665
  let result;
7440
7666
  try {
7441
- result = await response.json();
7667
+ result = await this.withDeadline(response.json(), deadline, `${operationName} response body`);
7442
7668
  } catch (error) {
7669
+ if (error instanceof GraphQLRequestError) throw error;
7443
7670
  throw new GraphQLRequestError(`Failed to parse GraphQL response: ${error instanceof Error ? error.message : String(error)}`, "parse");
7671
+ } finally {
7672
+ deadline.dispose();
7444
7673
  }
7445
7674
  this.logger.verbose("GQL response @channelId @operation status=@status data=@data errors=@errors", this.channelId, operationName, response.status, JSON.stringify(result.data), result.errors ? JSON.stringify(result.errors) : "none");
7446
7675
  if (result.errors) throw new GraphQLRequestError(`GraphQL errors: ${JSON.stringify(result.errors, null, 2)}`, "graphql", void 0, result.errors.map((error) => error.extensions?.code));
@@ -7524,6 +7753,12 @@ var GqlRequestChannelFactory = class {
7524
7753
  }
7525
7754
  if (retryBaseDelayMs !== void 0) gqlConfig.retryBaseDelayMs = retryBaseDelayMs;
7526
7755
  if (retryMaxDelayMs !== void 0) gqlConfig.retryMaxDelayMs = retryMaxDelayMs;
7756
+ let requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS;
7757
+ if (config.parameters.requestTimeoutMs !== void 0) {
7758
+ if (typeof config.parameters.requestTimeoutMs !== "number") throw new Error("\"requestTimeoutMs\" parameter must be a number");
7759
+ requestTimeoutMs = config.parameters.requestTimeoutMs;
7760
+ }
7761
+ gqlConfig.requestTimeoutMs = requestTimeoutMs;
7527
7762
  let maxQueueDepth;
7528
7763
  if (config.parameters.maxQueueDepth !== void 0) {
7529
7764
  if (typeof config.parameters.maxQueueDepth !== "number") throw new Error("\"maxQueueDepth\" parameter must be a number");
@@ -7540,6 +7775,7 @@ var GqlRequestChannelFactory = class {
7540
7775
  ...retryMaxDelayMs !== void 0 && { retryMaxDelayMs },
7541
7776
  ...maxQueueDepth !== void 0 && { maxQueueDepth },
7542
7777
  ...backpressureCheckIntervalMs !== void 0 && { backpressureCheckIntervalMs },
7778
+ delegateTimeoutMs: requestTimeoutMs > 0 ? Math.max(DELEGATE_TIMEOUT_FLOOR_MS, requestTimeoutMs * 6) : 0,
7543
7779
  startPaused: options?.pollBehavior === "manual"
7544
7780
  });
7545
7781
  return new GqlRequestChannel(this.logger, remoteId, remoteName, cursorStorage, gqlConfig, operationIndex, pollTimer);
@@ -7901,7 +8137,11 @@ var KyselySyncDeadLetterStorage = class {
7901
8137
  await acquirePurgeLocks(trx, [deadLetter.documentId], "shared");
7902
8138
  if ((await findPurged(trx, [deadLetter.documentId])).size > 0) throw new DocumentPurgedError(deadLetter.documentId);
7903
8139
  const insertable = deadLetterRecordToRow(deadLetter);
7904
- await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").doNothing()).execute();
8140
+ await trx.insertInto("sync_dead_letters").values(insertable).onConflict((oc) => oc.column("id").doUpdateSet((eb) => ({
8141
+ error_source: eb.ref("excluded.error_source"),
8142
+ error_message: eb.ref("excluded.error_message"),
8143
+ error_type: eb.ref("excluded.error_type")
8144
+ }))).execute();
7905
8145
  });
7906
8146
  if (signal?.aborted) throw new Error("Operation aborted");
7907
8147
  }
@@ -7919,9 +8159,11 @@ var KyselySyncDeadLetterStorage = class {
7919
8159
  });
7920
8160
  if (signal?.aborted) throw new Error("Operation aborted");
7921
8161
  }
7922
- async listQuarantinedDocumentIds(signal) {
8162
+ async listQuarantinedDocumentIds(signal, exceptIds = []) {
7923
8163
  if (signal?.aborted) throw new Error("Operation aborted");
7924
- const rows = await this.db.selectFrom("sync_dead_letters").select(["document_id", "error_type"]).distinct().execute();
8164
+ let query = this.db.selectFrom("sync_dead_letters").select(["document_id", "error_type"]).distinct();
8165
+ if (exceptIds.length > 0) query = query.where("id", "not in", [...exceptIds]);
8166
+ const rows = await query.execute();
7925
8167
  if (signal?.aborted) throw new Error("Operation aborted");
7926
8168
  const quarantined = /* @__PURE__ */ new Set();
7927
8169
  for (const row of rows) if (quarantinesDocument(row.error_type ?? "UNCLASSIFIED")) quarantined.add(row.document_id);
@@ -8672,7 +8914,7 @@ const PLAN_KEY_TO_JOB_UUID_CAP = 1e4;
8672
8914
  */
8673
8915
  function isCredentialOrNetworkError(error) {
8674
8916
  if (isDriveAuthError(error)) return true;
8675
- return error instanceof GraphQLRequestError && error.category === "network";
8917
+ return error instanceof GraphQLRequestError && (error.category === "network" || error.category === "timeout");
8676
8918
  }
8677
8919
  const holdKey = (documentId, branch) => `${documentId}\u0000${branch}`;
8678
8920
  const VERSION_CACHE_CAP = 1e4;
@@ -8719,7 +8961,10 @@ var SyncManager = class {
8719
8961
  config;
8720
8962
  connectionStateUnsubscribes = /* @__PURE__ */ new Map();
8721
8963
  quarantinedDocumentIds = /* @__PURE__ */ new Set();
8964
+ parkedByQuarantine = /* @__PURE__ */ new WeakSet();
8722
8965
  purgedDocumentIds = /* @__PURE__ */ new Set();
8966
+ requeuedDeadLetterIds = /* @__PURE__ */ new Map();
8967
+ requeuesInFlight = /* @__PURE__ */ new Set();
8723
8968
  purges;
8724
8969
  delivery;
8725
8970
  forgetDocument;
@@ -8731,6 +8976,8 @@ var SyncManager = class {
8731
8976
  derivingOutboxes = 0;
8732
8977
  pruneDrainDeferred = false;
8733
8978
  removing = /* @__PURE__ */ new Set();
8979
+ resets = /* @__PURE__ */ new Map();
8980
+ queuedResets = /* @__PURE__ */ new Map();
8734
8981
  lastEnqueuedJobIdByKey = /* @__PURE__ */ new Map();
8735
8982
  watermark;
8736
8983
  owed = /* @__PURE__ */ new Map();
@@ -9042,11 +9289,18 @@ var SyncManager = class {
9042
9289
  if (stale.length > 0) remote.channel.outbox.remove(...stale);
9043
9290
  const dead = remote.channel.deadLetter.items.filter((item) => item.documentId === documentId);
9044
9291
  if (dead.length > 0) remote.channel.deadLetter.remove(...dead);
9292
+ const parked = remote.channel.inbox.items.filter((item) => item.documentId === documentId && this.parkedByQuarantine.has(item));
9293
+ for (const item of parked) {
9294
+ this.parkedByQuarantine.delete(item);
9295
+ item.executed();
9296
+ }
9297
+ if (parked.length > 0) remote.channel.inbox.remove(...parked);
9045
9298
  }
9046
9299
  }
9047
9300
  async forgetRemote(name) {
9048
9301
  this.records.delete(name);
9049
9302
  this.heldKeys.delete(name);
9303
+ for (const [id, pending] of [...this.requeuedDeadLetterIds]) if (pending.remoteName === name) this.requeuedDeadLetterIds.delete(id);
9050
9304
  await this.holds.removeRemote(name);
9051
9305
  await Promise.allSettled(this.markerWritesOf([name]));
9052
9306
  await this.markerStorage.removeRemote(name);
@@ -9300,28 +9554,14 @@ var SyncManager = class {
9300
9554
  };
9301
9555
  this.remotes.set(name, remote);
9302
9556
  this.records.set(name, meta);
9303
- await this.loadDeadLetters(remote);
9304
- await this.restoreReceivedMarkers(remote);
9305
- const unheard = [...remote.channel.inbox.items];
9306
- this.wireChannelCallbacks(remote);
9557
+ const unheard = await this.wireRemote(remote);
9307
9558
  try {
9308
9559
  await channel.init();
9309
9560
  } catch (error) {
9310
9561
  await this.dropRemoteAfterFailedInit(remote, !isCredentialOrNetworkError(error));
9311
9562
  throw error;
9312
9563
  }
9313
- if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
9314
- await this.peerUpdates.get(name);
9315
- this.owe(name, await this.watermarkHead());
9316
- const backfillController = new AbortController();
9317
- this.backfillAbortControllers.set(name, backfillController);
9318
- this.updateOutbox(remote, 0, "backfill", backfillController.signal).catch((error) => {
9319
- if (backfillController.signal.aborted) return;
9320
- this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.meta.name, error instanceof Error ? error : new Error(String(error)));
9321
- }).finally(() => {
9322
- this.backfillAbortControllers.delete(name);
9323
- this.drainPrunes();
9324
- });
9564
+ await this.activateRemote(remote, unheard, 0);
9325
9565
  return remote;
9326
9566
  }
9327
9567
  triggerPull(name) {
@@ -9332,6 +9572,7 @@ var SyncManager = class {
9332
9572
  async remove(name) {
9333
9573
  const remote = this.remotes.get(name);
9334
9574
  if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
9575
+ if (this.resets.has(name) || this.queuedResets.has(name)) throw new Error(`Remote with name '${name}' is being reset`);
9335
9576
  this.removing.add(name);
9336
9577
  this.owed.delete(name);
9337
9578
  this.derivedThrough.delete(name);
@@ -9382,6 +9623,11 @@ var SyncManager = class {
9382
9623
  backfillController.abort();
9383
9624
  this.backfillAbortControllers.delete(name);
9384
9625
  }
9626
+ for (const [id, pending] of [...this.requeuedDeadLetterIds]) {
9627
+ if (pending.remoteName !== name) continue;
9628
+ const item = remote.channel.inbox.get(id);
9629
+ if (item && this.parkedByQuarantine.has(item)) this.requeuedDeadLetterIds.delete(id);
9630
+ }
9385
9631
  try {
9386
9632
  await remote.channel.shutdown();
9387
9633
  } finally {
@@ -9421,6 +9667,255 @@ var SyncManager = class {
9421
9667
  async receiptsStored(remoteNames) {
9422
9668
  await Promise.all(this.markerWritesOf(remoteNames));
9423
9669
  }
9670
+ /** Never joins a running rebuild: it may have read state older than the request. */
9671
+ resetChannel(remoteName) {
9672
+ const queued = this.queuedResets.get(remoteName);
9673
+ if (queued) return queued;
9674
+ const running = this.resets.get(remoteName);
9675
+ if (!running) return this.startReset(remoteName);
9676
+ const next = running.catch(() => void 0).then(() => {
9677
+ this.queuedResets.delete(remoteName);
9678
+ return this.startReset(remoteName);
9679
+ });
9680
+ this.queuedResets.set(remoteName, next);
9681
+ return next;
9682
+ }
9683
+ startReset(name) {
9684
+ const reset = this.rebuildChannel(name).finally(() => {
9685
+ this.resets.delete(name);
9686
+ });
9687
+ this.resets.set(name, reset);
9688
+ return reset;
9689
+ }
9690
+ /** On failure the remote is unregistered, record kept, and can be reset again. */
9691
+ async rebuildChannel(name) {
9692
+ if (this.isShutdown) throw new Error("SyncManager is shutdown and cannot reset remotes");
9693
+ if (this.removing.has(name)) throw new Error(`Remote with name '${name}' is being removed`);
9694
+ const live = this.remotes.get(name);
9695
+ const meta = live?.meta ?? this.records.get(name);
9696
+ if (!meta) throw new Error(`Remote with name '${name}' does not exist`);
9697
+ this.removing.add(name);
9698
+ const requeued = new Set([...this.requeuedDeadLetterIds].filter(([, pending]) => pending.remoteName === name).map(([id]) => id));
9699
+ if (live) try {
9700
+ await this.teardownRemoteResources(live);
9701
+ } catch (error) {
9702
+ this.abandonReset(name, error);
9703
+ throw error;
9704
+ }
9705
+ if (this.isShutdown) {
9706
+ const error = /* @__PURE__ */ new Error("SyncManager shut down during the reset");
9707
+ this.abandonReset(name, error);
9708
+ throw error;
9709
+ }
9710
+ let channel;
9711
+ try {
9712
+ channel = this.channelFactory.instance(meta.id, meta.name, meta.channelConfig, this.cursorStorage, meta.collectionId, meta.filter, this.operationIndex, meta.options);
9713
+ } catch (error) {
9714
+ this.abandonReset(name, error);
9715
+ throw error;
9716
+ }
9717
+ const fresh = {
9718
+ meta,
9719
+ channel
9720
+ };
9721
+ this.remotes.set(name, fresh);
9722
+ if (live) this.carryParked(live, fresh, requeued);
9723
+ let unheard;
9724
+ try {
9725
+ unheard = await this.wireRemote(fresh, true);
9726
+ } catch (error) {
9727
+ this.removing.delete(name);
9728
+ await this.failReset(fresh, error);
9729
+ throw error;
9730
+ }
9731
+ this.removing.delete(name);
9732
+ try {
9733
+ await channel.init();
9734
+ } catch (error) {
9735
+ await this.failReset(fresh, error);
9736
+ throw error;
9737
+ }
9738
+ await this.activateRemote(fresh, unheard, channel.outbox.ackOrdinal);
9739
+ }
9740
+ /** A pushing remote never resends them, so the fresh inbox parks them again. */
9741
+ carryParked(live, fresh, requeued) {
9742
+ for (const item of live.channel.inbox.items) {
9743
+ if (!this.parkedByQuarantine.has(item) || requeued.has(item.id)) continue;
9744
+ const copy = new SyncOperation(item.id, item.jobId, item.jobDependencies, item.remoteName, item.documentId, item.scopes, item.branch, item.operations);
9745
+ fresh.channel.inbox.add(copy);
9746
+ fresh.channel.inbox.hold?.(copy);
9747
+ }
9748
+ }
9749
+ /** Before a fresh channel is registered: nothing to tear down. */
9750
+ abandonReset(name, error) {
9751
+ this.logResetFailure(name, error);
9752
+ this.remotes.delete(name);
9753
+ this.removing.delete(name);
9754
+ }
9755
+ async failReset(fresh, error) {
9756
+ this.logResetFailure(fresh.meta.name, error);
9757
+ await this.dropRemoteAfterFailedInit(fresh, false);
9758
+ }
9759
+ logResetFailure(name, error) {
9760
+ this.logger.error("Resetting remote @name failed; it stays down until reset again or restarted: @error", name, error instanceof Error ? error.message : String(error));
9761
+ }
9762
+ /**
9763
+ * Restores and wires the mailboxes; returns the inbox items not yet heard.
9764
+ * `strict` fails on a storage read rather than wiring a partial channel.
9765
+ */
9766
+ async wireRemote(remote, strict = false) {
9767
+ await this.loadDeadLetters(remote, strict);
9768
+ await this.restoreReceivedMarkers(remote, strict);
9769
+ const unheard = [...remote.channel.inbox.items];
9770
+ this.wireChannelCallbacks(remote);
9771
+ return unheard;
9772
+ }
9773
+ /** After init: hands over unheard items and starts the outbox backfill. */
9774
+ async activateRemote(remote, unheard, backfillFrom) {
9775
+ const name = remote.meta.name;
9776
+ if (unheard.length > 0) this.handleInboxAdded(remote, unheard);
9777
+ await this.peerUpdates.get(name);
9778
+ this.owe(name, await this.watermarkHead());
9779
+ const backfillController = new AbortController();
9780
+ this.backfillAbortControllers.set(name, backfillController);
9781
+ this.updateOutbox(remote, backfillFrom, "backfill", backfillController.signal).catch((error) => {
9782
+ if (backfillController.signal.aborted) return;
9783
+ this.logger.error("Backfill failed for remote @RemoteName: @Error", name, error instanceof Error ? error : new Error(String(error)));
9784
+ }).finally(() => {
9785
+ this.backfillAbortControllers.delete(name);
9786
+ this.drainPrunes();
9787
+ });
9788
+ }
9789
+ /** Keeps the row until the retry succeeds, so a crash cannot lose the op. */
9790
+ async requeueDeadLetter(remoteName, id) {
9791
+ const remote = this.getByName(remoteName);
9792
+ if (this.requeuesInFlight.has(id) || this.requeuedDeadLetterIds.has(id) || remote.channel.inbox.get(id)) return;
9793
+ this.requeuesInFlight.add(id);
9794
+ try {
9795
+ await this.requeue(remote, remoteName, id);
9796
+ } finally {
9797
+ this.requeuesInFlight.delete(id);
9798
+ }
9799
+ }
9800
+ async requeue(remote, remoteName, id) {
9801
+ const source = await this.findDeadLetter(remote, id);
9802
+ if (!source) return;
9803
+ this.assertServing(remote, id);
9804
+ const errorSource = source instanceof SyncOperation ? source.error?.source : source.errorSource;
9805
+ if (errorSource !== "inbox") throw new SyncRepairRefusedError(`Cannot requeue dead letter '${id}' of '${remoteName}': it failed on the ${errorSource ?? "unknown"} side, not while applying here.`);
9806
+ const item = remote.channel.deadLetter.get(id);
9807
+ if (item) remote.channel.deadLetter.remove(item);
9808
+ this.requeuedDeadLetterIds.set(source.id, {
9809
+ remoteName: remote.meta.name,
9810
+ documentId: source.documentId
9811
+ });
9812
+ const lift = await this.mayLiftQuarantine(source.documentId);
9813
+ try {
9814
+ this.assertServing(remote, id);
9815
+ } catch (error) {
9816
+ this.requeuedDeadLetterIds.delete(source.id);
9817
+ throw error;
9818
+ }
9819
+ const requeued = new SyncOperation(source.id, source.jobId, source.jobDependencies, source.remoteName, source.documentId, source.scopes, source.branch, source.operations);
9820
+ remote.channel.inbox.add(requeued);
9821
+ if (lift) this.liftQuarantine(source.documentId);
9822
+ }
9823
+ /** A requeue into a channel torn down meanwhile would never load. */
9824
+ assertServing(remote, id) {
9825
+ const name = remote.meta.name;
9826
+ if (this.remotes.get(name) === remote && !this.removing.has(name)) return;
9827
+ throw new SyncRepairRefusedError(`Remote '${name}' was reset or removed while requeueing dead letter '${id}'; requeue it again`);
9828
+ }
9829
+ async liftQuarantineIfClear(documentId) {
9830
+ if (await this.mayLiftQuarantine(documentId)) this.liftQuarantine(documentId);
9831
+ }
9832
+ /** Whether the restart rule would lift it: no quarantining dead letter left. */
9833
+ async mayLiftQuarantine(documentId) {
9834
+ if (!this.quarantinedDocumentIds.has(documentId)) return false;
9835
+ if (this.holdsQuarantiningDeadLetter(documentId)) return false;
9836
+ let stored;
9837
+ try {
9838
+ stored = await this.storesQuarantiningDeadLetter(documentId);
9839
+ } catch (error) {
9840
+ this.logger.warn("Could not read dead letters for @documentId; its quarantine stays: @error", documentId, error instanceof Error ? error.message : String(error));
9841
+ return false;
9842
+ }
9843
+ return !stored && this.quarantinedDocumentIds.has(documentId) && !this.holdsQuarantiningDeadLetter(documentId);
9844
+ }
9845
+ holdsQuarantiningDeadLetter(documentId) {
9846
+ for (const remote of this.remotes.values()) for (const item of remote.channel.deadLetter.items) if (item.documentId === documentId && !this.requeuedDeadLetterIds.has(item.id) && quarantinesDocument(syncOperationErrorType(item.error))) return true;
9847
+ return false;
9848
+ }
9849
+ /** The restart rule, less the rows kept only for a pending requeue. */
9850
+ async storesQuarantiningDeadLetter(documentId) {
9851
+ return (await this.deadLetterStorage.listQuarantinedDocumentIds(void 0, [...this.requeuedDeadLetterIds.keys()])).includes(documentId);
9852
+ }
9853
+ /** Hands the document's parked inbox items to the apply path again. */
9854
+ liftQuarantine(documentId) {
9855
+ if (!this.quarantinedDocumentIds.delete(documentId)) return;
9856
+ for (const remote of this.remotes.values()) {
9857
+ if (this.removing.has(remote.meta.name)) continue;
9858
+ const parked = remote.channel.inbox.items.filter((item) => item.documentId === documentId && this.parkedByQuarantine.has(item)).sort((a, b) => firstOrdinalOf(a) - firstOrdinalOf(b));
9859
+ if (parked.length === 0) continue;
9860
+ for (const item of parked) this.parkedByQuarantine.delete(item);
9861
+ const rounds = [];
9862
+ const seen = /* @__PURE__ */ new Map();
9863
+ for (const item of parked) {
9864
+ const round = item.jobId ? seen.get(item.jobId) ?? 0 : 0;
9865
+ if (item.jobId) seen.set(item.jobId, round + 1);
9866
+ (rounds[round] ??= []).push(item);
9867
+ }
9868
+ for (const round of rounds) this.handleInboxAdded(remote, round);
9869
+ }
9870
+ }
9871
+ /** A failed remove leaves a row for a later clear; the op itself is durable. */
9872
+ async dropRequeuedDeadLetter(id, remoteName) {
9873
+ if (this.requeuedDeadLetterIds.get(id)?.remoteName !== remoteName) return;
9874
+ this.requeuedDeadLetterIds.delete(id);
9875
+ const current = this.remotes.get(remoteName);
9876
+ const reloaded = current?.channel.deadLetter.get(id);
9877
+ if (current && reloaded) current.channel.deadLetter.remove(reloaded);
9878
+ try {
9879
+ await this.deadLetterStorage.remove(id);
9880
+ } catch (error) {
9881
+ this.logger.error("Failed to remove a requeued dead letter after its retry succeeded (@id, @error)", id, error instanceof Error ? error.message : String(error));
9882
+ }
9883
+ }
9884
+ async clearDeadLetter(remoteName, id) {
9885
+ const remote = this.getByName(remoteName);
9886
+ const source = await this.findDeadLetter(remote, id);
9887
+ if (!source) return;
9888
+ await this.deadLetterStorage.remove(id);
9889
+ for (const holder of /* @__PURE__ */ new Set([remote, this.remotes.get(remoteName)])) {
9890
+ if (!holder) continue;
9891
+ const item = holder.channel.deadLetter.get(id);
9892
+ if (item) holder.channel.deadLetter.remove(item);
9893
+ const parked = holder.channel.inbox.get(id);
9894
+ if (parked && this.parkedByQuarantine.has(parked)) {
9895
+ this.parkedByQuarantine.delete(parked);
9896
+ parked.executed();
9897
+ holder.channel.inbox.remove(parked);
9898
+ }
9899
+ }
9900
+ this.requeuedDeadLetterIds.delete(id);
9901
+ await this.liftQuarantineIfClear(source.documentId);
9902
+ }
9903
+ /** Live item first; else scans storage for one the capped mailbox evicted. */
9904
+ async findDeadLetter(remote, id) {
9905
+ const item = remote.channel.deadLetter.get(id);
9906
+ if (item) return item;
9907
+ let cursor = "0";
9908
+ for (;;) {
9909
+ const page = await this.deadLetterStorage.list(remote.meta.name, {
9910
+ cursor,
9911
+ limit: this.config.maxDeadLettersPerRemote
9912
+ });
9913
+ const match = page.results.find((record) => record.id === id);
9914
+ if (match) return match;
9915
+ if (!page.nextCursor) return;
9916
+ cursor = page.nextCursor;
9917
+ }
9918
+ }
9424
9919
  recordPlanKeyMapping(planKey, jobId) {
9425
9920
  if (!this.planKeyToJobUuid.has(planKey) && this.planKeyToJobUuid.size >= PLAN_KEY_TO_JOB_UUID_CAP) {
9426
9921
  const oldest = this.planKeyToJobUuid.keys().next().value;
@@ -9471,6 +9966,7 @@ var SyncManager = class {
9471
9966
  if (purged.length > 0) remote.channel.deadLetter.remove(...purged);
9472
9967
  const syncOps = remaining.filter((syncOp) => !purged.includes(syncOp));
9473
9968
  for (const syncOp of syncOps) {
9969
+ this.requeuedDeadLetterIds.delete(syncOp.id);
9474
9970
  this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
9475
9971
  const errorType = syncOperationErrorType(syncOp.error);
9476
9972
  if (quarantinesDocument(errorType)) this.quarantinedDocumentIds.add(syncOp.documentId);
@@ -9511,7 +10007,7 @@ var SyncManager = class {
9511
10007
  }
9512
10008
  });
9513
10009
  }
9514
- async loadDeadLetters(remote) {
10010
+ async loadDeadLetters(remote, strict = false) {
9515
10011
  let records;
9516
10012
  try {
9517
10013
  records = (await this.deadLetterStorage.list(remote.meta.name, {
@@ -9519,6 +10015,7 @@ var SyncManager = class {
9519
10015
  limit: this.config.maxDeadLettersPerRemote
9520
10016
  })).results;
9521
10017
  } catch (error) {
10018
+ if (strict) throw error;
9522
10019
  this.logger.error("Failed to load dead letters for remote (@name, @error)", remote.meta.name, error instanceof Error ? error.message : String(error));
9523
10020
  return;
9524
10021
  }
@@ -9624,10 +10121,17 @@ var SyncManager = class {
9624
10121
  this.storeReceivedMarker(remote, syncOp);
9625
10122
  eligible.push(syncOp);
9626
10123
  }
9627
- } else if (this.purgedDocumentIds.has(syncOp.documentId)) dropped.push(syncOp);
9628
- else if (!this.quarantinedDocumentIds.has(syncOp.documentId)) eligible.push(syncOp);
10124
+ } else if (this.purgedDocumentIds.has(syncOp.documentId)) {
10125
+ dropped.push(syncOp);
10126
+ this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
10127
+ } else if (!this.quarantinedDocumentIds.has(syncOp.documentId)) eligible.push(syncOp);
10128
+ else if (!this.requeuedDeadLetterIds.has(syncOp.id) && this.parkedCopyOf(remote, syncOp)) dropped.push(syncOp);
10129
+ else {
10130
+ this.parkedByQuarantine.add(syncOp);
10131
+ remote.channel.inbox.hold?.(syncOp);
10132
+ }
9629
10133
  for (const syncOp of dropped) {
9630
- this.logger.debug("Dropping received operations of a purged or already-loading document (@remote, @documentId)", remote.meta.name, syncOp.documentId);
10134
+ this.logger.debug("Dropping received operations of a purged, already-loading or already-parked document (@remote, @documentId)", remote.meta.name, syncOp.documentId);
9631
10135
  syncOp.executed();
9632
10136
  }
9633
10137
  if (dropped.length > 0) remote.channel.inbox.remove(...dropped);
@@ -9645,6 +10149,11 @@ var SyncManager = class {
9645
10149
  this.processInboxChunks(chunks);
9646
10150
  }
9647
10151
  }
10152
+ parkedCopyOf(remote, syncOp) {
10153
+ const ops = syncOp.operations.map((op) => op.operation);
10154
+ if (ops.length === 0) return false;
10155
+ return remote.channel.inbox.items.some((item) => item !== syncOp && this.parkedByQuarantine.has(item) && item.documentId === syncOp.documentId && item.branch === syncOp.branch && item.jobId === syncOp.jobId && ops.every((op) => item.operations.some(({ operation }) => operation.id === op.id && operation.index === op.index)));
10156
+ }
9648
10157
  /** Forgets a marker once the item loading it leaves the inbox. */
9649
10158
  handleInboxRemoved(remote, syncOps) {
9650
10159
  const received = this.receivedMarkers.get(remote.meta.name);
@@ -9692,12 +10201,13 @@ var SyncManager = class {
9692
10201
  });
9693
10202
  }
9694
10203
  /** Queued before init resets latestOrdinal, so a puller is re-served above it. */
9695
- async restoreReceivedMarkers(remote) {
10204
+ async restoreReceivedMarkers(remote, strict = false) {
9696
10205
  const name = remote.meta.name;
9697
10206
  let records;
9698
10207
  try {
9699
10208
  records = await this.markerStorage.list(name);
9700
10209
  } catch (error) {
10210
+ if (strict) throw error;
9701
10211
  this.logger.error("Failed to load received markers for remote (@name, @error)", name, error instanceof Error ? error.message : String(error));
9702
10212
  return;
9703
10213
  }
@@ -9769,12 +10279,15 @@ var SyncManager = class {
9769
10279
  return;
9770
10280
  }
9771
10281
  if (this.isShutdown) return;
10282
+ let resolved = false;
9772
10283
  if (completedJobInfo.status !== "FAILED") {
9773
10284
  syncOp.executed();
9774
10285
  if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
10286
+ resolved = true;
9775
10287
  } else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
9776
10288
  this.tombstone(syncOp.documentId);
9777
10289
  syncOp.executed();
10290
+ resolved = true;
9778
10291
  } else {
9779
10292
  const errorMessage = completedJobInfo.error?.message || "Unknown error";
9780
10293
  this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
@@ -9787,6 +10300,7 @@ var SyncManager = class {
9787
10300
  }
9788
10301
  this.markerRetries.delete(syncOp.id);
9789
10302
  remote.channel.inbox.remove(syncOp);
10303
+ if (resolved) await this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
9790
10304
  }
9791
10305
  /** Reloads a marker with backoff; it stays in the inbox, not dead-lettered. */
9792
10306
  retryMarker(remote, syncOp, reason) {
@@ -9816,6 +10330,7 @@ var SyncManager = class {
9816
10330
  this.markerRetries.delete(syncOp.id);
9817
10331
  syncOp.executed();
9818
10332
  remote.channel.inbox.remove(syncOp);
10333
+ await this.dropRequeuedDeadLetter(syncOp.id, name);
9819
10334
  return;
9820
10335
  }
9821
10336
  await this.applyInboxJob(remote, syncOp);
@@ -9906,12 +10421,15 @@ var SyncManager = class {
9906
10421
  continue;
9907
10422
  }
9908
10423
  if (this.isShutdown) return;
10424
+ let resolved = false;
9909
10425
  if (completedJobInfo.status !== "FAILED") {
9910
10426
  syncOp.executed();
9911
10427
  if (carriesMarker(syncOp)) this.tombstone(syncOp.documentId);
10428
+ resolved = true;
9912
10429
  } else if (isPurgedFailure(completedJobInfo.error, syncOp.documentId)) {
9913
10430
  this.tombstone(syncOp.documentId);
9914
10431
  syncOp.executed();
10432
+ resolved = true;
9915
10433
  } else if (carriesMarker(syncOp) && !isRefusedMarker(completedJobInfo.error)) {
9916
10434
  this.retryMarker(remote, syncOp, completedJobInfo.error?.message || "Unknown error");
9917
10435
  continue;
@@ -9920,6 +10438,7 @@ var SyncManager = class {
9920
10438
  remote.channel.deadLetter.add(syncOp);
9921
10439
  }
9922
10440
  remote.channel.inbox.remove(syncOp);
10441
+ if (resolved) await this.dropRequeuedDeadLetter(syncOp.id, remote.meta.name);
9923
10442
  }
9924
10443
  }
9925
10444
  /**
@@ -10215,7 +10734,8 @@ var SyncBuilder = class {
10215
10734
  deadLetterStorage,
10216
10735
  holdStorage,
10217
10736
  channelFactory: this.channelFactory,
10218
- syncManager
10737
+ syncManager,
10738
+ syncAdmin: syncManager
10219
10739
  };
10220
10740
  }
10221
10741
  };
@@ -10224,8 +10744,8 @@ var SyncBuilder = class {
10224
10744
  async function createDefaultDatabase() {
10225
10745
  const { Kysely } = await import("kysely");
10226
10746
  const { PGlite } = await import("@electric-sql/pglite");
10227
- const { PGliteDialect } = await import("kysely-pglite-dialect");
10228
- return new Kysely({ dialect: new PGliteDialect(new PGlite()) });
10747
+ const { HardenedPGliteDialect } = await Promise.resolve().then(() => pglite_dialect_exports);
10748
+ return new Kysely({ dialect: new HardenedPGliteDialect(new PGlite()) });
10229
10749
  }
10230
10750
  //#endregion
10231
10751
  //#region src/shared/factories.ts
@@ -11016,6 +11536,7 @@ var ReactorBuilder = class {
11016
11536
  this.unsupportedStoredDocuments = mode;
11017
11537
  return this;
11018
11538
  }
11539
+ /** Mutually exclusive with {@link withChannelScheme}; setting both fails the build. */
11019
11540
  withSync(syncBuilder) {
11020
11541
  this.syncBuilder = syncBuilder;
11021
11542
  return this;
@@ -11079,6 +11600,7 @@ var ReactorBuilder = class {
11079
11600
  this.queueInstance = queue;
11080
11601
  return this;
11081
11602
  }
11603
+ /** Mutually exclusive with {@link withSync}; setting both fails the build. */
11082
11604
  withChannelScheme(scheme) {
11083
11605
  this.channelScheme = scheme;
11084
11606
  return this;
@@ -11164,6 +11686,7 @@ var ReactorBuilder = class {
11164
11686
  ...this.executorConfig,
11165
11687
  protocolSupport: localSupports(this.getPeerCapabilities(), featureFlags).protocols
11166
11688
  };
11689
+ if (this.channelScheme !== void 0 && this.syncBuilder !== void 0) throw new Error("withChannelScheme and withSync are mutually exclusive; use one (the scheme builds its own SyncBuilder)");
11167
11690
  if (this.readModelCoordinator !== void 0 && this.readModelCoordinatorFactory !== void 0) throw new Error("withReadModelCoordinator and withReadModelCoordinatorFactory are mutually exclusive; register one coordinator source");
11168
11691
  if (this.projectionShardConfig !== void 0 && this.readModelCoordinatorFactory !== void 0) throw new Error("withProjectionShards and withReadModelCoordinatorFactory are mutually exclusive; the factory owns the projection worker through its createProjectionShardManager dependency");
11169
11692
  if (this.projectionShardConfig !== void 0 && this.readModelFactories.length > 0) throw new Error("withProjectionShards does not support read models registered through withReadModelFactory; projection workers cannot receive host-only factory dependencies");
@@ -11453,7 +11976,7 @@ var ReactorBuilder = class {
11453
11976
  poolInstrumentations.push(forwarder);
11454
11977
  this.instrumentedPools.push(forwarder);
11455
11978
  }
11456
- const { ProjectionShardManager } = await import("./projection-shard-manager-MRJ9BjVM.js");
11979
+ const { ProjectionShardManager } = await import("./projection-shard-manager-CQ5FwQmv.js");
11457
11980
  const manager = new ProjectionShardManager({
11458
11981
  shardCount: config.shardCount,
11459
11982
  db,
@@ -11498,7 +12021,7 @@ var ReactorBuilder = class {
11498
12021
  */
11499
12022
  async createDefaultWorkerFactory(numWorkers, db) {
11500
12023
  const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
11501
- import("./worker-handle-B48V1hDm.js"),
12024
+ import("./worker-handle-CN2-oRv-.js"),
11502
12025
  import("./transport-CZDHzfvy.js"),
11503
12026
  Promise.resolve().then(() => worker_exports)
11504
12027
  ]);
@@ -12095,6 +12618,391 @@ function firstOutOfOrderPair(operations, options) {
12095
12618
  }
12096
12619
  }
12097
12620
  //#endregion
12621
+ //#region src/storage/kysely/pglite-dialect.ts
12622
+ var pglite_dialect_exports = /* @__PURE__ */ __exportAll({
12623
+ DEFAULT_LONG_STATEMENT_TIMEOUT_MS: () => DEFAULT_LONG_STATEMENT_TIMEOUT_MS,
12624
+ DEFAULT_RECOVERY_TIMEOUT_MS: () => DEFAULT_RECOVERY_TIMEOUT_MS,
12625
+ DEFAULT_STATEMENT_TIMEOUT_MS: () => DEFAULT_STATEMENT_TIMEOUT_MS,
12626
+ HardenedPGliteDialect: () => HardenedPGliteDialect,
12627
+ PGliteAbortedTransactionError: () => PGliteAbortedTransactionError,
12628
+ PGliteAcquireTimeoutError: () => PGliteAcquireTimeoutError,
12629
+ PGliteSessionError: () => PGliteSessionError,
12630
+ PGliteSessionPoisonedError: () => PGliteSessionPoisonedError,
12631
+ PGliteStatementTimeoutError: () => PGliteStatementTimeoutError,
12632
+ isLongRunningStatement: () => isLongRunningStatement,
12633
+ queryThroughDialect: () => queryThroughDialect
12634
+ });
12635
+ const DEFAULT_STATEMENT_TIMEOUT_MS = 12e4;
12636
+ const DEFAULT_LONG_STATEMENT_TIMEOUT_MS = 9e5;
12637
+ const DEFAULT_RECOVERY_TIMEOUT_MS = 15e3;
12638
+ const LONG_STATEMENT_PREFIXES = [
12639
+ "vacuum",
12640
+ "analyze",
12641
+ "reindex",
12642
+ "cluster",
12643
+ "checkpoint",
12644
+ "copy",
12645
+ "alter ",
12646
+ "create ",
12647
+ "drop ",
12648
+ "truncate",
12649
+ "refresh "
12650
+ ];
12651
+ /** Maintenance and DDL, matched on the leading keyword of the compiled SQL. */
12652
+ function isLongRunningStatement(statement) {
12653
+ const normalized = statement.trimStart().toLowerCase();
12654
+ return LONG_STATEMENT_PREFIXES.some((prefix) => normalized.startsWith(prefix));
12655
+ }
12656
+ function summarize(statement) {
12657
+ const collapsed = statement.replace(/\s+/g, " ").trim();
12658
+ return collapsed.length > 160 ? `${collapsed.slice(0, 160)}...` : collapsed;
12659
+ }
12660
+ /** COMMIT in an aborted transaction silently rolls back; the guard raises first. */
12661
+ const COMMIT_WITH_GUARD = "select 1 as __commit_guard; commit";
12662
+ const PROBE = "select 1 as __session_probe";
12663
+ const ABORTED_TRANSACTION_CODE = "25P02";
12664
+ var PGliteSessionError = class extends Error {
12665
+ cause;
12666
+ constructor(message, cause) {
12667
+ super(message);
12668
+ this.cause = cause;
12669
+ this.name = "PGliteSessionError";
12670
+ }
12671
+ };
12672
+ var PGliteAcquireTimeoutError = class extends PGliteSessionError {
12673
+ constructor(timeoutMs) {
12674
+ super(`Timed out after ${timeoutMs}ms waiting for the PGlite connection. The single session is held by a statement that never completed: a transaction awaiting a nested query on the base handle, or an abandoned stream iterator.`);
12675
+ this.name = "PGliteAcquireTimeoutError";
12676
+ }
12677
+ };
12678
+ /** The call cannot be aborted, so it is abandoned and the session treated as poisoned. */
12679
+ var PGliteStatementTimeoutError = class extends PGliteSessionError {
12680
+ constructor(timeoutMs, statement) {
12681
+ super(`A PGlite statement neither resolved nor rejected within ${timeoutMs}ms, so its wasm call is presumed dead and the session poisoned: ${summarize(statement)}`);
12682
+ this.name = "PGliteStatementTimeoutError";
12683
+ }
12684
+ };
12685
+ var PGliteAbortedTransactionError = class extends PGliteSessionError {
12686
+ constructor(cause) {
12687
+ super(`Refusing to report a commit for an aborted transaction: COMMIT would have been answered with a ROLLBACK tag and nothing written. Underlying error: ${errorOf(cause).message}`, cause);
12688
+ this.name = "PGliteAbortedTransactionError";
12689
+ }
12690
+ };
12691
+ /** No statement can clear the session (e.g. a stuck active portal). */
12692
+ var PGliteSessionPoisonedError = class extends PGliteSessionError {
12693
+ constructor(cause) {
12694
+ super(`The PGlite session is unrecoverable and cannot be reused; the component owning it must be restarted. Original error: ${errorOf(cause).message}`, cause);
12695
+ this.name = "PGliteSessionPoisonedError";
12696
+ }
12697
+ };
12698
+ function errorOf(error) {
12699
+ return error instanceof Error ? error : new Error(String(error));
12700
+ }
12701
+ const TIMED_OUT = Symbol("pglite-deadline-expired");
12702
+ /** Races a call that cannot be cancelled; on {@link TIMED_OUT} the call may still settle later. */
12703
+ async function withDeadline(pending, timeoutMs) {
12704
+ let handle;
12705
+ const expiry = new Promise((resolve) => {
12706
+ handle = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
12707
+ });
12708
+ try {
12709
+ return await Promise.race([pending, expiry]);
12710
+ } finally {
12711
+ clearTimeout(handle);
12712
+ }
12713
+ }
12714
+ function isAbortedTransactionError(error) {
12715
+ if (error?.code === ABORTED_TRANSACTION_CODE) return true;
12716
+ return errorOf(error).message.includes("current transaction is aborted");
12717
+ }
12718
+ /** `kysely-pglite-dialect` hardened for one shared session that a fault must not brick. */
12719
+ var HardenedPGliteDialect = class {
12720
+ client;
12721
+ inner;
12722
+ options;
12723
+ constructor(client, options = {}) {
12724
+ this.client = client;
12725
+ this.inner = new PGliteDialect(client);
12726
+ this.options = {
12727
+ acquireTimeoutMs: options.acquireTimeoutMs ?? 0,
12728
+ statementTimeoutMs: options.statementTimeoutMs ?? 12e4,
12729
+ longStatementTimeoutMs: options.longStatementTimeoutMs ?? 9e5,
12730
+ recoveryTimeoutMs: options.recoveryTimeoutMs ?? 15e3,
12731
+ onDiagnostic: options.onDiagnostic ?? ((message, error) => {
12732
+ console.error(`[pglite-dialect] ${message}`, error);
12733
+ }),
12734
+ onPoisoned: options.onPoisoned ?? (() => void 0)
12735
+ };
12736
+ }
12737
+ createAdapter() {
12738
+ return this.inner.createAdapter();
12739
+ }
12740
+ createDriver() {
12741
+ return new HardenedPGliteDriver(this.inner.createDriver(), this.client, this.options);
12742
+ }
12743
+ createQueryCompiler() {
12744
+ return this.inner.createQueryCompiler();
12745
+ }
12746
+ createIntrospector(db) {
12747
+ return this.inner.createIntrospector(db);
12748
+ }
12749
+ };
12750
+ var HardenedPGliteConnection = class {
12751
+ inner;
12752
+ guard;
12753
+ failure = void 0;
12754
+ rollbackFailure = void 0;
12755
+ transactionOpen = false;
12756
+ constructor(inner, guard) {
12757
+ this.inner = inner;
12758
+ this.guard = guard;
12759
+ }
12760
+ /** Never retried: a replay could commit alone a write meant for another consumer's transaction. */
12761
+ async executeQuery(compiledQuery) {
12762
+ const generation = this.guard.generation;
12763
+ try {
12764
+ return await this.guard.runStatement(compiledQuery.sql, () => this.inner.executeQuery(compiledQuery));
12765
+ } catch (error) {
12766
+ this.guard.recordFailure(this, generation, error);
12767
+ throw error;
12768
+ }
12769
+ }
12770
+ streamQuery(compiledQuery, chunkSize) {
12771
+ const inner = this.inner.streamQuery(compiledQuery, chunkSize);
12772
+ return this.deadlinedStream(compiledQuery, inner);
12773
+ }
12774
+ /** Upstream issues the whole query on the first pull, so each pull is bounded like a statement. */
12775
+ async *deadlinedStream(compiledQuery, inner) {
12776
+ for (;;) {
12777
+ const generation = this.guard.generation;
12778
+ let next;
12779
+ try {
12780
+ next = await this.guard.runStatement(compiledQuery.sql, () => inner.next());
12781
+ } catch (error) {
12782
+ this.guard.recordFailure(this, generation, error);
12783
+ throw error;
12784
+ }
12785
+ if (next.done === true) return;
12786
+ yield next.value;
12787
+ }
12788
+ }
12789
+ get suspect() {
12790
+ return this.failure !== void 0 || this.rollbackFailure !== void 0 || this.transactionOpen;
12791
+ }
12792
+ };
12793
+ var HardenedPGliteDriver = class {
12794
+ inner;
12795
+ client;
12796
+ options;
12797
+ /** Set when release could not reset the session; each acquire retries the reset. */
12798
+ sessionFault = void 0;
12799
+ /** Lets an abandoned call's late settlement be told apart from current failures. */
12800
+ statementGeneration = 0;
12801
+ /** A call abandoned at its deadline; PGlite would queue every later call behind it, so none is issued. */
12802
+ deadCall = void 0;
12803
+ poisonReported = false;
12804
+ constructor(inner, client, options) {
12805
+ this.inner = inner;
12806
+ this.client = client;
12807
+ this.options = options;
12808
+ }
12809
+ get generation() {
12810
+ return this.statementGeneration;
12811
+ }
12812
+ /** Bounds one statement; an expiry is escalated as a poisoned session. */
12813
+ async runStatement(statement, execute) {
12814
+ if (this.deadCall !== void 0) throw new PGliteSessionPoisonedError(this.deadCall);
12815
+ const timeoutMs = this.timeoutFor(statement);
12816
+ if (timeoutMs <= 0) return execute();
12817
+ const generation = this.statementGeneration;
12818
+ const outcome = await withDeadline(execute(), timeoutMs);
12819
+ if (outcome !== TIMED_OUT) return outcome;
12820
+ throw this.escalateHungStatement(new PGliteStatementTimeoutError(timeoutMs, statement), generation);
12821
+ }
12822
+ recordFailure(connection, generation, error) {
12823
+ if (generation === this.statementGeneration) connection.failure = errorOf(error);
12824
+ }
12825
+ async init() {
12826
+ await this.inner.init();
12827
+ }
12828
+ async acquireConnection() {
12829
+ const innerConnection = await this.acquireWithTimeout();
12830
+ if (this.deadCall !== void 0) {
12831
+ await this.inner.releaseConnection(innerConnection);
12832
+ throw new PGliteSessionPoisonedError(this.deadCall);
12833
+ }
12834
+ if (this.sessionFault !== void 0) {
12835
+ const fault = this.sessionFault;
12836
+ let recovered = false;
12837
+ try {
12838
+ recovered = await this.tryRecoverSession(false);
12839
+ } finally {
12840
+ if (!recovered) await this.inner.releaseConnection(innerConnection);
12841
+ }
12842
+ if (!recovered) {
12843
+ this.reportPoisoned(fault);
12844
+ throw new PGliteSessionPoisonedError(fault);
12845
+ }
12846
+ this.sessionFault = void 0;
12847
+ }
12848
+ return new HardenedPGliteConnection(innerConnection, this);
12849
+ }
12850
+ /** Kysely never rolls back after a failed BEGIN, so an aborted session is reset here. */
12851
+ async beginTransaction(connection, settings) {
12852
+ const wrapper = asWrapper(connection);
12853
+ const generation = this.statementGeneration;
12854
+ try {
12855
+ await this.runStatement("begin", () => this.inner.beginTransaction(wrapper.inner, settings));
12856
+ } catch (error) {
12857
+ this.recordFailure(wrapper, generation, error);
12858
+ if (!isAbortedTransactionError(error)) throw error;
12859
+ if (!await this.tryRecoverSession(false)) {
12860
+ this.sessionFault = errorOf(error);
12861
+ throw error;
12862
+ }
12863
+ const retryGeneration = this.statementGeneration;
12864
+ try {
12865
+ await this.runStatement("begin", () => this.inner.beginTransaction(wrapper.inner, settings));
12866
+ } catch (retryError) {
12867
+ this.recordFailure(wrapper, retryGeneration, retryError);
12868
+ throw retryError;
12869
+ }
12870
+ wrapper.failure = void 0;
12871
+ }
12872
+ wrapper.transactionOpen = true;
12873
+ }
12874
+ async commitTransaction(connection) {
12875
+ const wrapper = asWrapper(connection);
12876
+ const generation = this.statementGeneration;
12877
+ try {
12878
+ await this.runStatement(COMMIT_WITH_GUARD, () => this.client.exec(COMMIT_WITH_GUARD));
12879
+ } catch (error) {
12880
+ this.recordFailure(wrapper, generation, error);
12881
+ if (isAbortedTransactionError(error)) throw new PGliteAbortedTransactionError(error);
12882
+ throw error;
12883
+ }
12884
+ wrapper.transactionOpen = false;
12885
+ wrapper.failure = void 0;
12886
+ }
12887
+ /** Swallowed so the original failure propagates; release resets the session. */
12888
+ async rollbackTransaction(connection) {
12889
+ const wrapper = asWrapper(connection);
12890
+ const generation = this.statementGeneration;
12891
+ try {
12892
+ await this.runStatement("rollback", () => this.inner.rollbackTransaction(wrapper.inner));
12893
+ wrapper.transactionOpen = false;
12894
+ } catch (error) {
12895
+ if (generation === this.statementGeneration) wrapper.rollbackFailure = errorOf(error);
12896
+ this.options.onDiagnostic("rollback failed; preserving the original failure and recovering the session on release", error);
12897
+ }
12898
+ }
12899
+ async releaseConnection(connection) {
12900
+ const wrapper = asWrapper(connection);
12901
+ try {
12902
+ if (wrapper.suspect && this.deadCall === void 0) {
12903
+ if (!await this.tryRecoverSession(wrapper.transactionOpen)) {
12904
+ this.sessionFault = wrapper.rollbackFailure ?? wrapper.failure ?? /* @__PURE__ */ new Error("session left in a transaction");
12905
+ this.options.onDiagnostic("the PGlite session could not be reset and is being marked unrecoverable", this.sessionFault);
12906
+ }
12907
+ }
12908
+ } finally {
12909
+ await this.inner.releaseConnection(wrapper.inner);
12910
+ }
12911
+ }
12912
+ async destroy() {
12913
+ await this.inner.destroy();
12914
+ }
12915
+ reportPoisoned(cause) {
12916
+ if (this.poisonReported) return;
12917
+ this.poisonReported = true;
12918
+ try {
12919
+ this.options.onPoisoned(cause);
12920
+ } catch (error) {
12921
+ this.options.onDiagnostic("onPoisoned threw", error);
12922
+ }
12923
+ }
12924
+ timeoutFor(statement) {
12925
+ return isLongRunningStatement(statement) ? this.options.longStatementTimeoutMs : this.options.statementTimeoutMs;
12926
+ }
12927
+ /** Marks the session faulted once per generation; a stale expiry reports only itself. */
12928
+ escalateHungStatement(cause, generation) {
12929
+ if (generation !== this.statementGeneration) return cause;
12930
+ this.statementGeneration += 1;
12931
+ this.deadCall = cause;
12932
+ this.reportPoisoned(cause);
12933
+ this.options.onDiagnostic("a PGlite statement never settled within its deadline; treating the session as poisoned", cause);
12934
+ return new PGliteSessionPoisonedError(cause);
12935
+ }
12936
+ /** A waiter that gave up still passes the lease on, so it cannot wedge the queue. */
12937
+ async acquireWithTimeout() {
12938
+ const timeoutMs = this.options.acquireTimeoutMs;
12939
+ if (timeoutMs <= 0) return this.inner.acquireConnection();
12940
+ let abandoned = false;
12941
+ const pending = this.inner.acquireConnection().then((connection) => {
12942
+ if (!abandoned) return connection;
12943
+ this.inner.releaseConnection(connection).catch(() => void 0);
12944
+ throw new PGliteAcquireTimeoutError(timeoutMs);
12945
+ });
12946
+ let handle;
12947
+ const expiry = new Promise((_resolve, reject) => {
12948
+ handle = setTimeout(() => {
12949
+ abandoned = true;
12950
+ reject(new PGliteAcquireTimeoutError(timeoutMs));
12951
+ }, timeoutMs);
12952
+ });
12953
+ try {
12954
+ return await Promise.race([pending, expiry]);
12955
+ } finally {
12956
+ clearTimeout(handle);
12957
+ }
12958
+ }
12959
+ /** {@link recoverSession}, with a recovery that throws counted as failed. */
12960
+ async tryRecoverSession(mustEndTransaction) {
12961
+ try {
12962
+ return await this.recoverSession(mustEndTransaction);
12963
+ } catch (error) {
12964
+ this.options.onDiagnostic("session recovery threw", error);
12965
+ return false;
12966
+ }
12967
+ }
12968
+ /** Rolls back only when told to or when the probe fails, sparing another consumer's live transaction. */
12969
+ async recoverSession(mustEndTransaction) {
12970
+ if (!mustEndTransaction && !this.client.isInTransaction() && await this.probeSession()) return true;
12971
+ if (!await this.recoveryExec("rollback")) this.options.onDiagnostic("session rollback did not complete; the session stays suspect");
12972
+ if (!await this.probeSession()) return false;
12973
+ return !this.client.isInTransaction();
12974
+ }
12975
+ async probeSession() {
12976
+ return this.recoveryExec(PROBE);
12977
+ }
12978
+ /** Kysely awaits release, so an unbounded recovery statement would hold the lease forever. */
12979
+ async recoveryExec(statement) {
12980
+ if (this.deadCall !== void 0) return false;
12981
+ const timeoutMs = this.options.recoveryTimeoutMs;
12982
+ const pending = this.client.exec(statement).then(() => true, (error) => {
12983
+ this.options.onDiagnostic("recovery statement failed", error);
12984
+ return false;
12985
+ });
12986
+ if (timeoutMs <= 0) return pending;
12987
+ const outcome = await withDeadline(pending, timeoutMs);
12988
+ if (outcome === TIMED_OUT) {
12989
+ this.deadCall ??= new PGliteStatementTimeoutError(timeoutMs, statement);
12990
+ this.reportPoisoned(this.deadCall);
12991
+ this.options.onDiagnostic(`a PGlite recovery statement did not settle within ${timeoutMs}ms; the session is unrecoverable from SQL`);
12992
+ return false;
12993
+ }
12994
+ return outcome;
12995
+ }
12996
+ };
12997
+ function asWrapper(connection) {
12998
+ if (!(connection instanceof HardenedPGliteConnection)) throw new PGliteSessionError("Connection was not created by HardenedPGliteDialect");
12999
+ return connection;
13000
+ }
13001
+ /** Raw SQL through the dialect's queue, instead of into whatever transaction holds the session. */
13002
+ async function queryThroughDialect(db, sql, params) {
13003
+ return [...(await db.executeQuery(CompiledQuery.raw(sql, params ?? []))).rows];
13004
+ }
13005
+ //#endregion
12098
13006
  //#region src/read-models/interfaces.ts
12099
13007
  function supportsLiveReadModelRegistration(coordinator) {
12100
13008
  return "addReadModel" in coordinator && typeof coordinator.addReadModel === "function";
@@ -12248,6 +13156,6 @@ var DocumentIntegrityService = class {
12248
13156
  }
12249
13157
  };
12250
13158
  //#endregion
12251
- export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, BatchJobFailedError, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelNotWorkerImportableError, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isPurgeMarker, isRecoverableGraphQLError, listPurged, loadDocumentModelSpec, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
13159
+ export { ALWAYS_READABLE_SCOPES, APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, AuthEnforcementDisabledError, BareReadGate, BaseReadModel, BatchJobFailedError, CatchUpScheduler, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, ContiguousCursor, DEFAULT_COMMIT_CHUNK_SIZE, DEFAULT_DRIVE_CONTAINER_TYPES, DEFAULT_LONG_STATEMENT_TIMEOUT_MS, DEFAULT_MAX_PURGE_OPERATIONS, DEFAULT_PURGE_DELETE_BATCH, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, DEFAULT_RECOVERY_TIMEOUT_MS, DEFAULT_STATEMENT_TIMEOUT_MS, DOCUMENT_INDEXER_READ_MODEL, DOCUMENT_VIEW_READ_MODEL, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DeletedDocumentRead, DocumentAlreadyExistsError, DocumentChangeType, DocumentExistence, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DocumentNotDeletedError, DocumentPurgeService, DocumentPurgedError, DriveClient, DriveCollectionId, DriveRequestError, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, GroupInUseError, HardenedPGliteDialect, HybridProjectionCoordinator, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, InMemorySyncHoldStorage, InMemorySyncPurgeRefusalStorage, InMemorySyncReceivedMarkerStorage, IntervalPollTimer, InvalidModuleError, InvalidSignatureError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentPurger, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncHoldStorage, KyselySyncPurgeRefusalStorage, KyselySyncReceivedMarkerStorage, KyselySyncRemoteStorage, KyselyWriteCache, MAX_POLLED_REFUSALS, Mailbox, ModelNotWorkerImportableError, ModelReadGate, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PGliteAbortedTransactionError, PGliteAcquireTimeoutError, PGliteSessionError, PGliteSessionPoisonedError, PGliteStatementTimeoutError, PURGE_DOCUMENT, PURGE_LOCK_BUCKETS, PURGE_NS, PollBehavior, PollingChannelError, ProbeSettler, ProcessorManager, PropagationMode, PurgeTooLargeError, QueueEventTypes, REACTOR_SCHEMA, RECOVERABLE_GRAPHQL_ERROR_CODES, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RelationshipNotFoundError, ReservedActionError, RetryAccounting, RevisionMismatchError, SIGNATURE_REFUSAL_CODES, SeededStateReader, SettledWatermark, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncRepairRefusedError, SyncScopeGate, SyncStatus, SyncStatusTracker, UnsupportedStoredProtocolError, acquirePurgeLocks, addRelationshipAction, appliedDelete, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createHybridProjectionCoordinatorFactory, createKyselyWatermarkProbe, createMutableShutdownStatus, createPeerAgreement, createReactorHostModuleBase, createRelationalDb, decideAtHead, defaultCatchUpConfig, defaultReadModelIndexingConfig, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, findPurged, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isCatchUpConsumer, isDriveAuthError, isLongRunningStatement, isPurgeMarker, isRecoverableGraphQLError, listPurged, loadDocumentModelSpec, makeConsistencyKey, parseDriveUrl, parsePagingOptions, parseSnapshot, quarantinesDocument, queryThroughDialect, readDecisionModel, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, snapshotFunctionsFor, supportsDeliveryTracking, supportsLiveReadModelRegistration, supportsPurgeRefusals, syncOperationErrorType, trimMailboxFromAckOrdinal, unchunkedReadModelIndexingConfig, updateRelationshipAction, upgradeDocumentAction, verifyActionSignature, workerEntryPath };
12252
13160
 
12253
13161
  //# sourceMappingURL=index.js.map