@indigoai-us/hq-cloud 6.16.87-alpha.0 → 6.16.89-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/bin/sync-runner-company.d.ts +2 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +8 -2
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-watch-loop.d.ts +26 -25
  6. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.js +251 -183
  8. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  9. package/dist/bin/sync-runner.d.ts +22 -2
  10. package/dist/bin/sync-runner.d.ts.map +1 -1
  11. package/dist/bin/sync-runner.js +5 -3
  12. package/dist/bin/sync-runner.js.map +1 -1
  13. package/dist/bin/sync-runner.test.js +74 -509
  14. package/dist/bin/sync-runner.test.js.map +1 -1
  15. package/dist/cli/share.d.ts +18 -6
  16. package/dist/cli/share.d.ts.map +1 -1
  17. package/dist/cli/share.js +108 -97
  18. package/dist/cli/share.js.map +1 -1
  19. package/dist/cli/share.test.js +4 -177
  20. package/dist/cli/share.test.js.map +1 -1
  21. package/dist/cli/sync-scope.test.js +0 -34
  22. package/dist/cli/sync-scope.test.js.map +1 -1
  23. package/dist/cli/sync.d.ts +17 -4
  24. package/dist/cli/sync.d.ts.map +1 -1
  25. package/dist/cli/sync.js +138 -20
  26. package/dist/cli/sync.js.map +1 -1
  27. package/dist/cli/sync.test.js +230 -1
  28. package/dist/cli/sync.test.js.map +1 -1
  29. package/dist/lib/machine-id.d.ts +0 -7
  30. package/dist/lib/machine-id.d.ts.map +1 -1
  31. package/dist/lib/machine-id.js +7 -54
  32. package/dist/lib/machine-id.js.map +1 -1
  33. package/dist/lib/machine-id.test.js +1 -20
  34. package/dist/lib/machine-id.test.js.map +1 -1
  35. package/dist/object-io.d.ts +21 -9
  36. package/dist/object-io.d.ts.map +1 -1
  37. package/dist/object-io.js +133 -93
  38. package/dist/object-io.js.map +1 -1
  39. package/dist/object-io.test.js +140 -108
  40. package/dist/object-io.test.js.map +1 -1
  41. package/dist/prefix-coalesce.d.ts +30 -37
  42. package/dist/prefix-coalesce.d.ts.map +1 -1
  43. package/dist/prefix-coalesce.js +16 -104
  44. package/dist/prefix-coalesce.js.map +1 -1
  45. package/dist/prefix-coalesce.test.js +3 -34
  46. package/dist/prefix-coalesce.test.js.map +1 -1
  47. package/dist/remote-pull.d.ts.map +1 -1
  48. package/dist/remote-pull.js +8 -15
  49. package/dist/remote-pull.js.map +1 -1
  50. package/dist/remote-pull.test.js +4 -44
  51. package/dist/remote-pull.test.js.map +1 -1
  52. package/dist/s3.d.ts +8 -3
  53. package/dist/s3.d.ts.map +1 -1
  54. package/dist/s3.js +30 -7
  55. package/dist/s3.js.map +1 -1
  56. package/dist/scope-shrink.d.ts.map +1 -1
  57. package/dist/scope-shrink.js +1 -7
  58. package/dist/scope-shrink.js.map +1 -1
  59. package/dist/sync/event-sync.d.ts +5 -19
  60. package/dist/sync/event-sync.d.ts.map +1 -1
  61. package/dist/sync/event-sync.js +14 -150
  62. package/dist/sync/event-sync.js.map +1 -1
  63. package/dist/sync/event-sync.test.js +2 -386
  64. package/dist/sync/event-sync.test.js.map +1 -1
  65. package/dist/sync/inbound-event-admission.d.ts.map +1 -1
  66. package/dist/sync/inbound-event-admission.js +3 -0
  67. package/dist/sync/inbound-event-admission.js.map +1 -1
  68. package/dist/sync/inbound-event-admission.test.js +6 -0
  69. package/dist/sync/inbound-event-admission.test.js.map +1 -1
  70. package/dist/sync/parallel-sync-coordinator.d.ts +0 -1
  71. package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
  72. package/dist/sync/parallel-sync-coordinator.js +8 -39
  73. package/dist/sync/parallel-sync-coordinator.js.map +1 -1
  74. package/dist/sync/parallel-sync-coordinator.test.js +1 -33
  75. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  76. package/dist/sync/pull-scope.d.ts +0 -2
  77. package/dist/sync/pull-scope.d.ts.map +1 -1
  78. package/dist/sync/pull-scope.js +5 -15
  79. package/dist/sync/pull-scope.js.map +1 -1
  80. package/dist/sync/push-receiver.d.ts +1 -22
  81. package/dist/sync/push-receiver.d.ts.map +1 -1
  82. package/dist/sync/push-receiver.js +20 -114
  83. package/dist/sync/push-receiver.js.map +1 -1
  84. package/dist/sync/push-receiver.test.js +24 -398
  85. package/dist/sync/push-receiver.test.js.map +1 -1
  86. package/dist/sync/push-transport.d.ts +0 -15
  87. package/dist/sync/push-transport.d.ts.map +1 -1
  88. package/dist/sync/push-transport.js +0 -27
  89. package/dist/sync/push-transport.js.map +1 -1
  90. package/dist/sync/push-transport.test.js +1 -14
  91. package/dist/sync/push-transport.test.js.map +1 -1
  92. package/dist/sync/sync-work.d.ts +0 -5
  93. package/dist/sync/sync-work.d.ts.map +1 -1
  94. package/dist/sync/sync-work.js.map +1 -1
  95. package/dist/sync-core.d.ts +2 -1
  96. package/dist/sync-core.d.ts.map +1 -1
  97. package/dist/sync-core.js +5 -10
  98. package/dist/sync-core.js.map +1 -1
  99. package/dist/sync-core.test.js +16 -0
  100. package/dist/sync-core.test.js.map +1 -1
  101. package/dist/watcher.d.ts +0 -4
  102. package/dist/watcher.d.ts.map +1 -1
  103. package/dist/watcher.js +4 -16
  104. package/dist/watcher.js.map +1 -1
  105. package/dist/watcher.test.js +3 -50
  106. package/dist/watcher.test.js.map +1 -1
  107. package/package.json +1 -1
@@ -11,13 +11,13 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
11
11
  import { describeError } from "../lib/describe-error.js";
12
12
  import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
13
13
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
14
- import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
14
+ import { getOrCreateMachineId } from "../lib/machine-id.js";
15
15
  import { localPathForVaultKey } from "../local-path-codec.js";
16
16
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
17
17
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
18
18
  import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
19
19
  import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
20
- import { NoopPushReceiver, ReceiverAdmissionRefusedError, } from "../sync/push-receiver.js";
20
+ import { NoopPushReceiver, } from "../sync/push-receiver.js";
21
21
  import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
22
22
  import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
23
23
  import { resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
@@ -85,13 +85,6 @@ export class RealtimeDrainBurstError extends Error {
85
85
  this.name = "RealtimeDrainBurst";
86
86
  }
87
87
  }
88
- function receiverAdmissionRefusedError(error) {
89
- if (error instanceof RealtimeAdmissionTimeoutError ||
90
- error instanceof RealtimeDrainBurstError) {
91
- return new ReceiverAdmissionRefusedError(error.message, { cause: error });
92
- }
93
- return null;
94
- }
95
88
  /**
96
89
  * Coordinator path status for one sync-pass exit. Exit 19 (vanished company)
97
90
  * is durable so a leftover local slug is not restored and retried forever;
@@ -160,6 +153,19 @@ export class CooperativePassScheduler {
160
153
  activeFastRealtime = null;
161
154
  stopped = false;
162
155
  _handoffActive = false;
156
+ /** A chore's payload transfer is in flight while it lends the root lock. */
157
+ networkTransferActive = false;
158
+ detachedChore = null;
159
+ networkTransferClosing = false;
160
+ networkTransferCount = 0;
161
+ networkTransferEpoch = 0;
162
+ networkTransferWaiters = [];
163
+ networkTransferRelease = null;
164
+ resolveNetworkTransferRelease = null;
165
+ /** Successful callers still applying results after the network hand-off. */
166
+ networkTransferCompletionCount = 0;
167
+ networkTransferApplyRelease = null;
168
+ resolveNetworkTransferApplyRelease = null;
163
169
  constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
164
170
  this.now = now;
165
171
  this.starvationCapMs = starvationCapMs;
@@ -313,6 +319,7 @@ export class CooperativePassScheduler {
313
319
  return;
314
320
  if (this._handoffActive) {
315
321
  this.promoteOccupancyArrivals();
322
+ this.admitHandoffBacklog();
316
323
  return;
317
324
  }
318
325
  if (this.active !== null) {
@@ -422,15 +429,215 @@ export class CooperativePassScheduler {
422
429
  execute(entry) {
423
430
  if (entry.resourceCap)
424
431
  this.options.caps?.beginChore();
432
+ const checkpoint = Object.assign(() => this.checkpoint(entry), {
433
+ shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
434
+ runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
435
+ });
425
436
  return entry.task({
426
- checkpoint: () => this.checkpoint(entry),
437
+ checkpoint,
427
438
  signal: entry.controller.signal,
428
439
  });
429
440
  }
441
+ shouldYieldToRealtime(entry) {
442
+ return !this.stopped &&
443
+ entry.workClass === "chore" &&
444
+ this.active === entry &&
445
+ this.pending.some((pending) => pending.workClass === "realtime");
446
+ }
430
447
  withinHandoffAdmission() {
431
448
  if (this.handoffDeferredAt === null)
432
449
  return false;
433
- return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
450
+ return this.networkTransferActive ||
451
+ this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
452
+ }
453
+ /**
454
+ * Run payload I/O without owning the pass's durable state. Keep the chore
455
+ * slot reserved while realtime uses the root lock, then wait for active
456
+ * realtime writers before restoring chore ownership.
457
+ */
458
+ async runChoreNetworkTransfer(entry, work) {
459
+ if (this.detachedChore === entry &&
460
+ this._handoffActive &&
461
+ this.networkTransferActive &&
462
+ !this.networkTransferClosing) {
463
+ return this.joinChoreNetworkTransfer(work);
464
+ }
465
+ if (this.detachedChore === entry &&
466
+ this._handoffActive &&
467
+ this.networkTransferRelease !== null) {
468
+ await this.networkTransferRelease;
469
+ return this.runChoreNetworkTransfer(entry, work);
470
+ }
471
+ if (this.active === entry && this.networkTransferApplyRelease !== null) {
472
+ await this.networkTransferApplyRelease;
473
+ return this.runChoreNetworkTransfer(entry, work);
474
+ }
475
+ if (this.stopped ||
476
+ entry.workClass !== "chore" ||
477
+ this.active !== entry) {
478
+ return { value: await work(), realtimeRan: false, complete: () => undefined };
479
+ }
480
+ this.active = null;
481
+ this._handoffActive = true;
482
+ this.networkTransferActive = true;
483
+ this.detachedChore = entry;
484
+ this.networkTransferClosing = false;
485
+ this.networkTransferCount = 1;
486
+ this.networkTransferCompletionCount = 1;
487
+ const applyReleased = new Promise((resolve) => {
488
+ this.resolveNetworkTransferApplyRelease = resolve;
489
+ });
490
+ this.networkTransferApplyRelease = applyReleased;
491
+ const released = new Promise((resolve) => {
492
+ this.resolveNetworkTransferRelease = resolve;
493
+ });
494
+ this.networkTransferRelease = released;
495
+ this.handoffDeferredAt = this.now();
496
+ this.handoffAdmitted = false;
497
+ this.handoffBacklog = null;
498
+ this.options.caps?.suspendForHandoff();
499
+ let value;
500
+ let failure;
501
+ let failed = false;
502
+ try {
503
+ // Admit an arrival that was queued just before the transfer began.
504
+ this.drain();
505
+ value = await work();
506
+ }
507
+ catch (err) {
508
+ failed = true;
509
+ failure = err;
510
+ }
511
+ finally {
512
+ this.networkTransferClosing = true;
513
+ this.networkTransferCount -= 1;
514
+ this.notifyNetworkTransferChange();
515
+ while (this.networkTransferCount > 0) {
516
+ const seen = this.networkTransferEpoch;
517
+ if (this.networkTransferCount === 0)
518
+ break;
519
+ await this.waitNetworkTransferChange(seen);
520
+ }
521
+ this.networkTransferActive = false;
522
+ // A long transfer may outlive the ordinary realtime burst. Start a new
523
+ // bounded window for arrivals that queued safely during network I/O.
524
+ this.handoffDeferredAt = this.now();
525
+ try {
526
+ this.drain();
527
+ while (!this.stopped &&
528
+ (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
529
+ if (this.realtimeOccupants.size === 0) {
530
+ this.admitHandoffBacklog();
531
+ if (this.realtimeOccupants.size === 0)
532
+ break;
533
+ }
534
+ const seen = this.occupantEpoch;
535
+ if (this.realtimeOccupants.size === 0)
536
+ break;
537
+ await this.waitOccupantChange(seen);
538
+ this.promoteOccupancyArrivals();
539
+ this.admitHandoffBacklog();
540
+ }
541
+ while (this.realtimeOccupants.size > 0) {
542
+ const seen = this.occupantEpoch;
543
+ if (this.realtimeOccupants.size === 0)
544
+ break;
545
+ await this.waitOccupantChange(seen);
546
+ this.promoteOccupancyArrivals();
547
+ }
548
+ }
549
+ finally {
550
+ this._handoffActive = false;
551
+ this.handoffDeferredAt = null;
552
+ this.handoffBacklog = null;
553
+ this.detachedChore = null;
554
+ if (!this.stopped)
555
+ this.active = entry;
556
+ this.options.caps?.resumeAfterHandoff();
557
+ const resolveRelease = this.resolveNetworkTransferRelease;
558
+ const realtimeRan = this.handoffAdmitted;
559
+ this.networkTransferRelease = null;
560
+ this.resolveNetworkTransferRelease = null;
561
+ this.networkTransferClosing = false;
562
+ resolveRelease?.(realtimeRan);
563
+ }
564
+ }
565
+ const realtimeRan = await released;
566
+ if (failed) {
567
+ this.completeNetworkTransferParticipant();
568
+ throw failure;
569
+ }
570
+ return {
571
+ value,
572
+ realtimeRan,
573
+ complete: this.createNetworkTransferCompletion(),
574
+ };
575
+ }
576
+ async joinChoreNetworkTransfer(work) {
577
+ const released = this.networkTransferRelease;
578
+ if (released === null) {
579
+ throw new Error("network transfer hand-off is missing its release signal");
580
+ }
581
+ this.networkTransferCount += 1;
582
+ this.networkTransferCompletionCount += 1;
583
+ let value;
584
+ let failure;
585
+ let failed = false;
586
+ try {
587
+ value = await work();
588
+ }
589
+ catch (err) {
590
+ failed = true;
591
+ failure = err;
592
+ }
593
+ finally {
594
+ this.networkTransferCount -= 1;
595
+ this.notifyNetworkTransferChange();
596
+ }
597
+ const realtimeRan = await released;
598
+ if (failed) {
599
+ this.completeNetworkTransferParticipant();
600
+ throw failure;
601
+ }
602
+ return {
603
+ value,
604
+ realtimeRan,
605
+ complete: this.createNetworkTransferCompletion(),
606
+ };
607
+ }
608
+ createNetworkTransferCompletion() {
609
+ let completed = false;
610
+ return () => {
611
+ if (completed)
612
+ return;
613
+ completed = true;
614
+ this.completeNetworkTransferParticipant();
615
+ };
616
+ }
617
+ completeNetworkTransferParticipant() {
618
+ if (this.networkTransferCompletionCount <= 0)
619
+ return;
620
+ this.networkTransferCompletionCount -= 1;
621
+ if (this.networkTransferCompletionCount > 0)
622
+ return;
623
+ const resolve = this.resolveNetworkTransferApplyRelease;
624
+ this.resolveNetworkTransferApplyRelease = null;
625
+ this.networkTransferApplyRelease = null;
626
+ resolve?.();
627
+ }
628
+ waitNetworkTransferChange(seenEpoch) {
629
+ if (this.networkTransferEpoch !== seenEpoch)
630
+ return Promise.resolve();
631
+ return new Promise((resolve) => {
632
+ this.networkTransferWaiters.push(resolve);
633
+ });
634
+ }
635
+ notifyNetworkTransferChange() {
636
+ this.networkTransferEpoch += 1;
637
+ const waiters = this.networkTransferWaiters;
638
+ this.networkTransferWaiters = [];
639
+ for (const resolve of waiters)
640
+ resolve();
434
641
  }
435
642
  waitOccupantChange(seenEpoch) {
436
643
  if (this.occupantEpoch !== seenEpoch)
@@ -907,16 +1114,6 @@ export function resolveFullReconcileMs(env = process.env) {
907
1114
  const ADAPTIVE_POLL_FLOOR_MS = 60_000;
908
1115
  const ADAPTIVE_POLL_CEIL_MS = 600_000;
909
1116
  const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
910
- /**
911
- * A failed watch pass is still retried in-process, but even an explicitly
912
- * tiny poll interval must not turn a broken dependency into a hot loop.
913
- * Three failures in thirty minutes escalate from this floor to exponential
914
- * backoff; success or a quiet window starts a fresh budget.
915
- */
916
- export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
917
- export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
918
- export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
919
- export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
920
1117
  /** Covers coarse timestamp filesystems and scheduling just before watcher start. */
921
1118
  const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
922
1119
  /** Initial poll-only recovery delay after a watcher conclusively stands down. */
@@ -966,41 +1163,6 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
966
1163
  nextDueAt = now + intervalMs;
967
1164
  return { nextDueAt, delayMs: intervalMs };
968
1165
  }
969
- /**
970
- * Bounds in-process retries after a completed watch pass returned non-zero.
971
- *
972
- * This is deliberately separate from the adaptive poll cadence: cadence
973
- * controls normal remote work, whereas this budget prevents a configured
974
- * short poll from continuously re-running a pass that is already known to
975
- * have failed. The caller supplies the monotonic scheduler clock so tests do
976
- * not need wall-clock waits.
977
- */
978
- export class WatchPassExitRetryBudget {
979
- lastFailureAt = null;
980
- failures = 0;
981
- record(exitCode, now) {
982
- if (exitCode === 0) {
983
- this.lastFailureAt = null;
984
- this.failures = 0;
985
- return 0;
986
- }
987
- if (this.lastFailureAt === null ||
988
- now < this.lastFailureAt ||
989
- now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
990
- this.lastFailureAt = now;
991
- this.failures = 0;
992
- }
993
- this.failures += 1;
994
- this.lastFailureAt = now;
995
- if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
996
- // Preserve the established normal-cadence retry for isolated failures.
997
- // The bounded cooldown starts only after repeated exits in the window.
998
- return 0;
999
- }
1000
- const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
1001
- return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
1002
- }
1003
- }
1004
1166
  /**
1005
1167
  * Select journal entries covered by one watcher delete. Exact file unlinks
1006
1168
  * deliberately avoid enumeration; only directory unlinks expand descendants.
@@ -1320,7 +1482,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1320
1482
  // repair/seed and before fanout. It starts ingress early, but never
1321
1483
  // admits a second durable writer: queued realtime still waits for the
1322
1484
  // existing scheduler's cooperative hand-off checkpoint.
1323
- return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
1485
+ return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
1324
1486
  };
1325
1487
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
1326
1488
  // admitting a pass and immediately after it returns. A pass that runs a long
@@ -1513,8 +1675,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1513
1675
  resolveInterruptWait = null;
1514
1676
  resolve?.();
1515
1677
  };
1516
- const waitForNextPass = async (delayMs, interruptible = true) => {
1517
- if (interruptible && interruptGeneration !== observedInterruptGeneration) {
1678
+ const waitForNextPass = async (delayMs) => {
1679
+ if (interruptGeneration !== observedInterruptGeneration) {
1518
1680
  observedInterruptGeneration = interruptGeneration;
1519
1681
  return;
1520
1682
  }
@@ -1529,9 +1691,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1529
1691
  }, idleHeartbeatIntervalMs);
1530
1692
  heartbeatTimer.unref?.();
1531
1693
  try {
1532
- const waits = [sleep(delayMs), stoppedSignal];
1533
- if (interruptible) {
1534
- waits.push(new Promise((resolve) => {
1694
+ await Promise.race([
1695
+ sleep(delayMs),
1696
+ new Promise((resolve) => {
1535
1697
  resolveInterruptWait = resolve;
1536
1698
  // Do not lose an event that arrived between the first generation
1537
1699
  // check and registration of this wait.
@@ -1539,9 +1701,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1539
1701
  resolveInterruptWait = null;
1540
1702
  resolve();
1541
1703
  }
1542
- }));
1543
- }
1544
- await Promise.race(waits);
1704
+ }),
1705
+ stoppedSignal,
1706
+ ]);
1545
1707
  }
1546
1708
  finally {
1547
1709
  clearInterval(heartbeatTimer);
@@ -1628,32 +1790,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1628
1790
  let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
1629
1791
  const runCoordinatorWork = async (work, task, options = {}) => {
1630
1792
  const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
1631
- ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr, monotonicNow)
1793
+ ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
1632
1794
  : undefined;
1633
1795
  if (!coordinator) {
1634
- try {
1635
- return {
1636
- result: await task({
1637
- id: "legacy-direct",
1638
- work,
1639
- paths: [],
1640
- signal: new AbortController().signal,
1641
- }),
1642
- accepted: true,
1643
- coordinatorRefusedPaths: [],
1644
- engineDispatched: true,
1645
- };
1646
- }
1647
- catch (error) {
1648
- if (work.class === "realtime" &&
1649
- work.source === "receiver" &&
1650
- work.direction === "pull") {
1651
- const refused = receiverAdmissionRefusedError(error);
1652
- if (refused)
1653
- throw refused;
1654
- }
1655
- throw error;
1656
- }
1796
+ return {
1797
+ result: await task({
1798
+ id: "legacy-direct",
1799
+ work,
1800
+ paths: [],
1801
+ signal: new AbortController().signal,
1802
+ }),
1803
+ accepted: true,
1804
+ coordinatorRefusedPaths: [],
1805
+ };
1657
1806
  }
1658
1807
  telemetry?.mark("worker");
1659
1808
  const execution = {
@@ -1670,16 +1819,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1670
1819
  const outcome = await coordinator.submit(work);
1671
1820
  const executorSucceeded = execution.result !== undefined &&
1672
1821
  coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
1673
- // A receiver refusal belongs to a newer replacement pull. Even when
1674
- // this slice durably completed its other paths, the SQS receipt batch
1675
- // cannot be acknowledged until every path is durable. Watcher work
1676
- // retains its subset-publish contract and filters refused paths below.
1677
- const requireEveryPathDurable = work.class === "realtime" &&
1678
- work.source === "receiver" &&
1679
- work.direction === "pull";
1680
- const accepted = executorSucceeded && outcome.paths.every((path) => requireEveryPathDurable
1681
- ? path.status === "durable"
1682
- : path.status === "durable" || path.status === "refused");
1822
+ const allRefused = outcome.paths.length > 0 && outcome.paths.every((path) => path.status === "refused");
1823
+ const accepted = executorSucceeded
1824
+ ? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
1825
+ : allRefused;
1683
1826
  if (telemetry) {
1684
1827
  const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
1685
1828
  ? "durable"
@@ -1689,9 +1832,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1689
1832
  telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
1690
1833
  }
1691
1834
  return {
1692
- result: execution.result ?? 1,
1835
+ result: execution.result ?? (allRefused ? 0 : 1),
1693
1836
  accepted,
1694
- engineDispatched: execution.result !== undefined,
1695
1837
  coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
1696
1838
  .filter((path) => path.status === "refused")
1697
1839
  .map((path) => path.path)),
@@ -1703,13 +1845,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1703
1845
  : error instanceof RealtimeDrainBurstError
1704
1846
  ? "drain-burst"
1705
1847
  : undefined);
1706
- if (work.class === "realtime" &&
1707
- work.source === "receiver" &&
1708
- work.direction === "pull") {
1709
- const refused = receiverAdmissionRefusedError(error);
1710
- if (refused)
1711
- throw refused;
1712
- }
1713
1848
  throw error;
1714
1849
  }
1715
1850
  finally {
@@ -1717,17 +1852,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1717
1852
  }
1718
1853
  };
1719
1854
  let nextCoordinatorCausality = 0;
1720
- const realtimeWork = (source, direction, paths, lane, sequenceNumberByPath) => ({
1855
+ const realtimeWork = (source, direction, paths, lane) => ({
1721
1856
  class: "realtime",
1722
1857
  source,
1723
1858
  direction,
1724
1859
  lane,
1725
1860
  paths: normalizeCoordinatorPaths(paths),
1726
- ...(sequenceNumberByPath === undefined ? {} : { sequenceNumberByPath }),
1727
1861
  causality: `${source}-${++nextCoordinatorCausality}`,
1728
1862
  });
1729
1863
  const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
1730
- const runRealtimePass = (source, direction, paths, lane, task, sequenceNumberByPath) => runCoordinatorWork(realtimeWork(source, direction, paths, lane, sequenceNumberByPath), task);
1864
+ const runRealtimePass = (source, direction, paths, lane, task) => runCoordinatorWork(realtimeWork(source, direction, paths, lane), task);
1731
1865
  const enqueueChore = (kind, task, onSettled, options = { wholeMutation: true }) => {
1732
1866
  void runCoordinatorWork(choreWork(kind), task, options).then(onSettled).catch((error) => {
1733
1867
  process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
@@ -3146,20 +3280,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3146
3280
  // Submit every route in this lane before awaiting any result. In
3147
3281
  // particular, a later route's fast receipt must never wait for an
3148
3282
  // earlier route's slow pull to finish.
3149
- const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced), new Map(events.map((event) => [event.relativePath, event.sequenceNumber])))));
3283
+ const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced))));
3150
3284
  for (const execution of completed) {
3151
- if (!execution.engineDispatched && !execution.accepted) {
3152
- throw new ReceiverAdmissionRefusedError("targeted pull failed: refused before engine admission");
3153
- }
3154
- if (passExitCode(execution.result) !== 0) {
3285
+ if (!execution.accepted || passExitCode(execution.result) !== 0) {
3155
3286
  throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
3156
3287
  }
3157
- if (!execution.accepted) {
3158
- if (execution.coordinatorRefusedPaths.length > 0) {
3159
- throw new ReceiverAdmissionRefusedError("targeted pull failed: paths were refused before engine admission");
3160
- }
3161
- throw new Error("targeted pull was not accepted by the coordinator");
3162
- }
3163
3288
  }
3164
3289
  };
3165
3290
  const receiverSyncBatchFn = async ({ events }) => {
@@ -3209,16 +3334,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3209
3334
  }
3210
3335
  for (const { route, paths } of pathsByRoute.values()) {
3211
3336
  const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
3212
- let result;
3213
- try {
3214
- result = await runGuarded(targetedArgv, undefined, lane, "realtime");
3215
- }
3216
- catch (error) {
3217
- const refused = receiverAdmissionRefusedError(error);
3218
- if (refused)
3219
- throw refused;
3220
- throw error;
3221
- }
3337
+ const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
3222
3338
  if (passExitCode(result) !== 0) {
3223
3339
  throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
3224
3340
  }
@@ -3316,14 +3432,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3316
3432
  authToken: getAccessToken,
3317
3433
  region: runtime.region,
3318
3434
  });
3319
- const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
3320
3435
  const handles = await startES({
3321
3436
  hqRoot,
3322
3437
  apiUrl: runtime.apiUrl,
3323
3438
  authToken: getAccessToken,
3324
- deviceId: deviceIdentity.machineId,
3325
- deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
3326
- deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
3439
+ deviceId: getOrCreateMachineId(hqRoot),
3327
3440
  resolveTenantId: async () => {
3328
3441
  const persons = await eventSyncClient.entity.listByType("person");
3329
3442
  const pick = pickCanonicalPersonEntity(persons);
@@ -3339,18 +3452,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3339
3452
  publishedContentHashStateDir: getStateDir(),
3340
3453
  publishedContentHashLookup,
3341
3454
  onReceiveActivity,
3342
- onReceiptOutcome: (receipt) => {
3343
- // This is a bounded diagnostic trace, deliberately separate from
3344
- // low-cardinality metrics. It includes only the irreversible hash
3345
- // of the event path and the closed receipt outcome.
3346
- process.stderr.write(`${JSON.stringify({
3347
- event: "sync.realtime.receipt",
3348
- schemaVersion: 1,
3349
- source: "receiver",
3350
- direction: "pull",
3351
- ...receipt,
3352
- })}\n`);
3353
- },
3354
3455
  });
3355
3456
  if (!handles)
3356
3457
  return;
@@ -3626,15 +3727,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3626
3727
  // The initial reconcile is due immediately. On-time passes advance from this
3627
3728
  // deadline; overruns restart their cooldown from completion.
3628
3729
  let nextDueAt = monotonicNow();
3629
- const passExitRetryBudget = new WatchPassExitRetryBudget();
3630
- let passExitRetryNotBefore = null;
3631
- const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
3632
- const delayMs = passExitRetryBudget.record(exitCode, now);
3633
- passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
3634
- if (delayMs > 0) {
3635
- process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
3636
- }
3637
- };
3638
3730
  let pollTick = 0;
3639
3731
  let lastCompletedFullReconcileAt = null;
3640
3732
  // Anchor for the elapsed-time cadence: the most recent full reconcile that
@@ -3664,17 +3756,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3664
3756
  return RUNNER_HEAP_RECYCLE_EXIT;
3665
3757
  if (coordinatorAuthRequired)
3666
3758
  return runtime.authRequiredPassExit;
3667
- if (passExitRetryNotBefore !== null) {
3668
- const retryNow = monotonicNow();
3669
- if (retryNow < passExitRetryNotBefore) {
3670
- // A watcher/receiver interrupt must not turn a known failing pass
3671
- // back into a tight loop. Pending work remains queued and runs when
3672
- // this non-interruptible cooldown has elapsed.
3673
- await waitForNextPass(passExitRetryNotBefore - retryNow, false);
3674
- continue;
3675
- }
3676
- passExitRetryNotBefore = null;
3677
- }
3678
3759
  pollTick++;
3679
3760
  areaAutoMigration.onTick();
3680
3761
  const pending = takePendingWatcherChange();
@@ -3770,7 +3851,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3770
3851
  };
3771
3852
  let result;
3772
3853
  let ranFullReconcile = false;
3773
- let ranWatchPass = false;
3774
3854
  let enqueuedFullChore = false;
3775
3855
  if (shouldRunFull) {
3776
3856
  emitDecision("full", reasons);
@@ -3790,9 +3870,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3790
3870
  interruptWait();
3791
3871
  return;
3792
3872
  }
3793
- recordWatchPassExit(exit);
3794
- if (exit !== 0)
3795
- interruptWait();
3796
3873
  // Preserve the numeric success seam accepted by the prior
3797
3874
  // coordinator while also accepting a completed RunnerPassOutcome
3798
3875
  // (including a partial per-company pass) that proves journal
@@ -3837,7 +3914,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3837
3914
  }
3838
3915
  else {
3839
3916
  ranFullReconcile = true;
3840
- ranWatchPass = true;
3841
3917
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3842
3918
  }
3843
3919
  }
@@ -3867,7 +3943,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3867
3943
  result = 0;
3868
3944
  }
3869
3945
  else {
3870
- ranWatchPass = true;
3871
3946
  result = overflowWithKnownRoutes
3872
3947
  ? await runOverflowRouteDrain(workingBatch)
3873
3948
  : await runScopedDrain(workingBatch, pending.generation);
@@ -3885,7 +3960,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3885
3960
  process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
3886
3961
  `falling back to a full reconcile\n`);
3887
3962
  ranFullReconcile = true;
3888
- ranWatchPass = true;
3889
3963
  emitDecision("full", ["targeted-pass-failed"]);
3890
3964
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3891
3965
  }
@@ -4031,8 +4105,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4031
4105
  realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
4032
4106
  }
4033
4107
  const now = monotonicNow();
4034
- if (ranWatchPass)
4035
- recordWatchPassExit(exitCode, now);
4036
4108
  if (ranFullReconcile) {
4037
4109
  // The awaited full pass finished on this tick; reuse the tick's clock
4038
4110
  // sample so the cadence anchor never costs a second read.
@@ -4052,14 +4124,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4052
4124
  ? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
4053
4125
  : advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
4054
4126
  nextDueAt = schedule.nextDueAt;
4055
- const retryDelayMs = passExitRetryNotBefore === null
4056
- ? 0
4057
- : Math.max(0, passExitRetryNotBefore - now);
4058
- const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
4059
4127
  const untilHighWaterMs = realtimeState.nextHighWaterAt === null
4060
- ? nextPassDelayMs
4128
+ ? schedule.delayMs
4061
4129
  : Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
4062
- await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
4130
+ await waitForNextPass(Math.min(schedule.delayMs, untilHighWaterMs));
4063
4131
  }
4064
4132
  return 0;
4065
4133
  }