@indigoai-us/hq-cloud 6.18.49 → 6.18.50-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/bin/sync-runner-company.d.ts +5 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +24 -2
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-watch-loop.d.ts +85 -5
  6. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.js +1062 -123
  8. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  9. package/dist/bin/sync-runner.d.ts +30 -11
  10. package/dist/bin/sync-runner.d.ts.map +1 -1
  11. package/dist/bin/sync-runner.js +133 -15
  12. package/dist/bin/sync-runner.js.map +1 -1
  13. package/dist/bin/watcher-catchup-skip-store.d.ts +60 -0
  14. package/dist/bin/watcher-catchup-skip-store.d.ts.map +1 -0
  15. package/dist/bin/watcher-catchup-skip-store.js +282 -0
  16. package/dist/bin/watcher-catchup-skip-store.js.map +1 -0
  17. package/dist/cli/share.d.ts +12 -0
  18. package/dist/cli/share.d.ts.map +1 -1
  19. package/dist/cli/share.js +57 -4
  20. package/dist/cli/share.js.map +1 -1
  21. package/dist/cli/sync.d.ts +3 -0
  22. package/dist/cli/sync.d.ts.map +1 -1
  23. package/dist/cli/sync.js +27 -7
  24. package/dist/cli/sync.js.map +1 -1
  25. package/dist/cli/tombstones.d.ts.map +1 -1
  26. package/dist/cli/tombstones.js +7 -2
  27. package/dist/cli/tombstones.js.map +1 -1
  28. package/dist/ignore.d.ts +1 -1
  29. package/dist/ignore.d.ts.map +1 -1
  30. package/dist/ignore.js +2 -2
  31. package/dist/ignore.js.map +1 -1
  32. package/dist/lib/cloud-authoritative.d.ts +9 -2
  33. package/dist/lib/cloud-authoritative.d.ts.map +1 -1
  34. package/dist/lib/cloud-authoritative.js +34 -2
  35. package/dist/lib/cloud-authoritative.js.map +1 -1
  36. package/dist/local-path-codec.d.ts.map +1 -1
  37. package/dist/local-path-codec.js +3 -2
  38. package/dist/local-path-codec.js.map +1 -1
  39. package/dist/personal-vault-exclusions.d.ts +2 -2
  40. package/dist/personal-vault-exclusions.d.ts.map +1 -1
  41. package/dist/personal-vault-exclusions.js +5 -3
  42. package/dist/personal-vault-exclusions.js.map +1 -1
  43. package/dist/personal-vault.d.ts +7 -0
  44. package/dist/personal-vault.d.ts.map +1 -1
  45. package/dist/personal-vault.js +10 -0
  46. package/dist/personal-vault.js.map +1 -1
  47. package/dist/symlink-target-policy.d.ts +11 -0
  48. package/dist/symlink-target-policy.d.ts.map +1 -1
  49. package/dist/symlink-target-policy.js +80 -0
  50. package/dist/symlink-target-policy.js.map +1 -1
  51. package/dist/sync/parallel-sync-coordinator.d.ts +29 -0
  52. package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
  53. package/dist/sync/parallel-sync-coordinator.js +229 -16
  54. package/dist/sync/parallel-sync-coordinator.js.map +1 -1
  55. package/dist/sync/push-receiver.d.ts +5 -0
  56. package/dist/sync/push-receiver.d.ts.map +1 -1
  57. package/dist/sync/push-receiver.js +32 -10
  58. package/dist/sync/push-receiver.js.map +1 -1
  59. package/dist/watcher.d.ts +16 -3
  60. package/dist/watcher.d.ts.map +1 -1
  61. package/dist/watcher.js +114 -21
  62. package/dist/watcher.js.map +1 -1
  63. package/package.json +1 -1
@@ -1,6 +1,7 @@
1
1
  import * as os from "os";
2
2
  import * as path from "path";
3
3
  import * as fs from "fs";
4
+ import { createHash } from "node:crypto";
4
5
  import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
5
6
  import * as v8 from "node:v8";
6
7
  import { performance } from "node:perf_hooks";
@@ -15,13 +16,18 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
15
16
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
16
17
  import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
17
18
  import { localPathForVaultKey } from "../local-path-codec.js";
19
+ import { isEphemeralPath } from "../cli/share.js";
20
+ import { isSafeRelativePath } from "../sync/push-event.js";
21
+ import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
22
+ import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
18
23
  import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
19
24
  import { createIgnoreFilter } from "../ignore.js";
25
+ import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
20
26
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
21
27
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
22
28
  import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
23
- import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
24
- import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
29
+ import { isPersonalVaultDeletionExcluded, readCloudBackedSlugsFromManifest, computeContinuityPointerPaths, computeAgencySyncPaths, computeSessionLogsSyncPaths, isPersonalVaultScopeExcluded, } from "../personal-vault.js";
30
+ import { NoopPushReceiver, ReceiverDispatchError, RealtimePushYieldError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
25
31
  import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
26
32
  import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
27
33
  import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
@@ -61,13 +67,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
61
67
  */
62
68
  export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
63
69
  /**
64
- * How many realtime executes may run during one chore hand-off.
70
+ * Maximum guarded realtime executes, including the reserved watcher push peer.
65
71
  * HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
66
- * workers. One long transfer keeps its worker; a realtime item that arrives
67
- * while that execute() is in flight starts on the other slot instead of
68
- * waiting for the whole batch. In-flight work is never aborted.
72
+ * workers. One long pull and one long push may each retain a normal worker;
73
+ * one additional watcher push can still make bounded progress. In-flight
74
+ * work is never aborted for this admission.
69
75
  */
70
- export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
76
+ export const REALTIME_WORKER_OCCUPANCY_CAP = 3;
71
77
  /** Objects at least one multipart part are sent through the chore hand-off. */
72
78
  export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
73
79
  let realtimeAdmissionDeadlineMsForTest;
@@ -91,6 +97,29 @@ function realtimeDrainBurstMs() {
91
97
  * so a long chore costs at most one attempt per deadline plus this delay.
92
98
  */
93
99
  export const REALTIME_ADMISSION_RETRY_MS = 5_000;
100
+ /** Bound consecutive pull admissions while a watcher push is queued. */
101
+ export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
102
+ /** Bound synchronous warm-up candidate work between event-loop turns. */
103
+ export const WATCHER_WARMUP_YIELD_EVERY = 128;
104
+ export function shouldYieldWatcherWarmup(examinedPaths) {
105
+ return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
106
+ }
107
+ export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
108
+ return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
109
+ }
110
+ export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
111
+ // TreeWatcher marks this only when its production filter has already applied
112
+ // the same personal scope predicate to every collected candidate. Unrouted
113
+ // paths retain the runner check because the collector may include company paths.
114
+ return pushFiltersApplied && routeKind === "personal";
115
+ }
116
+ export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
117
+ const ignoreFilter = createIgnore(hqRoot);
118
+ const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
119
+ onPersonalExcluded(relativePath, match.id);
120
+ });
121
+ return { ignoreFilter, personalPushFilter };
122
+ }
94
123
  /**
95
124
  * In-process result of a scoped drain whose only shortfall was admission
96
125
  * deferrals: the paths are restored and a retry is scheduled. Never a
@@ -191,6 +220,32 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
191
220
  return explicitlyAllowed ||
192
221
  (workClass === "realtime" && isFullPassDurableProgressEnabled());
193
222
  }
223
+ export function resolveWatcherPushWaitStage(input) {
224
+ if (input.capYieldActive)
225
+ return "CPU/heap cap wait";
226
+ if (input.holders.some((holder) => holder.kind === "pull"))
227
+ return "pull-yield wait";
228
+ if (input.queueDepth > 0)
229
+ return "queued behind realtime work";
230
+ if (input.holders.some((holder) => holder.kind === "chore"))
231
+ return "chore holder wait";
232
+ if (input.holders.length > 0)
233
+ return "scheduler holder wait";
234
+ return "scheduler admission pending";
235
+ }
236
+ export function watcherPushWaitDiagnostics(input) {
237
+ return {
238
+ stage: input.admissionWaiting
239
+ ? resolveWatcherPushWaitStage(input)
240
+ : input.stage,
241
+ waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
242
+ ? null
243
+ : Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
244
+ schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
245
+ ? null
246
+ : Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
247
+ };
248
+ }
194
249
  /**
195
250
  * One durable-state owner at a time for a full chore. Realtime work runs only
196
251
  * while that chore is suspended at `checkpoint()`. Only coordinator-fenced
@@ -205,6 +260,8 @@ export class CooperativePassScheduler {
205
260
  starvationCapMs;
206
261
  options;
207
262
  pending = [];
263
+ lastAnyAdmissionAtMonotonicMs = null;
264
+ nextEntryId = 0;
208
265
  active = null;
209
266
  /** Realtime executes started by the current hand-off. Not the chore slot. */
210
267
  realtimeOccupants = new Set();
@@ -241,12 +298,58 @@ export class CooperativePassScheduler {
241
298
  /** Successful callers still applying results after the network hand-off. */
242
299
  networkTransferCompletionCount = 0;
243
300
  networkTransferApplyRelease = null;
301
+ consecutivePullAdmissions = 0;
244
302
  resolveNetworkTransferApplyRelease = null;
245
303
  constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
246
304
  this.now = now;
247
305
  this.starvationCapMs = starvationCapMs;
248
306
  this.options = options;
249
307
  }
308
+ recordAdmission(entry) {
309
+ const monotonicMs = performance.now();
310
+ const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
311
+ ? null
312
+ : Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
313
+ const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
314
+ const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
315
+ const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
316
+ const waitingSinceAdmissionRequestMs = oldestPush
317
+ ? Math.max(0, this.now() - oldestPush.enqueuedAt)
318
+ : null;
319
+ if (waitingPush || admittedIsPush) {
320
+ this.options.onAdmission?.({
321
+ entryId: entry.entryId,
322
+ workClass: entry.workClass,
323
+ lane: entry.lane,
324
+ transferDirection: entry.transferDirection,
325
+ transferSizeClass: entry.transferSizeClass ?? "unknown",
326
+ label: entry.label.slice(0, 160),
327
+ admittedAt: new Date().toISOString(),
328
+ schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
329
+ realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
330
+ oldestWaitingWatcherPush: oldestPush
331
+ ? {
332
+ entryId: oldestPush.entryId,
333
+ label: oldestPush.label.slice(0, 160),
334
+ waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
335
+ }
336
+ : null,
337
+ passedOver: waitingPush && waitingPush !== entry
338
+ ? {
339
+ entryId: waitingPush.entryId,
340
+ label: waitingPush.label.slice(0, 160),
341
+ waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
342
+ reason: entry.transferDirection === "pull"
343
+ ? "pull-admitted-ahead-of-waiting-push"
344
+ : this._handoffActive
345
+ ? "handoff-selector-admitted-other-work"
346
+ : "scheduler-admitted-other-work",
347
+ }
348
+ : null,
349
+ });
350
+ }
351
+ this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
352
+ }
250
353
  get handoffActive() {
251
354
  return this._handoffActive;
252
355
  }
@@ -263,12 +366,40 @@ export class CooperativePassScheduler {
263
366
  get hasActiveSlowRealtime() {
264
367
  return this.active?.workClass === "realtime" && this.active.lane === "slow";
265
368
  }
369
+ get isInCapYield() {
370
+ return this.capYieldDepth > 0;
371
+ }
266
372
  /** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
267
373
  get activeRealtimeWorkerCount() {
268
374
  return ((this.active?.workClass === "realtime" ? 1 : 0) +
269
375
  (this.activeFastRealtime === null ? 0 : 1) +
270
376
  this.realtimeOccupants.size);
271
377
  }
378
+ get diagnosticAdmissionSnapshot() {
379
+ const active = [
380
+ this.active,
381
+ this.activeFastRealtime,
382
+ ...this.realtimeOccupants,
383
+ this.detachedChore,
384
+ ].filter((entry) => entry !== null);
385
+ const holders = active.map((entry) => ({
386
+ entryId: entry.entryId,
387
+ kind: entry.transferDirection === "pull"
388
+ ? "pull"
389
+ : entry.workClass === "chore"
390
+ ? "chore"
391
+ : "push",
392
+ scope: entry.label.slice(0, 160),
393
+ ageMs: entry.startedAtMonotonicMs === undefined
394
+ ? 0
395
+ : Math.max(0, performance.now() - entry.startedAtMonotonicMs),
396
+ }));
397
+ return {
398
+ queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
399
+ holders: holders.slice(0, 20),
400
+ lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
401
+ };
402
+ }
272
403
  pendingLaneCount(lane) {
273
404
  return this.pending.filter((entry) => entry.lane === lane).length;
274
405
  }
@@ -283,6 +414,7 @@ export class CooperativePassScheduler {
283
414
  signal.addEventListener("abort", () => controller.abort(), { once: true });
284
415
  }
285
416
  const directEntry = {
417
+ entryId: `guarded-${++this.nextEntryId}`,
286
418
  task,
287
419
  workClass,
288
420
  label,
@@ -306,7 +438,8 @@ export class CooperativePassScheduler {
306
438
  if (workClass === "realtime" &&
307
439
  this._handoffActive &&
308
440
  directEntry.allowConcurrentHandoff &&
309
- this.canAdmitRealtimeOccupant(directEntry)) {
441
+ this.canAdmitRealtimeOccupant(directEntry) &&
442
+ !this.pullMustYieldForWaitingPush(directEntry)) {
310
443
  return new Promise((resolve, reject) => {
311
444
  const entry = {
312
445
  ...directEntry,
@@ -455,16 +588,21 @@ export class CooperativePassScheduler {
455
588
  this.admitHandoffBacklog();
456
589
  return;
457
590
  }
458
- if (this.active !== null) {
591
+ if (this.active !== null || this.activeFastRealtime !== null || this.realtimeOccupants.size > 0) {
459
592
  const fastRealtime = this.nextFastRealtime();
460
- if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
593
+ if (this.active !== null && fastRealtime && this.canStartFastRealtime(fastRealtime)) {
461
594
  this.pending.splice(this.pending.indexOf(fastRealtime), 1);
462
595
  this.startFastRealtime(fastRealtime);
596
+ return;
597
+ }
598
+ const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
599
+ if (peerIndex >= 0) {
600
+ const peer = this.pending.splice(peerIndex, 1)[0];
601
+ if (peer)
602
+ this.startRealtimeOccupant(peer);
463
603
  }
464
604
  return;
465
605
  }
466
- if (this.activeFastRealtime !== null)
467
- return;
468
606
  const next = this.takeNext();
469
607
  if (next)
470
608
  this.start(next);
@@ -483,11 +621,29 @@ export class CooperativePassScheduler {
483
621
  return this.pending.splice(realtime, 1)[0];
484
622
  return this.pending.shift();
485
623
  }
486
- nextRealtimeIndex() {
487
- const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
488
- return slowRealtime >= 0
624
+ pullMustYieldForWaitingPush(entry) {
625
+ return entry.transferDirection === "pull" &&
626
+ this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH &&
627
+ this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
628
+ }
629
+ nextRealtimeIndex(eligible = () => true) {
630
+ const waitingPush = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.transferDirection === "push" && eligible(entry));
631
+ if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
632
+ this.consecutivePullAdmissions = 0;
633
+ return waitingPush;
634
+ }
635
+ const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow" && eligible(entry));
636
+ const next = slowRealtime >= 0
489
637
  ? slowRealtime
490
- : this.pending.findIndex((entry) => entry.workClass === "realtime");
638
+ : this.pending.findIndex((entry) => entry.workClass === "realtime" && eligible(entry));
639
+ if (next >= 0) {
640
+ const selected = this.pending[next];
641
+ if (selected?.transferDirection === "pull")
642
+ this.consecutivePullAdmissions += 1;
643
+ else if (selected?.transferDirection === "push")
644
+ this.consecutivePullAdmissions = 0;
645
+ }
646
+ return next;
491
647
  }
492
648
  nextFastRealtime() {
493
649
  return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
@@ -505,8 +661,20 @@ export class CooperativePassScheduler {
505
661
  this.hasActiveSlowRealtime &&
506
662
  !this.hasStarvedChore());
507
663
  }
664
+ /** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
665
+ canAdmitRealtimePeer(entry) {
666
+ if (entry.workClass !== "realtime" ||
667
+ !entry.allowConcurrentHandoff ||
668
+ entry.transferDirection === undefined ||
669
+ this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
670
+ return false;
671
+ return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
672
+ active.transferDirection !== undefined &&
673
+ active.transferDirection !== entry.transferDirection);
674
+ }
508
675
  start(entry) {
509
676
  this.clearAdmission(entry);
677
+ this.recordAdmission(entry);
510
678
  entry.startedAt = new Date().toISOString();
511
679
  // Holder duration uses the process monotonic clock; do not consume the
512
680
  // injected scheduler clock, whose reads define cadence-slot deadlines.
@@ -545,6 +713,8 @@ export class CooperativePassScheduler {
545
713
  }
546
714
  startFastRealtime(entry) {
547
715
  this.clearAdmission(entry);
716
+ this.recordAdmission(entry);
717
+ entry.startedAtMonotonicMs = performance.now();
548
718
  this.activeFastRealtime = entry;
549
719
  let current;
550
720
  try {
@@ -800,6 +970,8 @@ export class CooperativePassScheduler {
800
970
  /** Run one realtime execute without aborting it and without taking the chore slot. */
801
971
  startRealtimeOccupant(entry) {
802
972
  this.clearAdmission(entry);
973
+ this.recordAdmission(entry);
974
+ entry.startedAtMonotonicMs = performance.now();
803
975
  this.realtimeOccupants.add(entry);
804
976
  this.handoffAdmitted = true;
805
977
  if (this.capYieldDepth > 0)
@@ -814,6 +986,7 @@ export class CooperativePassScheduler {
814
986
  this.handoffBacklog = null;
815
987
  entry.reject(error);
816
988
  this.notifyOccupantChange();
989
+ this.drain();
817
990
  return;
818
991
  }
819
992
  void running.then((outcome) => {
@@ -823,6 +996,7 @@ export class CooperativePassScheduler {
823
996
  entry.resolve(outcome);
824
997
  this.promoteOccupancyArrivals();
825
998
  this.notifyOccupantChange();
999
+ this.drain();
826
1000
  }, (error) => {
827
1001
  this.realtimeOccupants.delete(entry);
828
1002
  if (this.handoffBacklog === entry)
@@ -830,6 +1004,7 @@ export class CooperativePassScheduler {
830
1004
  entry.reject(error);
831
1005
  this.promoteOccupancyArrivals();
832
1006
  this.notifyOccupantChange();
1007
+ this.drain();
833
1008
  });
834
1009
  }
835
1010
  /** Eligible arrivals may take a free slot before hand-off admission closes. */
@@ -838,8 +1013,7 @@ export class CooperativePassScheduler {
838
1013
  (!this.networkTransferActive && !this.withinHandoffAdmission()))
839
1014
  return;
840
1015
  while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
841
- const index = this.pending.findIndex((pending) => pending.workClass === "realtime" &&
842
- pending.occupancyArrival === true &&
1016
+ const index = this.nextRealtimeIndex((pending) => pending.occupancyArrival === true &&
843
1017
  pending.allowConcurrentHandoff &&
844
1018
  this.canAdmitRealtimeOccupant(pending));
845
1019
  if (index < 0)
@@ -858,7 +1032,7 @@ export class CooperativePassScheduler {
858
1032
  return;
859
1033
  if (!this.withinHandoffAdmission())
860
1034
  return;
861
- const realtimeIndex = this.nextRealtimeIndex();
1035
+ const realtimeIndex = this.nextRealtimeIndex((pending) => !pending.occupancyArrival && this.canAdmitRealtimeOccupant(pending));
862
1036
  if (realtimeIndex < 0)
863
1037
  return;
864
1038
  const pending = this.pending[realtimeIndex];
@@ -1292,6 +1466,151 @@ export function isChoreLocalPush(paths, largeUpload) {
1292
1466
  }
1293
1467
  return count > 0;
1294
1468
  }
1469
+ /** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
1470
+ export function watcherWarmupPathsByRoute(relativePaths) {
1471
+ const counts = new Map();
1472
+ for (const relativePath of relativePaths) {
1473
+ const route = routeChangeToTarget(relativePath);
1474
+ const key = route ? routeKey(route) : "unrouted";
1475
+ counts.set(key, (counts.get(key) ?? 0) + 1);
1476
+ }
1477
+ const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1478
+ const result = {};
1479
+ for (const [key, count] of ranked.slice(0, 20))
1480
+ result[key] = count;
1481
+ const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
1482
+ if (other > 0)
1483
+ result.other = other;
1484
+ return result;
1485
+ }
1486
+ export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
1487
+ const candidatePrefixes = new Map();
1488
+ const candidateCounts = new Map();
1489
+ for (const relativePath of candidateRelativePaths) {
1490
+ const route = routeChangeToTarget(relativePath);
1491
+ const key = route ? routeKey(route) : "unrouted";
1492
+ candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
1493
+ const routePath = route?.kind === "company"
1494
+ ? relativePath.slice(`companies/${route.slug}/`.length)
1495
+ : relativePath;
1496
+ const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
1497
+ const routeCounts = candidatePrefixes.get(key) ?? new Map();
1498
+ routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
1499
+ candidatePrefixes.set(key, routeCounts);
1500
+ }
1501
+ const eligibleCounts = new Map();
1502
+ for (const relativePath of eligibleRelativePaths) {
1503
+ const route = routeChangeToTarget(relativePath);
1504
+ const key = route ? routeKey(route) : "unrouted";
1505
+ const routePath = route?.kind === "company"
1506
+ ? relativePath.slice(`companies/${route.slug}/`.length)
1507
+ : relativePath;
1508
+ const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
1509
+ const routeCounts = eligibleCounts.get(key) ?? new Map();
1510
+ routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
1511
+ eligibleCounts.set(key, routeCounts);
1512
+ }
1513
+ const result = {};
1514
+ // Eligible routes are all retained; fill any remaining diagnostic slots
1515
+ // with the largest candidate routes. This keeps alphabetical one-path
1516
+ // stubs from crowding out a large route such as personal.
1517
+ const eligibleRoutes = [...eligibleCounts.keys()];
1518
+ const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
1519
+ const rankedCandidates = [...candidateCounts.entries()]
1520
+ .filter(([route]) => !eligibleCounts.has(route))
1521
+ .sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1522
+ const selectedRoutes = [
1523
+ ...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
1524
+ ...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
1525
+ ];
1526
+ for (const route of selectedRoutes) {
1527
+ const topPrefixes = (counts) => {
1528
+ const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1529
+ const top = {};
1530
+ for (const [prefix, count] of ranked.slice(0, 5))
1531
+ top[prefix] = count;
1532
+ const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
1533
+ if (other > 0)
1534
+ top.other = other;
1535
+ return top;
1536
+ };
1537
+ result[route] = {
1538
+ candidate: topPrefixes(candidatePrefixes.get(route)),
1539
+ eligible: topPrefixes(eligibleCounts.get(route)),
1540
+ };
1541
+ }
1542
+ const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
1543
+ if (otherRoutes.length > 0) {
1544
+ const merged = new Map();
1545
+ for (const [, routeCounts] of otherRoutes) {
1546
+ for (const [prefix, count] of routeCounts)
1547
+ merged.set(prefix, (merged.get(prefix) ?? 0) + count);
1548
+ }
1549
+ result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
1550
+ }
1551
+ return result;
1552
+ }
1553
+ function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
1554
+ counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
1555
+ const firstSlash = routePath.indexOf("/");
1556
+ const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
1557
+ const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
1558
+ let byPrefix = prefixes.get(routeName);
1559
+ if (!byPrefix) {
1560
+ byPrefix = new Map();
1561
+ prefixes.set(routeName, byPrefix);
1562
+ }
1563
+ byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
1564
+ }
1565
+ function rankedWarmupCounts(counts) {
1566
+ const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1567
+ const result = {};
1568
+ for (const [key, count] of ranked.slice(0, 20))
1569
+ result[key] = count;
1570
+ const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
1571
+ if (other > 0)
1572
+ result.other = other;
1573
+ return result;
1574
+ }
1575
+ function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
1576
+ const topPrefixes = (counts) => {
1577
+ const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1578
+ const top = {};
1579
+ for (const [prefix, count] of ranked.slice(0, 5))
1580
+ top[prefix] = count;
1581
+ const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
1582
+ if (other > 0)
1583
+ top.other = other;
1584
+ return top;
1585
+ };
1586
+ const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
1587
+ const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
1588
+ const rankedCandidates = [...candidateCounts.entries()]
1589
+ .filter(([route]) => !eligibleCounts.has(route))
1590
+ .sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1591
+ const selectedRoutes = [
1592
+ ...eligibleRoutes,
1593
+ ...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
1594
+ ];
1595
+ const result = {};
1596
+ for (const route of selectedRoutes) {
1597
+ result[route] = {
1598
+ candidate: topPrefixes(candidatePrefixes.get(route)),
1599
+ eligible: topPrefixes(eligiblePrefixes.get(route)),
1600
+ };
1601
+ }
1602
+ const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
1603
+ if (otherRoutes.length > 0) {
1604
+ const merged = new Map();
1605
+ for (const route of otherRoutes) {
1606
+ for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
1607
+ merged.set(prefix, (merged.get(prefix) ?? 0) + count);
1608
+ }
1609
+ }
1610
+ result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
1611
+ }
1612
+ return result;
1613
+ }
1295
1614
  /** A receiver event needs a known small upsert size before it can go realtime. */
1296
1615
  export function isFastReceiverBatch(eventCount) {
1297
1616
  return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
@@ -1763,6 +2082,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1763
2082
  let recycleRequested = null;
1764
2083
  const pendingAppliedUpserts = new Map();
1765
2084
  const pendingAppliedDeletes = new Set();
2085
+ const uploadedAwaitingPublish = new Set();
2086
+ const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
2087
+ const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
2088
+ let uploadedAwaitingPublishDropLogged = false;
1766
2089
  const rememberAppliedUpsert = (relativePath, contentHash) => {
1767
2090
  pendingAppliedDeletes.delete(relativePath);
1768
2091
  if (eventSyncLive && eventSync) {
@@ -1793,6 +2116,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1793
2116
  pendingAppliedDeletes.delete(oldest);
1794
2117
  }
1795
2118
  };
2119
+ const flushPendingUploadEvents = () => {
2120
+ if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
2121
+ return;
2122
+ while (uploadedAwaitingPublish.size > 0) {
2123
+ const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
2124
+ const paths = new Map(uploadedPaths.map((relativePath) => [
2125
+ localPathForVaultKey(hqRoot, relativePath),
2126
+ relativePath,
2127
+ ]));
2128
+ const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
2129
+ if (expanded.size > 0)
2130
+ eventSync.publishBatch({ paths: expanded });
2131
+ for (const relativePath of uploadedPaths) {
2132
+ uploadedAwaitingPublish.delete(relativePath);
2133
+ }
2134
+ }
2135
+ };
1796
2136
  const flushPendingAppliedUpserts = () => {
1797
2137
  if (!eventSyncLive || !eventSync)
1798
2138
  return;
@@ -1804,6 +2144,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1804
2144
  eventSync.markContentDeleted?.(relativePath);
1805
2145
  }
1806
2146
  pendingAppliedDeletes.clear();
2147
+ flushPendingUploadEvents();
1807
2148
  };
1808
2149
  // `activeSlowPass` is assigned before its task has necessarily acquired the
1809
2150
  // cross-process lock (it may be waiting behind a rescue or a second runner).
@@ -1853,8 +2194,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1853
2194
  }, 0);
1854
2195
  refresh.unref?.();
1855
2196
  }
1856
- const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
1857
- const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
2197
+ const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
2198
+ caps: fullPassCaps,
2199
+ onAdmission: (event) => {
2200
+ process.stderr.write(`${JSON.stringify({
2201
+ type: "watcher-push-scheduler-admission",
2202
+ diagnostic: true,
2203
+ ...event,
2204
+ })}\n`);
2205
+ },
2206
+ });
2207
+ const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
1858
2208
  const passCheckpoint = pullSignal || receiverSignal
1859
2209
  ? Object.assign(async () => checkpoint?.(), {
1860
2210
  shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
@@ -1884,20 +2234,43 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1884
2234
  }
1885
2235
  };
1886
2236
  const invokePass = () => {
1887
- if (!runPassSupportsCooperativeCheckpoint) {
1888
- if (!runPassSupportsQmdLifecycle)
1889
- return runPass(passArgvForRun);
1890
- return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
2237
+ watcherPushStage?.("pass invocation");
2238
+ activePassCount += 1;
2239
+ emitSyncWorkState(true);
2240
+ const trackPass = (pass) => {
2241
+ const complete = () => {
2242
+ watcherPushStage?.("pass return");
2243
+ activePassCount = Math.max(0, activePassCount - 1);
2244
+ emitSyncWorkState(true);
2245
+ };
2246
+ // Observe settlement without returning a chained promise to the pass
2247
+ // scheduler. The scheduler's synchronous test seam and watcher wake
2248
+ // ordering depend on the original pass promise's completion turn.
2249
+ void pass.then(complete, complete);
2250
+ return pass;
2251
+ };
2252
+ try {
2253
+ if (!runPassSupportsCooperativeCheckpoint) {
2254
+ if (!runPassSupportsQmdLifecycle)
2255
+ return trackPass(runPass(passArgvForRun));
2256
+ return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
2257
+ }
2258
+ // This callback is emitted by the production runner after journal
2259
+ // repair/seed and before fanout. It starts ingress early, but never
2260
+ // admits a second durable writer: queued realtime still waits for the
2261
+ // existing scheduler's cooperative hand-off checkpoint.
2262
+ const onJournalsReady = () => activateEventPushSurfaces();
2263
+ if (!runPassSupportsQmdLifecycle) {
2264
+ return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
2265
+ }
2266
+ return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
1891
2267
  }
1892
- // This callback is emitted by the production runner after journal
1893
- // repair/seed and before fanout. It starts ingress early, but never
1894
- // admits a second durable writer: queued realtime still waits for the
1895
- // existing scheduler's cooperative hand-off checkpoint.
1896
- const onJournalsReady = () => activateEventPushSurfaces();
1897
- if (!runPassSupportsQmdLifecycle) {
1898
- return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
2268
+ catch (error) {
2269
+ watcherPushStage?.("pass return");
2270
+ activePassCount = Math.max(0, activePassCount - 1);
2271
+ emitSyncWorkState(true);
2272
+ throw error;
1899
2273
  }
1900
- return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
1901
2274
  };
1902
2275
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
1903
2276
  // admitting a pass and immediately after it returns. A pass that runs a long
@@ -1912,6 +2285,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1912
2285
  return;
1913
2286
  recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
1914
2287
  recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
2288
+ rememberUploadedPaths(outcome);
2289
+ if (typeof outcome !== "number") {
2290
+ acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
2291
+ }
2292
+ flushPendingUploadEvents();
1915
2293
  };
1916
2294
  // Pre-admit boundary observation.
1917
2295
  heapCheckpoint?.();
@@ -2162,15 +2540,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2162
2540
  // watcher fast path explicitly opts into its established scoped exception.
2163
2541
  return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
2164
2542
  };
2165
- const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
2543
+ const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass, watcherPushStage) => {
2544
+ if (watcherPushStage) {
2545
+ watcherPushStage("guarded scheduler admission requested");
2546
+ const admission = passScheduler.diagnosticAdmissionSnapshot;
2547
+ if (passScheduler.isInCapYield)
2548
+ watcherPushStage("CPU/heap cap wait");
2549
+ else if (admission.holders.some((holder) => holder.kind === "pull")) {
2550
+ watcherPushStage("pull-yield wait");
2551
+ }
2552
+ }
2166
2553
  telemetry?.mark("operationGate");
2167
2554
  const admissionSignal = signal && receiverSignal
2168
2555
  ? AbortSignal.any([signal, receiverSignal])
2169
2556
  : signal ?? receiverSignal;
2170
2557
  return runGuardedTask((context) => {
2558
+ watcherPushStage?.("guarded scheduler admission granted");
2171
2559
  telemetry?.mark("appendLock");
2172
2560
  telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
2173
- return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
2561
+ return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal, watcherPushStage);
2174
2562
  }, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
2175
2563
  };
2176
2564
  /*
@@ -2292,6 +2680,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2292
2680
  return {
2293
2681
  result: execution.result ?? (allRefused ? 0 : 1),
2294
2682
  accepted,
2683
+ yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
2295
2684
  coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
2296
2685
  .filter((path) => path.status === "refused")
2297
2686
  .map((path) => path.path)),
@@ -2489,6 +2878,222 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2489
2878
  /** Vanished never-journaled file deletes removed from batches at drain time. */
2490
2879
  let drainedUntrackedDeletes = 0;
2491
2880
  const pendingWatcherPaths = new Map();
2881
+ const durableWatcherPaths = new Map();
2882
+ const durableWatcherPathLimit = 10_000;
2883
+ let durableWatcherOverflowed = false;
2884
+ let durableWatcherPersistTimer;
2885
+ let durableWatcherPersistQueue = Promise.resolve();
2886
+ let durableWatcherPersistGeneration = 0;
2887
+ let durableWatcherPersistedGeneration = 0;
2888
+ const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
2889
+ const persistDurableWatcherPaths = () => {
2890
+ const generation = durableWatcherPersistGeneration;
2891
+ if (generation === durableWatcherPersistedGeneration)
2892
+ return durableWatcherPersistQueue;
2893
+ const snapshot = durableWatcherOverflowed
2894
+ ? { format: 1, hqRoot, overflowed: true }
2895
+ : { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
2896
+ durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
2897
+ try {
2898
+ if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
2899
+ await fs.promises.rm(watcherPushStatePath, { force: true });
2900
+ durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
2901
+ return;
2902
+ }
2903
+ await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
2904
+ const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
2905
+ await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
2906
+ await fs.promises.rename(tempPath, watcherPushStatePath);
2907
+ durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
2908
+ }
2909
+ catch (err) {
2910
+ process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
2911
+ }
2912
+ }).catch((err) => {
2913
+ process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
2914
+ });
2915
+ return durableWatcherPersistQueue;
2916
+ };
2917
+ const scheduleDurableWatcherPersist = () => {
2918
+ durableWatcherPersistGeneration += 1;
2919
+ if (durableWatcherPersistTimer !== undefined)
2920
+ clearTimeout(durableWatcherPersistTimer);
2921
+ durableWatcherPersistTimer = setTimeout(() => {
2922
+ durableWatcherPersistTimer = undefined;
2923
+ void persistDurableWatcherPaths();
2924
+ }, 1_000);
2925
+ durableWatcherPersistTimer.unref?.();
2926
+ };
2927
+ const addDurableWatcherPath = (absolutePath, relativePath) => {
2928
+ if (durableWatcherOverflowed)
2929
+ return;
2930
+ if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
2931
+ durableWatcherPaths.clear();
2932
+ durableWatcherOverflowed = true;
2933
+ }
2934
+ else if (!durableWatcherOverflowed) {
2935
+ durableWatcherPaths.set(absolutePath, relativePath);
2936
+ }
2937
+ scheduleDurableWatcherPersist();
2938
+ };
2939
+ const clearDurableWatcherOverflow = () => {
2940
+ if (!durableWatcherOverflowed)
2941
+ return;
2942
+ durableWatcherOverflowed = false;
2943
+ durableWatcherPaths.clear();
2944
+ for (const [absolutePath, relativePath] of pendingWatcherPaths) {
2945
+ addDurableWatcherPath(absolutePath, relativePath);
2946
+ }
2947
+ scheduleDurableWatcherPersist();
2948
+ };
2949
+ const acknowledgeDurableWatcherUploads = (relativePaths) => {
2950
+ let changed = false;
2951
+ const uploaded = new Set(relativePaths);
2952
+ for (const [absolutePath, relativePath] of durableWatcherPaths) {
2953
+ if (!uploaded.has(relativePath))
2954
+ continue;
2955
+ durableWatcherPaths.delete(absolutePath);
2956
+ changed = true;
2957
+ }
2958
+ if (changed)
2959
+ scheduleDurableWatcherPersist();
2960
+ };
2961
+ const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
2962
+ const stillPending = new Set(pendingWatcherPaths.values());
2963
+ acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
2964
+ };
2965
+ let activePassCount = 0;
2966
+ let inFlightWatcherPushes = 0;
2967
+ const watcherPushStages = new Map();
2968
+ let nextWatcherPushGroupId = 0;
2969
+ const setWatcherPushStage = (groupId, stage) => {
2970
+ const record = watcherPushStages.get(groupId);
2971
+ if (!record || record.stage === stage)
2972
+ return;
2973
+ record.stage = stage;
2974
+ record.stageEnteredAt = Date.now();
2975
+ if (stage === "guarded scheduler admission requested") {
2976
+ record.admissionRequestedAtMonotonicMs = performance.now();
2977
+ record.admissionWaiting = true;
2978
+ }
2979
+ else if (stage === "guarded scheduler admission granted") {
2980
+ record.admissionWaiting = false;
2981
+ }
2982
+ process.stderr.write(`${JSON.stringify({
2983
+ type: "watcher-push-group-stage",
2984
+ diagnostic: true,
2985
+ groupId: record.groupId,
2986
+ route: record.route,
2987
+ paths: record.paths,
2988
+ stage: record.stage,
2989
+ stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
2990
+ })}\n`);
2991
+ };
2992
+ const startWatcherPushStageTracking = (route, paths) => {
2993
+ const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
2994
+ watcherPushStages.set(groupId, {
2995
+ groupId,
2996
+ route,
2997
+ paths,
2998
+ stage: "",
2999
+ stageEnteredAt: Date.now(),
3000
+ admissionRequestedAtMonotonicMs: null,
3001
+ admissionWaiting: false,
3002
+ });
3003
+ return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
3004
+ };
3005
+ const emitWatcherPushInflightStages = () => {
3006
+ if (inFlightWatcherPushes <= 0)
3007
+ return;
3008
+ const now = Date.now();
3009
+ const entries = [...watcherPushStages.values()];
3010
+ const admission = passScheduler.diagnosticAdmissionSnapshot;
3011
+ const coordinatorAdmission = coordinator?.inspect();
3012
+ for (const record of entries) {
3013
+ if (!record.admissionWaiting)
3014
+ continue;
3015
+ const liveStage = resolveWatcherPushWaitStage({
3016
+ capYieldActive: passScheduler.isInCapYield,
3017
+ queueDepth: admission.queueDepth,
3018
+ holders: admission.holders,
3019
+ });
3020
+ if (liveStage !== record.stage)
3021
+ setWatcherPushStage(record.groupId, liveStage);
3022
+ }
3023
+ process.stderr.write(`${JSON.stringify({
3024
+ type: "watcher-push-inflight-stages",
3025
+ diagnostic: true,
3026
+ groups: entries.slice(0, 20).map((record) => {
3027
+ const monotonicNow = performance.now();
3028
+ const current = watcherPushWaitDiagnostics({
3029
+ stage: record.stage,
3030
+ admissionWaiting: record.admissionWaiting,
3031
+ admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
3032
+ lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
3033
+ nowMonotonicMs: monotonicNow,
3034
+ capYieldActive: passScheduler.isInCapYield,
3035
+ queueDepth: admission.queueDepth,
3036
+ holders: admission.holders,
3037
+ });
3038
+ return {
3039
+ route: record.route,
3040
+ paths: record.paths,
3041
+ ...current,
3042
+ ageMs: Math.max(0, now - record.stageEnteredAt),
3043
+ };
3044
+ }),
3045
+ other: Math.max(0, entries.length - 20),
3046
+ realtimeAdmissionQueueDepth: admission.queueDepth,
3047
+ holders: admission.holders,
3048
+ coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
3049
+ coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
3050
+ coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
3051
+ fast: null,
3052
+ slow: null,
3053
+ overflowPush: null,
3054
+ },
3055
+ coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
3056
+ coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
3057
+ coordinatorAdmission === undefined
3058
+ ? null
3059
+ : Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
3060
+ coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
3061
+ coordinatorAdmission === undefined
3062
+ ? null
3063
+ : Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
3064
+ })}\n`);
3065
+ };
3066
+ const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
3067
+ watcherPushStagesTimer.unref?.();
3068
+ let pendingReceivers = 0;
3069
+ let lastSyncWorkStateEmitAt = 0;
3070
+ let syncWorkStateTimer;
3071
+ const emitSyncWorkState = (force = false) => {
3072
+ const now = Date.now();
3073
+ const elapsed = now - lastSyncWorkStateEmitAt;
3074
+ if (force || elapsed >= 250) {
3075
+ if (syncWorkStateTimer !== undefined)
3076
+ clearTimeout(syncWorkStateTimer);
3077
+ syncWorkStateTimer = undefined;
3078
+ lastSyncWorkStateEmitAt = now;
3079
+ const snapshot = {
3080
+ passActive: activePassCount > 0,
3081
+ queuedWatcherPushes: pendingWatcherPaths.size,
3082
+ inFlightWatcherPushes,
3083
+ pendingReceivers,
3084
+ };
3085
+ deps.onSyncWorkStateSnapshot?.(snapshot);
3086
+ stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
3087
+ return;
3088
+ }
3089
+ if (syncWorkStateTimer !== undefined)
3090
+ return;
3091
+ syncWorkStateTimer = setTimeout(() => {
3092
+ syncWorkStateTimer = undefined;
3093
+ emitSyncWorkState(true);
3094
+ }, Math.max(1, 250 - elapsed));
3095
+ syncWorkStateTimer.unref?.();
3096
+ };
2492
3097
  const droppedUnresolvedWatcherSlugs = new Set();
2493
3098
  const logDroppedUnresolvedWatcherRoute = (slug) => {
2494
3099
  if (droppedUnresolvedWatcherSlugs.has(slug))
@@ -2556,6 +3161,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2556
3161
  }
2557
3162
  }
2558
3163
  if (batch) {
3164
+ if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
3165
+ durableWatcherPaths.clear();
3166
+ durableWatcherOverflowed = true;
3167
+ scheduleDurableWatcherPersist();
3168
+ }
2559
3169
  const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
2560
3170
  const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
2561
3171
  return previousSequence === undefined ||
@@ -2564,6 +3174,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2564
3174
  });
2565
3175
  if (entries.length === 0)
2566
3176
  return false;
3177
+ const queueWasEmpty = pendingWatcherPaths.size === 0;
2567
3178
  pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
2568
3179
  pendingWatcherOriginalBatch =
2569
3180
  pendingWatcherPaths.size === 0 &&
@@ -2575,11 +3186,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2575
3186
  : null;
2576
3187
  for (const [absolutePath, relativePath] of entries) {
2577
3188
  pendingWatcherPaths.set(absolutePath, relativePath);
3189
+ addDurableWatcherPath(absolutePath, relativePath);
2578
3190
  const change = batch.changes?.get(absolutePath);
2579
3191
  if (change)
2580
3192
  pendingWatcherChanges.set(absolutePath, change);
2581
3193
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2582
3194
  }
3195
+ emitSyncWorkState(queueWasEmpty);
2583
3196
  if (batch.overflowed) {
2584
3197
  pendingWatcherOverflowed = true;
2585
3198
  pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
@@ -2603,16 +3216,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2603
3216
  // counterpart must be re-derived through the codec — a raw join would
2604
3217
  // point at the impossible colon path on Windows.
2605
3218
  const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
3219
+ const queueWasEmpty = pendingWatcherPaths.size === 0;
2606
3220
  pendingWatcherPaths.set(absolutePath, changedRelPath);
3221
+ addDurableWatcherPath(absolutePath, changedRelPath);
2607
3222
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2608
3223
  // A bare path signal carries no delete intent; drop any change record
2609
3224
  // a prior batch left for this path so the stale intent cannot leak
2610
3225
  // into the coalesced batch (#227).
2611
3226
  pendingWatcherChanges.delete(absolutePath);
3227
+ emitSyncWorkState(queueWasEmpty);
2612
3228
  return true;
2613
3229
  }
2614
3230
  pendingWatcherOriginalBatch = null;
2615
3231
  pendingWatcherBareChange = true;
3232
+ emitSyncWorkState();
2616
3233
  return true;
2617
3234
  };
2618
3235
  const takePendingWatcherChange = () => {
@@ -2657,6 +3274,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2657
3274
  pendingWatcherDroppedRouteHints.clear();
2658
3275
  pendingWatcherDroppedRoutesUnknown = false;
2659
3276
  pendingWatcherBatchGeneration = null;
3277
+ emitSyncWorkState();
2660
3278
  return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
2661
3279
  };
2662
3280
  const restorePendingWatcherPaths = (paths, generation, changes) => {
@@ -2670,12 +3288,54 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2670
3288
  if (pendingWatcherChangeSequences.has(absolutePath))
2671
3289
  continue;
2672
3290
  pendingWatcherPaths.set(absolutePath, relativePath);
3291
+ addDurableWatcherPath(absolutePath, relativePath);
2673
3292
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2674
3293
  const change = changes?.get(absolutePath);
2675
3294
  if (change)
2676
3295
  pendingWatcherChanges.set(absolutePath, change);
2677
3296
  }
3297
+ emitSyncWorkState();
2678
3298
  };
3299
+ if (eventPush) {
3300
+ try {
3301
+ const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
3302
+ if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
3303
+ (!Array.isArray(stored.paths) && stored.overflowed !== true)) {
3304
+ throw new Error("stored watcher push state has an unsupported shape or root");
3305
+ }
3306
+ if (stored.overflowed === true) {
3307
+ durableWatcherOverflowed = true;
3308
+ pendingWatcherOverflowed = true;
3309
+ pendingWatcherWarmupCatchup = true;
3310
+ pendingWatcherDroppedRoutesUnknown = true;
3311
+ pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3312
+ emitSyncWorkState(true);
3313
+ process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
3314
+ }
3315
+ for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
3316
+ if (typeof relativePath !== "string" || relativePath.length === 0)
3317
+ continue;
3318
+ const route = routeChangeToTarget(relativePath);
3319
+ if (!route || !isWatchRouteSelected(route, watchRouteSelection))
3320
+ continue;
3321
+ const absolutePath = localPathForVaultKey(hqRoot, relativePath);
3322
+ addDurableWatcherPath(absolutePath, relativePath);
3323
+ pendingWatcherPaths.set(absolutePath, relativePath);
3324
+ pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
3325
+ }
3326
+ if (pendingWatcherPaths.size > 0) {
3327
+ pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3328
+ pendingWatcherWarmupCatchup = true;
3329
+ emitSyncWorkState(true);
3330
+ process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
3331
+ }
3332
+ }
3333
+ catch (err) {
3334
+ if (!(err instanceof Error && err.code === "ENOENT")) {
3335
+ process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
3336
+ }
3337
+ }
3338
+ }
2679
3339
  const routePushFailureBackoff = new RoutePushFailureBackoff();
2680
3340
  const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
2681
3341
  const routePushRetryTimerHandles = new Map();
@@ -3256,14 +3916,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3256
3916
  const relativePath = route.kind === "personal"
3257
3917
  ? key
3258
3918
  : ["companies", route.slug, key].join("/");
3259
- const localPath = path.resolve(hqRoot, ...relativePath.split("/"));
3260
- const fromRoot = path.relative(hqRoot, localPath);
3261
- const parentPrefix = ["..", path.sep].join("");
3262
- if (fromRoot === ".." ||
3263
- fromRoot.startsWith(parentPrefix) ||
3264
- path.isAbsolute(fromRoot)) {
3919
+ // Journal keys are POSIX relative paths. Reject unsafe segments with
3920
+ // a string-only check before building the absolute path; path.resolve
3921
+ // plus path.relative allocated and normalized every journal row.
3922
+ if (!isSafeRelativePath(relativePath))
3265
3923
  continue;
3266
- }
3924
+ const localPath = path.join(hqRoot, relativePath);
3267
3925
  if (shouldSync(localPath)) {
3268
3926
  paths.push(relativePath);
3269
3927
  continue;
@@ -3526,8 +4184,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3526
4184
  // a file and still be refused as stale (a download changed the directory
3527
4185
  // mid-pass); the retry then finds the file already in the cloud and uploads
3528
4186
  // nothing, so without this the file is never announced to peers.
3529
- const uploadedAwaitingPublish = new Set();
3530
- const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
3531
4187
  const rememberUploadedPaths = (result) => {
3532
4188
  if (result === undefined || typeof result === "number")
3533
4189
  return;
@@ -3540,6 +4196,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3540
4196
  const oldest = uploadedAwaitingPublish.values().next().value;
3541
4197
  if (oldest !== undefined)
3542
4198
  uploadedAwaitingPublish.delete(oldest);
4199
+ if (!uploadedAwaitingPublishDropLogged) {
4200
+ uploadedAwaitingPublishDropLogged = true;
4201
+ process.stderr.write(`${JSON.stringify({
4202
+ type: "event-sync-upload-buffer-overflow",
4203
+ cap: UPLOADED_AWAITING_PUBLISH_CAP,
4204
+ action: "drop-oldest",
4205
+ })}\n`);
4206
+ }
3543
4207
  }
3544
4208
  }
3545
4209
  };
@@ -3654,19 +4318,48 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3654
4318
  ? localUploadSizeClass(uploadPaths)
3655
4319
  : "small";
3656
4320
  const largeUpload = sizeClass !== "small";
3657
- const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
3658
- const execution = parallelCoordinatorEnabled && !chorePush
3659
- ? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
3660
- const admittedPaths = slice.paths.map((path) => path.canonicalPath);
3661
- const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
3662
- const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
3663
- return runGuarded(scopedArgv, admittedChanges.size > 0
3664
- ? () => prepareWatcherChanges(admittedChanges)
3665
- : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
3666
- }, sizeClass)
3667
- : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
3668
- ? () => prepareWatcherChanges(group.changes)
3669
- : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
4321
+ // With coordinator fencing enabled, even large watcher batches can
4322
+ // use bounded realtime admission. Keeping them as chores would make
4323
+ // them wait behind a long-lived receiver pull's slot indefinitely.
4324
+ const chorePush = isChoreLocalPush(group.relPaths, largeUpload && !parallelCoordinatorEnabled);
4325
+ const { groupId, setStage: watcherPushStage } = startWatcherPushStageTracking(routeId, group.relPaths.length);
4326
+ watcherPushStage("coordinator submit");
4327
+ inFlightWatcherPushes += group.relPaths.length;
4328
+ emitSyncWorkState(true);
4329
+ emitWatcherPushInflightStages();
4330
+ let execution;
4331
+ try {
4332
+ execution = await (parallelCoordinatorEnabled && !chorePush
4333
+ ? runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
4334
+ watcherPushStage("worker entry");
4335
+ const admittedPaths = slice.paths.map((path) => path.canonicalPath);
4336
+ const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
4337
+ const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
4338
+ return runGuarded(scopedArgv, admittedChanges.size > 0
4339
+ ? () => prepareWatcherChanges(admittedChanges)
4340
+ : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
4341
+ // Watcher-originated pushes may share the guarded
4342
+ // realtime slot with an in-flight pull. Coordinator
4343
+ // fencing already serializes overlapping path epochs;
4344
+ // this must not depend on the optional full-pass flag.
4345
+ true, undefined, routeId, watcherPushStage);
4346
+ }, sizeClass)
4347
+ : (() => {
4348
+ watcherPushStage("worker entry");
4349
+ return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
4350
+ ? () => prepareWatcherChanges(group.changes)
4351
+ : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
4352
+ result,
4353
+ accepted: true,
4354
+ coordinatorRefusedPaths: [],
4355
+ }));
4356
+ })());
4357
+ }
4358
+ finally {
4359
+ watcherPushStages.delete(groupId);
4360
+ inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
4361
+ emitSyncWorkState(true);
4362
+ }
3670
4363
  const result = execution.result;
3671
4364
  rememberUploadedPaths(result);
3672
4365
  const pushExit = passExitCode(result);
@@ -3685,6 +4378,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3685
4378
  }
3686
4379
  const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
3687
4380
  relativePath.startsWith(`${refusedPath}/`));
4381
+ acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
3688
4382
  const publishPaths = new Map();
3689
4383
  for (const [absolutePath, relativePath] of group.paths) {
3690
4384
  if (isRefusedPath(relativePath))
@@ -3724,9 +4418,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3724
4418
  deferPushAfterAdmissionTimeout(group, routeId);
3725
4419
  }
3726
4420
  else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
4421
+ acknowledgeDurableWatcherUploads(group.relPaths);
3727
4422
  routePushFailureBackoff.succeeded(routeId);
3728
4423
  clearRoutePushRetry(routeId);
3729
4424
  if (group.route.kind === "company") {
4425
+ deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
3730
4426
  logDroppedUnresolvedWatcherRoute(group.route.slug);
3731
4427
  }
3732
4428
  }
@@ -3788,7 +4484,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3788
4484
  const execution = parallelCoordinatorEnabled
3789
4485
  ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
3790
4486
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
3791
- return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
4487
+ return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route));
3792
4488
  })
3793
4489
  : await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
3794
4490
  if (error instanceof RealtimeAdmissionTimeoutError) {
@@ -3840,6 +4536,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3840
4536
  addRoute(hint);
3841
4537
  const skipCompanies = resolveSkipCompanies();
3842
4538
  if (parallelCoordinatorEnabled) {
4539
+ let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
3843
4540
  for (const route of routes.values()) {
3844
4541
  if (route.kind === "company" &&
3845
4542
  skipCompanies.has(route.slug.toLowerCase())) {
@@ -3868,6 +4565,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3868
4565
  process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
3869
4566
  interruptWait();
3870
4567
  }
4568
+ else {
4569
+ routesRemaining -= 1;
4570
+ if (routesRemaining === 0)
4571
+ clearDurableWatcherOverflow();
4572
+ }
3871
4573
  });
3872
4574
  }
3873
4575
  return 0;
@@ -3983,6 +4685,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3983
4685
  // churn overflowed the 200-path batch and skipped publishBatch.
3984
4686
  personalMode: true,
3985
4687
  includeCompanyPaths: companiesMode || companyMode,
4688
+ // Inventory filtered candidates so the route's push filter remains
4689
+ // authoritative and can attribute exclusions in warm-up diagnostics.
4690
+ includeFilteredWarmupCandidates: true,
3986
4691
  onDegraded: opts.onDegraded,
3987
4692
  captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
3988
4693
  queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
@@ -4104,6 +4809,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4104
4809
  }
4105
4810
  }
4106
4811
  watcher.start();
4812
+ let catchupPathStatWarningReported = false;
4107
4813
  const reportWatcherReady = () => {
4108
4814
  if (stopped || watcher === null)
4109
4815
  return;
@@ -4144,14 +4850,202 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4144
4850
  : watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
4145
4851
  if (!catchup || catchup.cancelled || stopped || watcher === null)
4146
4852
  return;
4147
- const queued = addPendingWatcherChange(undefined, catchup.batch, {
4853
+ const skipStore = deps.watcherCatchupSkipStore;
4854
+ const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
4855
+ const eligiblePaths = new Map();
4856
+ const eligibleChanges = new Map();
4857
+ const excludedByReason = {};
4858
+ const excludedByReasonByRoute = {};
4859
+ const personalFilterExclusions = new Map();
4860
+ const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
4861
+ const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
4862
+ const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
4863
+ const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
4864
+ const clearedResolvedCompanies = new Set();
4865
+ const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
4866
+ const candidateRouteCounts = new Map();
4867
+ const candidateRoutePrefixes = new Map();
4868
+ const eligibleRouteCounts = new Map();
4869
+ const eligibleRoutePrefixes = new Map();
4870
+ const backupClass = (relativePath) => {
4871
+ if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
4872
+ return "attioCleanup";
4873
+ if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
4874
+ return "readModelRollout";
4875
+ return undefined;
4876
+ };
4877
+ const exclude = (reason, routeName) => {
4878
+ excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
4879
+ const routeReasons = excludedByReasonByRoute[routeName] ??= {};
4880
+ routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
4881
+ };
4882
+ let filteredCandidateCount = 0;
4883
+ for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
4884
+ filteredCandidateCount += count;
4885
+ excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
4886
+ const routeReasons = excludedByReasonByRoute.filtered ??= {};
4887
+ routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
4888
+ }
4889
+ if (filteredCandidateCount > 0)
4890
+ candidateRouteCounts.set("filtered", filteredCandidateCount);
4891
+ let examinedWarmupPaths = 0;
4892
+ for (const [absolutePath, relativePath] of catchup.batch.paths) {
4893
+ examinedWarmupPaths += 1;
4894
+ const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
4895
+ if (warmupYield)
4896
+ await warmupYield;
4897
+ if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
4898
+ recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
4899
+ exclude("unsafe-path", "unrouted");
4900
+ continue;
4901
+ }
4902
+ const route = routeChangeToTarget(relativePath);
4903
+ const routeName = route ? routeKey(route) : "unrouted";
4904
+ const summaryRoutePath = route?.kind === "company"
4905
+ ? relativePath.slice(`companies/${route.slug}/`.length)
4906
+ : relativePath;
4907
+ recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
4908
+ const ontologyBackupClass = backupClass(relativePath);
4909
+ if (ontologyBackupClass) {
4910
+ exclude("local-rule:ontology-backup", routeName);
4911
+ ontologyBackupPaths[ontologyBackupClass] += 1;
4912
+ continue;
4913
+ }
4914
+ if (route?.kind === "company") {
4915
+ const resolvedNow = cloudCompanySlugs.has(route.slug) ||
4916
+ skipStore?.hasCurrentGrantCompany(route.slug) ||
4917
+ skipStore?.hasCurrentGrantVersion(route.slug) ||
4918
+ (companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
4919
+ if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
4920
+ skipStore?.clearUnresolvedCompany(route.slug);
4921
+ clearedResolvedCompanies.add(route.slug);
4922
+ }
4923
+ if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
4924
+ exclude("unresolved-company", routeName);
4925
+ continue;
4926
+ }
4927
+ }
4928
+ if (route?.kind === "company" &&
4929
+ !cloudCompanySlugs.has(route.slug) &&
4930
+ !skipStore?.hasCurrentGrantCompany(route.slug) &&
4931
+ !skipStore?.hasCurrentGrantVersion(route.slug) &&
4932
+ !(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
4933
+ exclude("local-only-company", routeName);
4934
+ continue;
4935
+ }
4936
+ const company = route?.kind === "company" ? route.slug : "personal";
4937
+ const routePath = route?.kind === "company"
4938
+ ? relativePath.slice(`companies/${route.slug}/`.length)
4939
+ : relativePath;
4940
+ if (isEphemeralPath(routePath)) {
4941
+ exclude("push-filter:conflict-mirror", routeName);
4942
+ continue;
4943
+ }
4944
+ // The production TreeWatcher collector has already applied this
4945
+ // shared filter stack and reports rejected files as reason counts.
4946
+ // Injected/custom collectors without that proof still evaluate the
4947
+ // route filter here, with personal defaults first for `.env` attribution.
4948
+ const personalDefaultExclusion = catchup.pushFiltersApplied === true
4949
+ ? undefined
4950
+ : route?.kind === "personal"
4951
+ ? matchPersonalVaultExclusion(relativePath, false)
4952
+ : undefined;
4953
+ const isPersonalRoute = route?.kind !== "company";
4954
+ const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
4955
+ ? personalPushFilter(absolutePath, false, relativePath)
4956
+ : ignoreFilter(absolutePath, false, relativePath)));
4957
+ if (!acceptedByPushFilter) {
4958
+ const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
4959
+ ? personalFilterExclusions.get(relativePath)
4960
+ : undefined);
4961
+ exclude(exclusionId
4962
+ ? `push-filter:personal:${exclusionId}`
4963
+ : "push-filter:ignore", routeName);
4964
+ continue;
4965
+ }
4966
+ if (isPersonalRoute &&
4967
+ !shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
4968
+ isPersonalVaultScopeExcluded(relativePath, {
4969
+ continuityPaths,
4970
+ hasAgencyCarveout,
4971
+ hasSessionLogsCarveout,
4972
+ })) {
4973
+ exclude("push-filter:personal:excluded-top-level", routeName);
4974
+ continue;
4975
+ }
4976
+ if (route && isLocallyIneligibleForPush(routePath, {
4977
+ boardClientOwned: true,
4978
+ ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
4979
+ })) {
4980
+ const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
4981
+ exclude(`local-rule:${pathClass}`, routeName);
4982
+ continue;
4983
+ }
4984
+ const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
4985
+ const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
4986
+ try {
4987
+ const stat = !catchup.symlinkPaths || hasSkipRecord
4988
+ ? fs.lstatSync(absolutePath)
4989
+ : undefined;
4990
+ if (route && (knownSymlink || stat?.isSymbolicLink())) {
4991
+ const scopeRoot = route.kind === "company"
4992
+ ? path.join(hqRoot, "companies", route.slug)
4993
+ : relativePath.startsWith("personal/")
4994
+ ? path.join(hqRoot, "personal")
4995
+ : hqRoot;
4996
+ const inspection = inspectLocalSymlinkScope({
4997
+ absoluteLinkPath: absolutePath,
4998
+ linkPath: relativePath,
4999
+ scopeRoot,
5000
+ scope: route.kind === "company"
5001
+ ? { kind: "company", companySlug: route.slug }
5002
+ : { kind: "personal" },
5003
+ });
5004
+ if (inspection.refused) {
5005
+ exclude("symlink", routeName);
5006
+ continue;
5007
+ }
5008
+ }
5009
+ if (hasSkipRecord && stat &&
5010
+ skipStore?.matches(company, routePath, stat)) {
5011
+ exclude("record-match", routeName);
5012
+ continue;
5013
+ }
5014
+ }
5015
+ catch (error) {
5016
+ // Fail open: unreadable state or a filesystem race must not suppress a candidate.
5017
+ if (!catchupPathStatWarningReported) {
5018
+ catchupPathStatWarningReported = true;
5019
+ process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
5020
+ }
5021
+ }
5022
+ eligiblePaths.set(absolutePath, relativePath);
5023
+ recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
5024
+ const change = catchup.batch.changes?.get(absolutePath);
5025
+ if (change)
5026
+ eligibleChanges.set(absolutePath, change);
5027
+ }
5028
+ const queued = addPendingWatcherChange(undefined, {
5029
+ paths: eligiblePaths,
5030
+ changes: eligibleChanges,
5031
+ }, {
4148
5032
  preserveChangesAfter: liveChangeSequenceAtWarmupStart,
4149
5033
  });
4150
5034
  pendingWatcherWarmupCatchup ||= queued;
5035
+ const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
4151
5036
  process.stderr.write(JSON.stringify({
4152
5037
  type: "watcher-warmup-catchup",
4153
5038
  reason: "watch-set-ready",
4154
- pathsFound: catchup.batch.paths.size,
5039
+ pathsFound: eligiblePaths.size,
5040
+ candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
5041
+ pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
5042
+ pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
5043
+ excludedByReason: {
5044
+ ...excludedByReason,
5045
+ ...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
5046
+ },
5047
+ excludedByReasonByRoute,
5048
+ ontologyBackupPaths,
4155
5049
  scannedPaths: catchup.scannedPaths,
4156
5050
  elapsedMs: Date.now() - startedAt,
4157
5051
  queued,
@@ -4218,13 +5112,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4218
5112
  const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
4219
5113
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
4220
5114
  const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
4221
- return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
5115
+ return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal, routeKey(route)));
4222
5116
  });
4223
5117
  const pathCount = events.length;
4224
5118
  // Rethrow the typed refusal so the receiver keeps the lease and
4225
5119
  // retries the whole batch, instead of bisecting a "failed" pull.
4226
5120
  if (execution.admissionTimeout)
4227
5121
  throw execution.admissionTimeout;
5122
+ if (execution.yieldedToPush) {
5123
+ throw new RealtimePushYieldError();
5124
+ }
4228
5125
  const exitCode = passExitCode(execution.result);
4229
5126
  if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
4230
5127
  if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
@@ -4246,74 +5143,82 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4246
5143
  const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
4247
5144
  if (stopped)
4248
5145
  return;
4249
- // The injectable receiver seam and the production SQS receiver both
4250
- // arrive here only after a successful receive. Keep the cadence tight
4251
- // until the receiver later reports an empty outer long-poll.
4252
- onReceiveActivity(true);
4253
- // A received wake/high-water signal means there is bounded work now.
4254
- // It wakes the scheduler but is still funneled through runGuarded below,
4255
- // so it cannot overlap the pass that may currently own the operation
4256
- // lock.
4257
- // Apply the caps once across the whole delivery, then retain the residue
4258
- // in the normal lane. The receiver already coalesces to one latest event
4259
- // per path, so the partition cannot split revisions of one path.
4260
- if (parallelCoordinatorEnabled) {
4261
- const lane = receiverLaneByBatch.get(events);
4262
- if (lane) {
4263
- receiverLaneByBatch.delete(events);
4264
- await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
5146
+ try {
5147
+ // The injectable receiver seam and the production SQS receiver both
5148
+ // arrive here only after a successful receive. Keep the cadence tight
5149
+ // until the receiver later reports an empty outer long-poll.
5150
+ onReceiveActivity(true);
5151
+ pendingReceivers += events.length;
5152
+ emitSyncWorkState();
5153
+ // A received wake/high-water signal means there is bounded work now.
5154
+ // It wakes the scheduler but is still funneled through runGuarded below,
5155
+ // so it cannot overlap the pass that may currently own the operation
5156
+ // lock.
5157
+ // Apply the caps once across the whole delivery, then retain the residue
5158
+ // in the normal lane. The receiver already coalesces to one latest event
5159
+ // per path, so the partition cannot split revisions of one path.
5160
+ if (parallelCoordinatorEnabled) {
5161
+ const lane = receiverLaneByBatch.get(events);
5162
+ if (lane) {
5163
+ receiverLaneByBatch.delete(events);
5164
+ await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
5165
+ return;
5166
+ }
5167
+ // Direct embedders may invoke the engine without first going through
5168
+ // SqsPushReceiver. Preserve that seam while applying the same
5169
+ // delivery-wide partition the receiver would expose.
5170
+ const lanes = partitionByKnownSizes(events, receiverEventSize);
5171
+ await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
5172
+ await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4265
5173
  return;
4266
5174
  }
4267
- // Direct embedders may invoke the engine without first going through
4268
- // SqsPushReceiver. Preserve that seam while applying the same
4269
- // delivery-wide partition the receiver would expose.
4270
5175
  const lanes = partitionByKnownSizes(events, receiverEventSize);
4271
- await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
4272
- await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4273
- return;
4274
- }
4275
- const lanes = partitionByKnownSizes(events, receiverEventSize);
4276
- const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
4277
- const eventsByRoute = new Map();
4278
- for (const event of laneEvents) {
4279
- const route = routeChangeToTarget(event.relativePath);
4280
- // The receiver is an independent ingress from the local watcher.
4281
- // Apply the same route-selection boundary before it can enqueue a
4282
- // fast pull: a --skip-personal runner must acknowledge, not pull,
4283
- // a personal-namespace delivery.
4284
- if (!route || !isWatchRouteSelected(route, watchRouteSelection))
4285
- continue;
4286
- const key = routeKey(route);
4287
- const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
4288
- group.events.push(event);
4289
- if (event.kind === "upsert")
4290
- group.upsertPaths.push(event.relativePath);
4291
- eventsByRoute.set(key, group);
4292
- }
4293
- for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
4294
- const paths = routeEvents.map((event) => event.relativePath);
4295
- const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
4296
- const exitCode = passExitCode(result);
4297
- if (exitCode === UNRESOLVED_COMPANY_EXIT) {
4298
- if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
5176
+ const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
5177
+ const eventsByRoute = new Map();
5178
+ for (const event of laneEvents) {
5179
+ const route = routeChangeToTarget(event.relativePath);
5180
+ // The receiver is an independent ingress from the local watcher.
5181
+ // Apply the same route-selection boundary before it can enqueue a
5182
+ // fast pull: a --skip-personal runner must acknowledge, not pull,
5183
+ // a personal-namespace delivery.
5184
+ if (!route || !isWatchRouteSelected(route, watchRouteSelection))
5185
+ continue;
5186
+ const key = routeKey(route);
5187
+ const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
5188
+ group.events.push(event);
5189
+ if (event.kind === "upsert")
5190
+ group.upsertPaths.push(event.relativePath);
5191
+ eventsByRoute.set(key, group);
5192
+ }
5193
+ for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
5194
+ const paths = routeEvents.map((event) => event.relativePath);
5195
+ const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
5196
+ const exitCode = passExitCode(result);
5197
+ if (exitCode === UNRESOLVED_COMPANY_EXIT) {
5198
+ if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
5199
+ throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
5200
+ }
5201
+ warnUnresolvedReceiverCompany(route, routeEvents.length);
5202
+ await settleEvents?.(routeEvents);
5203
+ continue;
5204
+ }
5205
+ if (exitCode !== 0) {
4299
5206
  throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4300
5207
  }
4301
- warnUnresolvedReceiverCompany(route, routeEvents.length);
4302
5208
  await settleEvents?.(routeEvents);
4303
- continue;
4304
- }
4305
- if (exitCode !== 0) {
4306
- throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4307
5209
  }
4308
- await settleEvents?.(routeEvents);
4309
- }
4310
- };
4311
- // Dispatch realtime work before awaiting its bulk residue. This lets an
4312
- // established slow pass continue while the small scoped pull uses its
4313
- // ref-counted lock carve-out; the residue remains serialized and cannot
4314
- // starve the slow lane.
4315
- await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
4316
- await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
5210
+ };
5211
+ // Dispatch realtime work before awaiting its bulk residue. This lets an
5212
+ // established slow pass continue while the small scoped pull uses its
5213
+ // ref-counted lock carve-out; the residue remains serialized and cannot
5214
+ // starve the slow lane.
5215
+ await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
5216
+ await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
5217
+ }
5218
+ finally {
5219
+ pendingReceivers = Math.max(0, pendingReceivers - events.length);
5220
+ emitSyncWorkState();
5221
+ }
4317
5222
  };
4318
5223
  receiverSyncBatchFn.partition = (events, maxPaths) => {
4319
5224
  const lanes = partitionByKnownSizes(events, receiverEventSize);
@@ -4470,6 +5375,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4470
5375
  if (stopped)
4471
5376
  return shutdownPromise ?? Promise.resolve();
4472
5377
  stopped = true;
5378
+ clearInterval(watcherPushStagesTimer);
5379
+ if (durableWatcherPersistTimer !== undefined)
5380
+ clearTimeout(durableWatcherPersistTimer);
5381
+ durableWatcherPersistTimer = undefined;
5382
+ if (syncWorkStateTimer !== undefined)
5383
+ clearTimeout(syncWorkStateTimer);
5384
+ syncWorkStateTimer = undefined;
5385
+ emitSyncWorkState(true);
4473
5386
  for (const route of routePushRetryTimerHandles.keys()) {
4474
5387
  clearRoutePushRetry(route);
4475
5388
  }
@@ -4499,6 +5412,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4499
5412
  coordinatorDisposal,
4500
5413
  boundedEventSyncStartup,
4501
5414
  eventSyncDisposal,
5415
+ persistDurableWatcherPaths(),
4502
5416
  ]).then(() => undefined);
4503
5417
  try {
4504
5418
  void receiver?.dispose();
@@ -4916,6 +5830,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4916
5830
  lastCompletedFullReconcileWallAt = Date.now();
4917
5831
  fullPassJournalHealthy = execution.accepted &&
4918
5832
  isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
5833
+ if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
5834
+ clearDurableWatcherOverflow();
5835
+ }
5836
+ else if (fullPassJournalHealthy && pending.batch !== null) {
5837
+ acknowledgeDurableWatcherPathsCoveredByFullPass([
5838
+ ...pending.batch.paths.values(),
5839
+ ]);
5840
+ }
4919
5841
  if (fullPassJournalHealthy) {
4920
5842
  fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
4921
5843
  }
@@ -4973,7 +5895,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4973
5895
  const catchupBatch = workingBatch;
4974
5896
  const catchupGeneration = pending.generation;
4975
5897
  void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
4976
- if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
5898
+ if (exit === 0) {
5899
+ clearDurableWatcherOverflow();
5900
+ return;
5901
+ }
5902
+ if (exit === SCOPED_DRAIN_DEFERRED)
4977
5903
  return;
4978
5904
  restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
4979
5905
  interruptWait();
@@ -5081,6 +6007,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5081
6007
  ? exitCode === 0
5082
6008
  : hasCompletedRunnerPass(result));
5083
6009
  if (completedPostRepairPass) {
6010
+ if (!parallelCoordinatorEnabled &&
6011
+ pending.batch?.overflowed === true &&
6012
+ ((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
6013
+ clearDurableWatcherOverflow();
6014
+ }
6015
+ else if (!parallelCoordinatorEnabled &&
6016
+ ranFullReconcile &&
6017
+ fullPassJournalHealthy &&
6018
+ pending.batch !== null) {
6019
+ acknowledgeDurableWatcherPathsCoveredByFullPass([
6020
+ ...pending.batch.paths.values(),
6021
+ ]);
6022
+ }
5084
6023
  // The production adapter returns RunnerPassResult only after fanout,
5085
6024
  // which follows journal repair. Its numeric seam has the same
5086
6025
  // contract: zero means the injected pass completed. This preserves