@indigoai-us/hq-cloud 6.18.53 → 6.18.56-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/dist/bin/sync-runner-audit.d.ts +26 -0
  2. package/dist/bin/sync-runner-audit.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-audit.js +51 -0
  4. package/dist/bin/sync-runner-audit.js.map +1 -0
  5. package/dist/bin/sync-runner-company.d.ts +18 -0
  6. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-company.js +186 -9
  8. package/dist/bin/sync-runner-company.js.map +1 -1
  9. package/dist/bin/sync-runner-watch-loop.d.ts +107 -5
  10. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-watch-loop.js +1382 -183
  12. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  13. package/dist/bin/sync-runner.d.ts +63 -11
  14. package/dist/bin/sync-runner.d.ts.map +1 -1
  15. package/dist/bin/sync-runner.js +350 -95
  16. package/dist/bin/sync-runner.js.map +1 -1
  17. package/dist/bin/watcher-catchup-skip-store.d.ts +60 -0
  18. package/dist/bin/watcher-catchup-skip-store.d.ts.map +1 -0
  19. package/dist/bin/watcher-catchup-skip-store.js +282 -0
  20. package/dist/bin/watcher-catchup-skip-store.js.map +1 -0
  21. package/dist/cli/conflict.d.ts +6 -0
  22. package/dist/cli/conflict.d.ts.map +1 -1
  23. package/dist/cli/conflict.js +3 -1
  24. package/dist/cli/conflict.js.map +1 -1
  25. package/dist/cli/reindex.d.ts.map +1 -1
  26. package/dist/cli/reindex.js +95 -5
  27. package/dist/cli/reindex.js.map +1 -1
  28. package/dist/cli/share.d.ts +27 -0
  29. package/dist/cli/share.d.ts.map +1 -1
  30. package/dist/cli/share.js +140 -23
  31. package/dist/cli/share.js.map +1 -1
  32. package/dist/cli/sync.d.ts +18 -2
  33. package/dist/cli/sync.d.ts.map +1 -1
  34. package/dist/cli/sync.js +90 -25
  35. package/dist/cli/sync.js.map +1 -1
  36. package/dist/ignore.d.ts +1 -1
  37. package/dist/ignore.d.ts.map +1 -1
  38. package/dist/ignore.js +2 -2
  39. package/dist/ignore.js.map +1 -1
  40. package/dist/journal-row-store.d.ts.map +1 -1
  41. package/dist/journal-row-store.js +6 -0
  42. package/dist/journal-row-store.js.map +1 -1
  43. package/dist/journal-snapshot-v4.d.ts +8 -0
  44. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  45. package/dist/journal-snapshot-v4.js +208 -17
  46. package/dist/journal-snapshot-v4.js.map +1 -1
  47. package/dist/journal.d.ts +18 -2
  48. package/dist/journal.d.ts.map +1 -1
  49. package/dist/journal.js +248 -17
  50. package/dist/journal.js.map +1 -1
  51. package/dist/lib/cloud-authoritative.d.ts +9 -2
  52. package/dist/lib/cloud-authoritative.d.ts.map +1 -1
  53. package/dist/lib/cloud-authoritative.js +34 -2
  54. package/dist/lib/cloud-authoritative.js.map +1 -1
  55. package/dist/lib/exit-codes.d.ts +2 -0
  56. package/dist/lib/exit-codes.d.ts.map +1 -1
  57. package/dist/lib/exit-codes.js +2 -0
  58. package/dist/lib/exit-codes.js.map +1 -1
  59. package/dist/personal-vault-exclusions.d.ts +2 -2
  60. package/dist/personal-vault-exclusions.d.ts.map +1 -1
  61. package/dist/personal-vault-exclusions.js +5 -3
  62. package/dist/personal-vault-exclusions.js.map +1 -1
  63. package/dist/personal-vault.d.ts +7 -0
  64. package/dist/personal-vault.d.ts.map +1 -1
  65. package/dist/personal-vault.js +10 -0
  66. package/dist/personal-vault.js.map +1 -1
  67. package/dist/qmd-runner-admission.d.ts +22 -0
  68. package/dist/qmd-runner-admission.d.ts.map +1 -0
  69. package/dist/qmd-runner-admission.js +107 -0
  70. package/dist/qmd-runner-admission.js.map +1 -0
  71. package/dist/symlink-target-policy.d.ts +11 -0
  72. package/dist/symlink-target-policy.d.ts.map +1 -1
  73. package/dist/symlink-target-policy.js +110 -0
  74. package/dist/symlink-target-policy.js.map +1 -1
  75. package/dist/sync/area-ledger-migration.d.ts +3 -1
  76. package/dist/sync/area-ledger-migration.d.ts.map +1 -1
  77. package/dist/sync/area-ledger-migration.js +3 -0
  78. package/dist/sync/area-ledger-migration.js.map +1 -1
  79. package/dist/sync/area-ledger.d.ts +3 -0
  80. package/dist/sync/area-ledger.d.ts.map +1 -1
  81. package/dist/sync/area-ledger.js +29 -6
  82. package/dist/sync/area-ledger.js.map +1 -1
  83. package/dist/sync/mutation-session.d.ts.map +1 -1
  84. package/dist/sync/mutation-session.js +4 -0
  85. package/dist/sync/mutation-session.js.map +1 -1
  86. package/dist/sync/parallel-sync-coordinator.d.ts +29 -0
  87. package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
  88. package/dist/sync/parallel-sync-coordinator.js +229 -16
  89. package/dist/sync/parallel-sync-coordinator.js.map +1 -1
  90. package/dist/sync/push-receiver.d.ts +5 -0
  91. package/dist/sync/push-receiver.d.ts.map +1 -1
  92. package/dist/sync/push-receiver.js +32 -10
  93. package/dist/sync/push-receiver.js.map +1 -1
  94. package/dist/sync/state-store.d.ts +7 -0
  95. package/dist/sync/state-store.d.ts.map +1 -1
  96. package/dist/sync/state-store.js +113 -6
  97. package/dist/sync/state-store.js.map +1 -1
  98. package/dist/watch-owner-lease.d.ts +4 -0
  99. package/dist/watch-owner-lease.d.ts.map +1 -1
  100. package/dist/watch-owner-lease.js +40 -6
  101. package/dist/watch-owner-lease.js.map +1 -1
  102. package/dist/watcher.d.ts +16 -3
  103. package/dist/watcher.d.ts.map +1 -1
  104. package/dist/watcher.js +114 -21
  105. package/dist/watcher.js.map +1 -1
  106. package/package.json +4 -3
@@ -1,6 +1,7 @@
1
1
  import * as os from "os";
2
2
  import * as path from "path";
3
3
  import * as fs from "fs";
4
+ import { createHash } from "node:crypto";
4
5
  import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
5
6
  import * as v8 from "node:v8";
6
7
  import { performance } from "node:perf_hooks";
@@ -15,13 +16,18 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
15
16
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
16
17
  import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
17
18
  import { localPathForVaultKey } from "../local-path-codec.js";
19
+ import { isEphemeralPath } from "../cli/share.js";
20
+ import { isSafeRelativePath } from "../sync/push-event.js";
21
+ import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
22
+ import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
18
23
  import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
19
24
  import { createIgnoreFilter } from "../ignore.js";
25
+ import { matchPersonalVaultExclusion, wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
20
26
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
21
27
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
22
- import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
23
- import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
24
- import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
28
+ import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, readQuarantinedJournalPaths, hashSymlinkTarget, applyJournalDeltaAsync, } from "../journal.js";
29
+ import { isPersonalVaultDeletionExcluded, readCloudBackedSlugsFromManifest, computeContinuityPointerPaths, computeAgencySyncPaths, computeSessionLogsSyncPaths, isPersonalVaultScopeExcluded, } from "../personal-vault.js";
30
+ import { NoopPushReceiver, ReceiverDispatchError, RealtimePushYieldError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
25
31
  import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
26
32
  import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
27
33
  import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
@@ -44,6 +50,36 @@ const FULL_RECONCILE_EVERY_TICKS = 10;
44
50
  const warnedUnresolvedReceiverCompanies = new Set();
45
51
  /** Maximum time shutdown waits for event-sync startup before abandoning it. */
46
52
  export const EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS = 5_000;
53
+ /** Group watcher changes by target route while retaining source insertion order. */
54
+ export function groupBatchByRoute(batch) {
55
+ const grouped = new Map();
56
+ for (const [absolutePath, relativePath] of batch.paths.entries()) {
57
+ const change = batch.changes?.get(absolutePath);
58
+ const route = routeChangeToTarget(relativePath, change?.kind);
59
+ if (!route)
60
+ continue;
61
+ const category = isSessionLogsPushOnlyPath(relativePath) ? "session-log-backup" : "interactive";
62
+ const key = `${routeKey(route)}:${category}`;
63
+ const existing = grouped.get(key) ?? {
64
+ route,
65
+ paths: new Map(),
66
+ changes: new Map(),
67
+ relPaths: [],
68
+ deleteScopeRoots: [],
69
+ };
70
+ existing.paths.set(absolutePath, relativePath);
71
+ if (change)
72
+ existing.changes.set(absolutePath, change);
73
+ grouped.set(key, existing);
74
+ }
75
+ for (const group of grouped.values()) {
76
+ group.relPaths = [...new Set(group.paths.values())];
77
+ group.deleteScopeRoots = [
78
+ ...new Set([...group.changes.values()].flatMap((item) => (item.deleteSnapshots ?? []).flatMap((snapshot) => snapshot.deleteScopeRoot === undefined ? [] : [snapshot.deleteScopeRoot]))),
79
+ ];
80
+ }
81
+ return grouped;
82
+ }
47
83
  /** A deferred full chore must get the next admission after this long. */
48
84
  export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
49
85
  /**
@@ -61,13 +97,13 @@ export const REALTIME_ADMISSION_DEADLINE_MS = 2 * DEFAULT_REALTIME_AUTHORIZATION
61
97
  */
62
98
  export const REALTIME_DRAIN_BURST_MS = REALTIME_ADMISSION_DEADLINE_MS;
63
99
  /**
64
- * How many realtime executes may run during one chore hand-off.
100
+ * Maximum guarded realtime executes, including the reserved watcher push peer.
65
101
  * HQ_SYNC_MAX_CONCURRENCY caps simultaneous S3 object transfers, not these
66
- * workers. One long transfer keeps its worker; a realtime item that arrives
67
- * while that execute() is in flight starts on the other slot instead of
68
- * waiting for the whole batch. In-flight work is never aborted.
102
+ * workers. One long pull and one long push may each retain a normal worker;
103
+ * one additional watcher push can still make bounded progress. In-flight
104
+ * work is never aborted for this admission.
69
105
  */
70
- export const REALTIME_WORKER_OCCUPANCY_CAP = 2;
106
+ export const REALTIME_WORKER_OCCUPANCY_CAP = 3;
71
107
  /** Objects at least one multipart part are sent through the chore hand-off. */
72
108
  export const REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES = MULTIPART_PART_SIZE_BYTES;
73
109
  let realtimeAdmissionDeadlineMsForTest;
@@ -91,6 +127,29 @@ function realtimeDrainBurstMs() {
91
127
  * so a long chore costs at most one attempt per deadline plus this delay.
92
128
  */
93
129
  export const REALTIME_ADMISSION_RETRY_MS = 5_000;
130
+ /** Bound consecutive pull admissions while a watcher push is queued. */
131
+ export const MAX_PULL_ADMISSIONS_BEFORE_PUSH = 3;
132
+ /** Bound synchronous warm-up candidate work between event-loop turns. */
133
+ export const WATCHER_WARMUP_YIELD_EVERY = 128;
134
+ export function shouldYieldWatcherWarmup(examinedPaths) {
135
+ return examinedPaths > 0 && examinedPaths % WATCHER_WARMUP_YIELD_EVERY === 0;
136
+ }
137
+ export function yieldWatcherWarmupIfNeeded(examinedPaths, yieldNow = () => new Promise((resolve) => setImmediate(resolve))) {
138
+ return shouldYieldWatcherWarmup(examinedPaths) ? yieldNow() : undefined;
139
+ }
140
+ export function shouldSkipPersonalWarmupScopeCheck(pushFiltersApplied, routeKind) {
141
+ // TreeWatcher marks this only when its production filter has already applied
142
+ // the same personal scope predicate to every collected candidate. Unrouted
143
+ // paths retain the runner check because the collector may include company paths.
144
+ return pushFiltersApplied && routeKind === "personal";
145
+ }
146
+ export function createWatcherWarmupPathFilters(hqRoot, onPersonalExcluded, createIgnore = createIgnoreFilter, wrapPersonal = wrapFilterWithPersonalVaultDefaults) {
147
+ const ignoreFilter = createIgnore(hqRoot);
148
+ const personalPushFilter = wrapPersonal(ignoreFilter, hqRoot, (relativePath, match) => {
149
+ onPersonalExcluded(relativePath, match.id);
150
+ });
151
+ return { ignoreFilter, personalPushFilter };
152
+ }
94
153
  /**
95
154
  * In-process result of a scoped drain whose only shortfall was admission
96
155
  * deferrals: the paths are restored and a retry is scheduled. Never a
@@ -191,6 +250,66 @@ export function allowConcurrentHandoffForGuardedWork(workClass, explicitlyAllowe
191
250
  return explicitlyAllowed ||
192
251
  (workClass === "realtime" && isFullPassDurableProgressEnabled());
193
252
  }
253
+ export async function runGuardedPassPreparation(prepare, onStage, onFailure) {
254
+ onStage?.("pass preparation");
255
+ if (!prepare)
256
+ return;
257
+ try {
258
+ await prepare();
259
+ }
260
+ catch (error) {
261
+ const name = error instanceof Error ? error.name : "UnknownError";
262
+ const errorClass = /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name)
263
+ ? name
264
+ : "UnknownError";
265
+ onStage?.("pass preparation failed");
266
+ onFailure?.({ type: "watcher-push-preparation-failed", errorClass });
267
+ onStage?.("pass return");
268
+ throw error;
269
+ }
270
+ }
271
+ function pathFreeErrorClass(error) {
272
+ if (!(error instanceof Error))
273
+ return undefined;
274
+ return /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name)
275
+ ? error.name
276
+ : "UnknownError";
277
+ }
278
+ export function buildScopedPushFailureDiagnostic(fields, error, caughtError, caughtErrorClass) {
279
+ const errorClass = caughtErrorClass ?? pathFreeErrorClass(caughtError);
280
+ return {
281
+ type: "scoped-push-failure",
282
+ ...fields,
283
+ ...(errorClass ? { errorClass } : {}),
284
+ error,
285
+ };
286
+ }
287
+ export function resolveWatcherPushWaitStage(input) {
288
+ if (input.capYieldActive)
289
+ return "CPU/heap cap wait";
290
+ if (input.holders.some((holder) => holder.kind === "pull"))
291
+ return "pull-yield wait";
292
+ if (input.queueDepth > 0)
293
+ return "queued behind realtime work";
294
+ if (input.holders.some((holder) => holder.kind === "chore"))
295
+ return "chore holder wait";
296
+ if (input.holders.length > 0)
297
+ return "scheduler holder wait";
298
+ return "scheduler admission pending";
299
+ }
300
+ export function watcherPushWaitDiagnostics(input) {
301
+ return {
302
+ stage: input.admissionWaiting
303
+ ? resolveWatcherPushWaitStage(input)
304
+ : input.stage,
305
+ waitingSinceAdmissionRequestMs: input.admissionRequestedAtMonotonicMs === null
306
+ ? null
307
+ : Math.max(0, input.nowMonotonicMs - input.admissionRequestedAtMonotonicMs),
308
+ schedulerLastAdmittedAnyWorkAgoMs: input.lastAnyAdmissionAtMonotonicMs === null
309
+ ? null
310
+ : Math.max(0, input.nowMonotonicMs - input.lastAnyAdmissionAtMonotonicMs),
311
+ };
312
+ }
194
313
  /**
195
314
  * One durable-state owner at a time for a full chore. Realtime work runs only
196
315
  * while that chore is suspended at `checkpoint()`. Only coordinator-fenced
@@ -205,6 +324,8 @@ export class CooperativePassScheduler {
205
324
  starvationCapMs;
206
325
  options;
207
326
  pending = [];
327
+ lastAnyAdmissionAtMonotonicMs = null;
328
+ nextEntryId = 0;
208
329
  active = null;
209
330
  /** Realtime executes started by the current hand-off. Not the chore slot. */
210
331
  realtimeOccupants = new Set();
@@ -241,12 +362,65 @@ export class CooperativePassScheduler {
241
362
  /** Successful callers still applying results after the network hand-off. */
242
363
  networkTransferCompletionCount = 0;
243
364
  networkTransferApplyRelease = null;
365
+ consecutivePullAdmissions = 0;
366
+ consecutiveFastAdmissions = 0;
244
367
  resolveNetworkTransferApplyRelease = null;
245
368
  constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
246
369
  this.now = now;
247
370
  this.starvationCapMs = starvationCapMs;
248
371
  this.options = options;
249
372
  }
373
+ recordAdmission(entry) {
374
+ const monotonicMs = performance.now();
375
+ const lastAdmissionAgoMs = this.lastAnyAdmissionAtMonotonicMs === null
376
+ ? null
377
+ : Math.max(0, monotonicMs - this.lastAnyAdmissionAtMonotonicMs);
378
+ const waitingPush = this.pending.find((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
379
+ const admittedIsPush = entry.workClass === "realtime" && entry.transferDirection === "push";
380
+ if (entry.workClass === "realtime") {
381
+ if (entry.lane === "fast")
382
+ this.consecutiveFastAdmissions += 1;
383
+ else
384
+ this.consecutiveFastAdmissions = 0;
385
+ }
386
+ const oldestPush = waitingPush ?? (admittedIsPush ? entry : undefined);
387
+ const waitingSinceAdmissionRequestMs = oldestPush
388
+ ? Math.max(0, this.now() - oldestPush.enqueuedAt)
389
+ : null;
390
+ if (waitingPush || admittedIsPush) {
391
+ this.options.onAdmission?.({
392
+ entryId: entry.entryId,
393
+ workClass: entry.workClass,
394
+ lane: entry.lane,
395
+ transferDirection: entry.transferDirection,
396
+ transferSizeClass: entry.transferSizeClass ?? "unknown",
397
+ label: entry.label.slice(0, 160),
398
+ admittedAt: new Date().toISOString(),
399
+ schedulerLastAdmittedAnyWorkAgoMs: lastAdmissionAgoMs,
400
+ realtimeAdmissionQueueDepth: this.pending.filter((pending) => pending.workClass === "realtime").length,
401
+ oldestWaitingWatcherPush: oldestPush
402
+ ? {
403
+ entryId: oldestPush.entryId,
404
+ label: oldestPush.label.slice(0, 160),
405
+ waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
406
+ }
407
+ : null,
408
+ passedOver: waitingPush && waitingPush !== entry
409
+ ? {
410
+ entryId: waitingPush.entryId,
411
+ label: waitingPush.label.slice(0, 160),
412
+ waitingSinceAdmissionRequestMs: waitingSinceAdmissionRequestMs ?? 0,
413
+ reason: entry.transferDirection === "pull"
414
+ ? "pull-admitted-ahead-of-waiting-push"
415
+ : this._handoffActive
416
+ ? "handoff-selector-admitted-other-work"
417
+ : "scheduler-admitted-other-work",
418
+ }
419
+ : null,
420
+ });
421
+ }
422
+ this.lastAnyAdmissionAtMonotonicMs = monotonicMs;
423
+ }
250
424
  get handoffActive() {
251
425
  return this._handoffActive;
252
426
  }
@@ -263,12 +437,40 @@ export class CooperativePassScheduler {
263
437
  get hasActiveSlowRealtime() {
264
438
  return this.active?.workClass === "realtime" && this.active.lane === "slow";
265
439
  }
440
+ get isInCapYield() {
441
+ return this.capYieldDepth > 0;
442
+ }
266
443
  /** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
267
444
  get activeRealtimeWorkerCount() {
268
445
  return ((this.active?.workClass === "realtime" ? 1 : 0) +
269
446
  (this.activeFastRealtime === null ? 0 : 1) +
270
447
  this.realtimeOccupants.size);
271
448
  }
449
+ get diagnosticAdmissionSnapshot() {
450
+ const active = [
451
+ this.active,
452
+ this.activeFastRealtime,
453
+ ...this.realtimeOccupants,
454
+ this.detachedChore,
455
+ ].filter((entry) => entry !== null);
456
+ const holders = active.map((entry) => ({
457
+ entryId: entry.entryId,
458
+ kind: entry.transferDirection === "pull"
459
+ ? "pull"
460
+ : entry.workClass === "chore"
461
+ ? "chore"
462
+ : "push",
463
+ scope: entry.label.slice(0, 160),
464
+ ageMs: entry.startedAtMonotonicMs === undefined
465
+ ? 0
466
+ : Math.max(0, performance.now() - entry.startedAtMonotonicMs),
467
+ }));
468
+ return {
469
+ queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
470
+ holders: holders.slice(0, 20),
471
+ lastAnyAdmissionAtMonotonicMs: this.lastAnyAdmissionAtMonotonicMs,
472
+ };
473
+ }
272
474
  pendingLaneCount(lane) {
273
475
  return this.pending.filter((entry) => entry.lane === lane).length;
274
476
  }
@@ -283,6 +485,7 @@ export class CooperativePassScheduler {
283
485
  signal.addEventListener("abort", () => controller.abort(), { once: true });
284
486
  }
285
487
  const directEntry = {
488
+ entryId: `guarded-${++this.nextEntryId}`,
286
489
  task,
287
490
  workClass,
288
491
  label,
@@ -306,7 +509,11 @@ export class CooperativePassScheduler {
306
509
  if (workClass === "realtime" &&
307
510
  this._handoffActive &&
308
511
  directEntry.allowConcurrentHandoff &&
309
- this.canAdmitRealtimeOccupant(directEntry)) {
512
+ this.canAdmitRealtimeOccupant(directEntry) &&
513
+ !(directEntry.lane === "fast" && this.consecutiveFastAdmissions > 0 &&
514
+ this.pending.some((pending) => pending.workClass === "realtime" &&
515
+ pending.lane === "slow" && pending.transferDirection === directEntry.transferDirection)) &&
516
+ !this.pullMustYieldForWaitingPush(directEntry)) {
310
517
  return new Promise((resolve, reject) => {
311
518
  const entry = {
312
519
  ...directEntry,
@@ -455,16 +662,21 @@ export class CooperativePassScheduler {
455
662
  this.admitHandoffBacklog();
456
663
  return;
457
664
  }
458
- if (this.active !== null) {
665
+ if (this.active !== null || this.activeFastRealtime !== null || this.realtimeOccupants.size > 0) {
459
666
  const fastRealtime = this.nextFastRealtime();
460
- if (fastRealtime && this.canStartFastRealtime(fastRealtime)) {
667
+ if (this.active !== null && fastRealtime && this.canStartFastRealtime(fastRealtime)) {
461
668
  this.pending.splice(this.pending.indexOf(fastRealtime), 1);
462
669
  this.startFastRealtime(fastRealtime);
670
+ return;
671
+ }
672
+ const peerIndex = this.nextRealtimeIndex((entry) => this.canAdmitRealtimePeer(entry));
673
+ if (peerIndex >= 0) {
674
+ const peer = this.pending.splice(peerIndex, 1)[0];
675
+ if (peer)
676
+ this.startRealtimeOccupant(peer);
463
677
  }
464
678
  return;
465
679
  }
466
- if (this.activeFastRealtime !== null)
467
- return;
468
680
  const next = this.takeNext();
469
681
  if (next)
470
682
  this.start(next);
@@ -483,30 +695,116 @@ export class CooperativePassScheduler {
483
695
  return this.pending.splice(realtime, 1)[0];
484
696
  return this.pending.shift();
485
697
  }
486
- nextRealtimeIndex() {
487
- const slowRealtime = this.pending.findIndex((entry) => entry.workClass === "realtime" && entry.lane === "slow");
488
- return slowRealtime >= 0
489
- ? slowRealtime
490
- : this.pending.findIndex((entry) => entry.workClass === "realtime");
698
+ pullMustYieldForWaitingPush(entry) {
699
+ return entry.transferDirection === "pull" &&
700
+ this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH &&
701
+ this.pending.some((pending) => pending.workClass === "realtime" && pending.transferDirection === "push");
702
+ }
703
+ nextRealtimeIndex(eligible = () => true) {
704
+ let firstRealtime = -1;
705
+ let waitingPush = -1;
706
+ let slowPull = -1;
707
+ let fastPush = -1;
708
+ let slowPush = -1;
709
+ let firstFastLane = -1;
710
+ let firstSlowLane = -1;
711
+ for (let index = 0; index < this.pending.length; index += 1) {
712
+ const entry = this.pending[index];
713
+ if (!entry || entry.workClass !== "realtime" || !eligible(entry))
714
+ continue;
715
+ if (firstRealtime < 0)
716
+ firstRealtime = index;
717
+ if (entry.transferDirection === undefined) {
718
+ if (entry.lane === "fast" && firstFastLane < 0)
719
+ firstFastLane = index;
720
+ if (entry.lane === "slow" && firstSlowLane < 0)
721
+ firstSlowLane = index;
722
+ }
723
+ if (entry.transferDirection === "push" && waitingPush < 0)
724
+ waitingPush = index;
725
+ if (entry.transferDirection === "pull" && entry.lane === "slow" && slowPull < 0) {
726
+ slowPull = index;
727
+ }
728
+ if (entry.transferDirection === "push" && entry.lane === "fast" && fastPush < 0) {
729
+ fastPush = index;
730
+ }
731
+ if (entry.transferDirection === "push" && entry.lane === "slow" && slowPush < 0) {
732
+ slowPush = index;
733
+ }
734
+ }
735
+ if (waitingPush >= 0 && this.consecutivePullAdmissions >= MAX_PULL_ADMISSIONS_BEFORE_PUSH) {
736
+ this.consecutivePullAdmissions = 0;
737
+ this.consecutiveFastAdmissions = 0;
738
+ return waitingPush;
739
+ }
740
+ // A waiting slow pull is the receiver's progress boundary and retains its
741
+ // bounded turn ahead of pushes. For pushes, let the oldest fast batch take
742
+ // one turn before a queued bulk batch, then give the bulk batch a turn
743
+ // before admitting another fast batch. This prevents bulk arrivals from
744
+ // starving an older interactive write and prevents a stream of fast writes
745
+ // from starving a queued bulk upload.
746
+ const next = slowPull >= 0
747
+ ? slowPull
748
+ : fastPush >= 0 && slowPush >= 0
749
+ ? this.consecutiveFastAdmissions === 0 ? fastPush : slowPush
750
+ : firstFastLane >= 0 && firstSlowLane >= 0
751
+ ? this.consecutiveFastAdmissions === 0 ? firstFastLane : firstSlowLane
752
+ : firstRealtime;
753
+ if (next >= 0) {
754
+ const selected = this.pending[next];
755
+ if (selected?.transferDirection === "pull")
756
+ this.consecutivePullAdmissions += 1;
757
+ else if (selected?.transferDirection === "push")
758
+ this.consecutivePullAdmissions = 0;
759
+ if (selected?.transferDirection === "push" && selected.lane === "fast" && slowPush >= 0) {
760
+ this.consecutiveFastAdmissions += 1;
761
+ }
762
+ else if (selected?.transferDirection === "push") {
763
+ this.consecutiveFastAdmissions = 0;
764
+ }
765
+ }
766
+ return next;
491
767
  }
492
768
  nextFastRealtime() {
493
- return this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
769
+ const fast = this.pending.find((entry) => entry.workClass === "realtime" && entry.lane === "fast");
770
+ if (!fast)
771
+ return undefined;
772
+ const slowOfSameDirectionWaiting = this.pending.some((entry) => entry.workClass === "realtime" && entry.lane === "slow" &&
773
+ entry.transferDirection === fast.transferDirection);
774
+ if (slowOfSameDirectionWaiting && this.consecutiveFastAdmissions > 0)
775
+ return undefined;
776
+ return fast;
494
777
  }
495
778
  hasStarvedChore() {
496
779
  const now = this.now();
497
780
  return this.pending.some((entry) => entry.workClass === "chore" && now - entry.enqueuedAt >= this.starvationCapMs);
498
781
  }
499
782
  canStartFastRealtime(entry) {
783
+ const slowOfSameDirectionWaiting = this.pending.some((pending) => pending.workClass === "realtime" && pending.lane === "slow" &&
784
+ pending.transferDirection === entry.transferDirection);
500
785
  return (!this._handoffActive &&
501
786
  this.activeFastRealtime === null &&
502
787
  entry.workClass === "realtime" &&
503
788
  entry.lane === "fast" &&
789
+ !(slowOfSameDirectionWaiting && this.consecutiveFastAdmissions > 0) &&
504
790
  entry.allowFastRealtimeOvertake &&
505
791
  this.hasActiveSlowRealtime &&
506
792
  !this.hasStarvedChore());
507
793
  }
794
+ /** Coordinator-fenced opposite-direction work may use the reserved realtime slot. */
795
+ canAdmitRealtimePeer(entry) {
796
+ if (entry.workClass !== "realtime" ||
797
+ !entry.allowConcurrentHandoff ||
798
+ entry.transferDirection === undefined ||
799
+ this.activeRealtimeWorkerCount >= REALTIME_WORKER_OCCUPANCY_CAP)
800
+ return false;
801
+ return [this.active, this.activeFastRealtime, ...this.realtimeOccupants].some((active) => active?.workClass === "realtime" &&
802
+ active.transferDirection !== undefined &&
803
+ active.transferDirection !== entry.transferDirection);
804
+ }
508
805
  start(entry) {
509
806
  this.clearAdmission(entry);
807
+ this.recordAdmission(entry);
510
808
  entry.startedAt = new Date().toISOString();
511
809
  // Holder duration uses the process monotonic clock; do not consume the
512
810
  // injected scheduler clock, whose reads define cadence-slot deadlines.
@@ -545,6 +843,8 @@ export class CooperativePassScheduler {
545
843
  }
546
844
  startFastRealtime(entry) {
547
845
  this.clearAdmission(entry);
846
+ this.recordAdmission(entry);
847
+ entry.startedAtMonotonicMs = performance.now();
548
848
  this.activeFastRealtime = entry;
549
849
  let current;
550
850
  try {
@@ -800,6 +1100,8 @@ export class CooperativePassScheduler {
800
1100
  /** Run one realtime execute without aborting it and without taking the chore slot. */
801
1101
  startRealtimeOccupant(entry) {
802
1102
  this.clearAdmission(entry);
1103
+ this.recordAdmission(entry);
1104
+ entry.startedAtMonotonicMs = performance.now();
803
1105
  this.realtimeOccupants.add(entry);
804
1106
  this.handoffAdmitted = true;
805
1107
  if (this.capYieldDepth > 0)
@@ -814,6 +1116,7 @@ export class CooperativePassScheduler {
814
1116
  this.handoffBacklog = null;
815
1117
  entry.reject(error);
816
1118
  this.notifyOccupantChange();
1119
+ this.drain();
817
1120
  return;
818
1121
  }
819
1122
  void running.then((outcome) => {
@@ -823,6 +1126,7 @@ export class CooperativePassScheduler {
823
1126
  entry.resolve(outcome);
824
1127
  this.promoteOccupancyArrivals();
825
1128
  this.notifyOccupantChange();
1129
+ this.drain();
826
1130
  }, (error) => {
827
1131
  this.realtimeOccupants.delete(entry);
828
1132
  if (this.handoffBacklog === entry)
@@ -830,6 +1134,7 @@ export class CooperativePassScheduler {
830
1134
  entry.reject(error);
831
1135
  this.promoteOccupancyArrivals();
832
1136
  this.notifyOccupantChange();
1137
+ this.drain();
833
1138
  });
834
1139
  }
835
1140
  /** Eligible arrivals may take a free slot before hand-off admission closes. */
@@ -838,8 +1143,7 @@ export class CooperativePassScheduler {
838
1143
  (!this.networkTransferActive && !this.withinHandoffAdmission()))
839
1144
  return;
840
1145
  while (this.realtimeOccupants.size < REALTIME_WORKER_OCCUPANCY_CAP) {
841
- const index = this.pending.findIndex((pending) => pending.workClass === "realtime" &&
842
- pending.occupancyArrival === true &&
1146
+ const index = this.nextRealtimeIndex((pending) => pending.occupancyArrival === true &&
843
1147
  pending.allowConcurrentHandoff &&
844
1148
  this.canAdmitRealtimeOccupant(pending));
845
1149
  if (index < 0)
@@ -858,7 +1162,7 @@ export class CooperativePassScheduler {
858
1162
  return;
859
1163
  if (!this.withinHandoffAdmission())
860
1164
  return;
861
- const realtimeIndex = this.nextRealtimeIndex();
1165
+ const realtimeIndex = this.nextRealtimeIndex((pending) => !pending.occupancyArrival && this.canAdmitRealtimeOccupant(pending));
862
1166
  if (realtimeIndex < 0)
863
1167
  return;
864
1168
  const pending = this.pending[realtimeIndex];
@@ -1292,6 +1596,151 @@ export function isChoreLocalPush(paths, largeUpload) {
1292
1596
  }
1293
1597
  return count > 0;
1294
1598
  }
1599
+ /** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
1600
+ export function watcherWarmupPathsByRoute(relativePaths) {
1601
+ const counts = new Map();
1602
+ for (const relativePath of relativePaths) {
1603
+ const route = routeChangeToTarget(relativePath);
1604
+ const key = route ? routeKey(route) : "unrouted";
1605
+ counts.set(key, (counts.get(key) ?? 0) + 1);
1606
+ }
1607
+ const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1608
+ const result = {};
1609
+ for (const [key, count] of ranked.slice(0, 20))
1610
+ result[key] = count;
1611
+ const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
1612
+ if (other > 0)
1613
+ result.other = other;
1614
+ return result;
1615
+ }
1616
+ export function watcherWarmupPrefixesByRoute(candidateRelativePaths, eligibleRelativePaths = candidateRelativePaths) {
1617
+ const candidatePrefixes = new Map();
1618
+ const candidateCounts = new Map();
1619
+ for (const relativePath of candidateRelativePaths) {
1620
+ const route = routeChangeToTarget(relativePath);
1621
+ const key = route ? routeKey(route) : "unrouted";
1622
+ candidateCounts.set(key, (candidateCounts.get(key) ?? 0) + 1);
1623
+ const routePath = route?.kind === "company"
1624
+ ? relativePath.slice(`companies/${route.slug}/`.length)
1625
+ : relativePath;
1626
+ const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
1627
+ const routeCounts = candidatePrefixes.get(key) ?? new Map();
1628
+ routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
1629
+ candidatePrefixes.set(key, routeCounts);
1630
+ }
1631
+ const eligibleCounts = new Map();
1632
+ for (const relativePath of eligibleRelativePaths) {
1633
+ const route = routeChangeToTarget(relativePath);
1634
+ const key = route ? routeKey(route) : "unrouted";
1635
+ const routePath = route?.kind === "company"
1636
+ ? relativePath.slice(`companies/${route.slug}/`.length)
1637
+ : relativePath;
1638
+ const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
1639
+ const routeCounts = eligibleCounts.get(key) ?? new Map();
1640
+ routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
1641
+ eligibleCounts.set(key, routeCounts);
1642
+ }
1643
+ const result = {};
1644
+ // Eligible routes are all retained; fill any remaining diagnostic slots
1645
+ // with the largest candidate routes. This keeps alphabetical one-path
1646
+ // stubs from crowding out a large route such as personal.
1647
+ const eligibleRoutes = [...eligibleCounts.keys()];
1648
+ const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
1649
+ const rankedCandidates = [...candidateCounts.entries()]
1650
+ .filter(([route]) => !eligibleCounts.has(route))
1651
+ .sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1652
+ const selectedRoutes = [
1653
+ ...eligibleRoutes.sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right)),
1654
+ ...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
1655
+ ];
1656
+ for (const route of selectedRoutes) {
1657
+ const topPrefixes = (counts) => {
1658
+ const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1659
+ const top = {};
1660
+ for (const [prefix, count] of ranked.slice(0, 5))
1661
+ top[prefix] = count;
1662
+ const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
1663
+ if (other > 0)
1664
+ top.other = other;
1665
+ return top;
1666
+ };
1667
+ result[route] = {
1668
+ candidate: topPrefixes(candidatePrefixes.get(route)),
1669
+ eligible: topPrefixes(eligibleCounts.get(route)),
1670
+ };
1671
+ }
1672
+ const otherRoutes = [...eligibleCounts.entries()].filter(([route]) => !selectedRoutes.includes(route));
1673
+ if (otherRoutes.length > 0) {
1674
+ const merged = new Map();
1675
+ for (const [, routeCounts] of otherRoutes) {
1676
+ for (const [prefix, count] of routeCounts)
1677
+ merged.set(prefix, (merged.get(prefix) ?? 0) + count);
1678
+ }
1679
+ result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
1680
+ }
1681
+ return result;
1682
+ }
1683
+ function recordWarmupRoutePath(counts, prefixes, routeName, routePath) {
1684
+ counts.set(routeName, (counts.get(routeName) ?? 0) + 1);
1685
+ const firstSlash = routePath.indexOf("/");
1686
+ const secondSlash = firstSlash < 0 ? -1 : routePath.indexOf("/", firstSlash + 1);
1687
+ const prefix = secondSlash < 0 ? routePath || "(root)" : routePath.slice(0, secondSlash);
1688
+ let byPrefix = prefixes.get(routeName);
1689
+ if (!byPrefix) {
1690
+ byPrefix = new Map();
1691
+ prefixes.set(routeName, byPrefix);
1692
+ }
1693
+ byPrefix.set(prefix, (byPrefix.get(prefix) ?? 0) + 1);
1694
+ }
1695
+ function rankedWarmupCounts(counts) {
1696
+ const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1697
+ const result = {};
1698
+ for (const [key, count] of ranked.slice(0, 20))
1699
+ result[key] = count;
1700
+ const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
1701
+ if (other > 0)
1702
+ result.other = other;
1703
+ return result;
1704
+ }
1705
+ function rankedWarmupPrefixes(candidateCounts, candidatePrefixes, eligibleCounts, eligiblePrefixes) {
1706
+ const topPrefixes = (counts) => {
1707
+ const ranked = [...(counts ?? new Map()).entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1708
+ const top = {};
1709
+ for (const [prefix, count] of ranked.slice(0, 5))
1710
+ top[prefix] = count;
1711
+ const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
1712
+ if (other > 0)
1713
+ top.other = other;
1714
+ return top;
1715
+ };
1716
+ const eligibleRoutes = [...eligibleCounts.keys()].sort((left, right) => (candidateCounts.get(right) ?? 0) - (candidateCounts.get(left) ?? 0) || left.localeCompare(right));
1717
+ const remainingSlots = Math.max(0, 20 - eligibleRoutes.length);
1718
+ const rankedCandidates = [...candidateCounts.entries()]
1719
+ .filter(([route]) => !eligibleCounts.has(route))
1720
+ .sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1721
+ const selectedRoutes = [
1722
+ ...eligibleRoutes,
1723
+ ...rankedCandidates.slice(0, remainingSlots).map(([route]) => route),
1724
+ ];
1725
+ const result = {};
1726
+ for (const route of selectedRoutes) {
1727
+ result[route] = {
1728
+ candidate: topPrefixes(candidatePrefixes.get(route)),
1729
+ eligible: topPrefixes(eligiblePrefixes.get(route)),
1730
+ };
1731
+ }
1732
+ const otherRoutes = [...eligibleCounts.keys()].filter((route) => !selectedRoutes.includes(route));
1733
+ if (otherRoutes.length > 0) {
1734
+ const merged = new Map();
1735
+ for (const route of otherRoutes) {
1736
+ for (const [prefix, count] of eligiblePrefixes.get(route) ?? []) {
1737
+ merged.set(prefix, (merged.get(prefix) ?? 0) + count);
1738
+ }
1739
+ }
1740
+ result.other = { candidate: {}, eligible: Object.fromEntries(merged) };
1741
+ }
1742
+ return result;
1743
+ }
1295
1744
  /** A receiver event needs a known small upsert size before it can go realtime. */
1296
1745
  export function isFastReceiverBatch(eventCount) {
1297
1746
  return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
@@ -1762,9 +2211,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1762
2211
  // RUNNER_HEAP_RECYCLE_EXIT through the normal shutdown()/finally teardown.
1763
2212
  let recycleRequested = null;
1764
2213
  const pendingAppliedUpserts = new Map();
2214
+ const appliedUpsertHashesForWatcher = new Map();
2215
+ const APPLIED_UPSERT_WATCHER_HASH_CAP = 10_000;
1765
2216
  const pendingAppliedDeletes = new Set();
2217
+ const uploadedAwaitingPublish = new Set();
2218
+ const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
2219
+ const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
2220
+ let uploadedAwaitingPublishDropLogged = false;
1766
2221
  const rememberAppliedUpsert = (relativePath, contentHash) => {
1767
2222
  pendingAppliedDeletes.delete(relativePath);
2223
+ appliedUpsertHashesForWatcher.delete(relativePath);
2224
+ appliedUpsertHashesForWatcher.set(relativePath, contentHash);
2225
+ while (appliedUpsertHashesForWatcher.size > APPLIED_UPSERT_WATCHER_HASH_CAP) {
2226
+ const oldest = appliedUpsertHashesForWatcher.keys().next().value;
2227
+ if (oldest === undefined)
2228
+ break;
2229
+ appliedUpsertHashesForWatcher.delete(oldest);
2230
+ }
1768
2231
  if (eventSyncLive && eventSync) {
1769
2232
  eventSync.markContentPublished?.(relativePath, contentHash);
1770
2233
  return;
@@ -1779,6 +2242,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1779
2242
  }
1780
2243
  };
1781
2244
  const rememberAppliedDelete = (relativePath) => {
2245
+ appliedUpsertHashesForWatcher.delete(relativePath);
1782
2246
  pendingAppliedUpserts.delete(relativePath);
1783
2247
  if (eventSyncLive && eventSync) {
1784
2248
  eventSync.markContentDeleted?.(relativePath);
@@ -1793,6 +2257,65 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1793
2257
  pendingAppliedDeletes.delete(oldest);
1794
2258
  }
1795
2259
  };
2260
+ const isUnchangedAppliedUpsert = async (relativePath) => {
2261
+ const expectedHash = appliedUpsertHashesForWatcher.get(relativePath);
2262
+ if (!expectedHash)
2263
+ return false;
2264
+ // Receiver-applied hashes suppress only the first watcher observation.
2265
+ // A later local edit may legitimately revert to this content.
2266
+ appliedUpsertHashesForWatcher.delete(relativePath);
2267
+ const absolutePath = localPathForVaultKey(hqRoot, relativePath);
2268
+ try {
2269
+ const stat = await fs.promises.lstat(absolutePath);
2270
+ let actualHash;
2271
+ if (stat.isSymbolicLink()) {
2272
+ actualHash = `sha256:${hashSymlinkTarget(await fs.promises.readlink(absolutePath))}`;
2273
+ }
2274
+ else if (stat.isFile()) {
2275
+ const digest = createHash("sha256");
2276
+ await new Promise((resolve, reject) => {
2277
+ const input = fs.createReadStream(absolutePath);
2278
+ input.on("data", (chunk) => digest.update(chunk));
2279
+ input.once("error", reject);
2280
+ input.once("end", resolve);
2281
+ });
2282
+ actualHash = `sha256:${digest.digest("hex")}`;
2283
+ }
2284
+ else {
2285
+ return false;
2286
+ }
2287
+ return actualHash === expectedHash;
2288
+ }
2289
+ catch (error) {
2290
+ const code = error && typeof error === "object" && "code" in error
2291
+ ? error.code
2292
+ : undefined;
2293
+ if (code !== "ENOENT") {
2294
+ process.stderr.write(`${JSON.stringify({
2295
+ type: "watcher-applied-upsert-hash-check-failed",
2296
+ errorClass: pathFreeErrorClass(error) ?? "UnknownError",
2297
+ })}\n`);
2298
+ }
2299
+ return false;
2300
+ }
2301
+ };
2302
+ const flushPendingUploadEvents = () => {
2303
+ if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
2304
+ return;
2305
+ while (uploadedAwaitingPublish.size > 0) {
2306
+ const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
2307
+ const paths = new Map(uploadedPaths.map((relativePath) => [
2308
+ localPathForVaultKey(hqRoot, relativePath),
2309
+ relativePath,
2310
+ ]));
2311
+ const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
2312
+ if (expanded.size > 0)
2313
+ eventSync.publishBatch({ paths: expanded });
2314
+ for (const relativePath of uploadedPaths) {
2315
+ uploadedAwaitingPublish.delete(relativePath);
2316
+ }
2317
+ }
2318
+ };
1796
2319
  const flushPendingAppliedUpserts = () => {
1797
2320
  if (!eventSyncLive || !eventSync)
1798
2321
  return;
@@ -1804,6 +2327,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1804
2327
  eventSync.markContentDeleted?.(relativePath);
1805
2328
  }
1806
2329
  pendingAppliedDeletes.clear();
2330
+ flushPendingUploadEvents();
1807
2331
  };
1808
2332
  // `activeSlowPass` is assigned before its task has necessarily acquired the
1809
2333
  // cross-process lock (it may be waiting behind a rescue or a second runner).
@@ -1853,8 +2377,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1853
2377
  }, 0);
1854
2378
  refresh.unref?.();
1855
2379
  }
1856
- const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
1857
- const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
2380
+ const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, {
2381
+ caps: fullPassCaps,
2382
+ onAdmission: (event) => {
2383
+ process.stderr.write(`${JSON.stringify({
2384
+ type: "watcher-push-scheduler-admission",
2385
+ diagnostic: true,
2386
+ ...event,
2387
+ })}\n`);
2388
+ },
2389
+ });
2390
+ const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
1858
2391
  const passCheckpoint = pullSignal || receiverSignal
1859
2392
  ? Object.assign(async () => checkpoint?.(), {
1860
2393
  shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
@@ -1884,21 +2417,50 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1884
2417
  }
1885
2418
  };
1886
2419
  const invokePass = () => {
1887
- if (!runPassSupportsCooperativeCheckpoint) {
1888
- if (!runPassSupportsQmdLifecycle)
1889
- return runPass(passArgvForRun);
1890
- return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
1891
- }
1892
- // This callback is emitted by the production runner after journal
1893
- // repair/seed and before fanout. It starts ingress early, but never
1894
- // admits a second durable writer: queued realtime still waits for the
1895
- // existing scheduler's cooperative hand-off checkpoint.
1896
- const onJournalsReady = () => activateEventPushSurfaces();
1897
- if (!runPassSupportsQmdLifecycle) {
1898
- return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
1899
- }
1900
- return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
2420
+ watcherPushStage?.("pass invocation");
2421
+ activePassCount += 1;
2422
+ emitSyncWorkState(true);
2423
+ const trackPass = (pass) => {
2424
+ const complete = () => {
2425
+ watcherPushStage?.("pass return");
2426
+ activePassCount = Math.max(0, activePassCount - 1);
2427
+ emitSyncWorkState(true);
2428
+ };
2429
+ // Observe settlement without returning a chained promise to the pass
2430
+ // scheduler. The scheduler's synchronous test seam and watcher wake
2431
+ // ordering depend on the original pass promise's completion turn.
2432
+ void pass.then(complete, complete);
2433
+ return pass;
2434
+ };
2435
+ try {
2436
+ if (!runPassSupportsCooperativeCheckpoint) {
2437
+ if (!runPassSupportsQmdLifecycle)
2438
+ return trackPass(runPass(passArgvForRun));
2439
+ return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
2440
+ }
2441
+ // This callback is emitted by the production runner after journal
2442
+ // repair/seed and before fanout. It starts ingress early, but never
2443
+ // admits a second durable writer: queued realtime still waits for the
2444
+ // existing scheduler's cooperative hand-off checkpoint.
2445
+ const onJournalsReady = () => activateEventPushSurfaces();
2446
+ if (!runPassSupportsQmdLifecycle) {
2447
+ return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
2448
+ }
2449
+ return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
2450
+ }
2451
+ catch (error) {
2452
+ watcherPushStage?.("pass return");
2453
+ activePassCount = Math.max(0, activePassCount - 1);
2454
+ emitSyncWorkState(true);
2455
+ throw error;
2456
+ }
1901
2457
  };
2458
+ const preparePass = () => runGuardedPassPreparation(prepare, watcherPushStage, watcherPushStage
2459
+ ? (event) => process.stderr.write(`${JSON.stringify({
2460
+ ...event,
2461
+ diagnostic: true,
2462
+ })}\n`)
2463
+ : undefined);
1902
2464
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
1903
2465
  // admitting a pass and immediately after it returns. A pass that runs a long
1904
2466
  // synchronous stretch starves the interval timer, so these boundary
@@ -1912,6 +2474,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1912
2474
  return;
1913
2475
  recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
1914
2476
  recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
2477
+ rememberUploadedPaths(outcome);
2478
+ if (typeof outcome !== "number") {
2479
+ acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
2480
+ }
2481
+ flushPendingUploadEvents();
1915
2482
  };
1916
2483
  // Pre-admit boundary observation.
1917
2484
  heapCheckpoint?.();
@@ -1931,9 +2498,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1931
2498
  try {
1932
2499
  telemetry?.mark("network");
1933
2500
  const outcome = telemetry ? await telemetry.measureNetwork(async () => {
1934
- await prepare?.();
2501
+ await preparePass();
1935
2502
  return invokePass();
1936
- }) : await (async () => { await prepare?.(); return invokePass(); })();
2503
+ }) : await (async () => { await preparePass(); return invokePass(); })();
1937
2504
  recordAppliedUpserts(outcome);
1938
2505
  return outcome;
1939
2506
  }
@@ -1948,9 +2515,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1948
2515
  try {
1949
2516
  telemetry?.mark("network");
1950
2517
  const outcome = telemetry ? await telemetry.measureNetwork(async () => {
1951
- await prepare?.();
2518
+ await preparePass();
1952
2519
  return invokePass();
1953
- }) : await (async () => { await prepare?.(); return invokePass(); })();
2520
+ }) : await (async () => { await preparePass(); return invokePass(); })();
1954
2521
  recordAppliedUpserts(outcome);
1955
2522
  return outcome;
1956
2523
  }
@@ -1972,9 +2539,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1972
2539
  acquireOperationLockUser();
1973
2540
  }
1974
2541
  const outcome = telemetry ? await telemetry.measureNetwork(async () => {
1975
- await prepare?.();
2542
+ await preparePass();
1976
2543
  return invokePass();
1977
- }) : await (async () => { await prepare?.(); return invokePass(); })();
2544
+ }) : await (async () => { await preparePass(); return invokePass(); })();
1978
2545
  recordAppliedUpserts(outcome);
1979
2546
  return outcome;
1980
2547
  }
@@ -2010,9 +2577,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2010
2577
  }
2011
2578
  try {
2012
2579
  const outcome = telemetry ? await telemetry.measureNetwork(async () => {
2013
- await prepare?.();
2580
+ await preparePass();
2014
2581
  return invokePass();
2015
- }) : await (async () => { await prepare?.(); return invokePass(); })();
2582
+ }) : await (async () => { await preparePass(); return invokePass(); })();
2016
2583
  recordAppliedUpserts(outcome);
2017
2584
  return outcome;
2018
2585
  }
@@ -2162,15 +2729,25 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2162
2729
  // watcher fast path explicitly opts into its established scoped exception.
2163
2730
  return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
2164
2731
  };
2165
- const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
2732
+ const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass, watcherPushStage) => {
2733
+ if (watcherPushStage) {
2734
+ watcherPushStage("guarded scheduler admission requested");
2735
+ const admission = passScheduler.diagnosticAdmissionSnapshot;
2736
+ if (passScheduler.isInCapYield)
2737
+ watcherPushStage("CPU/heap cap wait");
2738
+ else if (admission.holders.some((holder) => holder.kind === "pull")) {
2739
+ watcherPushStage("pull-yield wait");
2740
+ }
2741
+ }
2166
2742
  telemetry?.mark("operationGate");
2167
2743
  const admissionSignal = signal && receiverSignal
2168
2744
  ? AbortSignal.any([signal, receiverSignal])
2169
2745
  : signal ?? receiverSignal;
2170
2746
  return runGuardedTask((context) => {
2747
+ watcherPushStage?.("guarded scheduler admission granted");
2171
2748
  telemetry?.mark("appendLock");
2172
2749
  telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
2173
- return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
2750
+ return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal, watcherPushStage);
2174
2751
  }, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
2175
2752
  };
2176
2753
  /*
@@ -2206,11 +2783,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2206
2783
  }
2207
2784
  const pathOutcome = coordinatorPathOutcome(passExitCode(execution.result));
2208
2785
  return {
2209
- paths: slice.paths.map((item) => ({
2210
- path: item.canonicalPath,
2211
- epoch: item.epoch,
2212
- ...pathOutcome,
2213
- })),
2786
+ paths: slice.paths.map((item) => execution.refusedPaths?.has(item.canonicalPath)
2787
+ ? {
2788
+ path: item.canonicalPath,
2789
+ epoch: item.epoch,
2790
+ status: "refused",
2791
+ reason: "remote upsert already present locally",
2792
+ }
2793
+ : {
2794
+ path: item.canonicalPath,
2795
+ epoch: item.epoch,
2796
+ ...pathOutcome,
2797
+ }),
2214
2798
  };
2215
2799
  };
2216
2800
  const createCoordinator = () => new DefaultParallelSyncCoordinator({
@@ -2256,6 +2840,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2256
2840
  result = await task(slice, telemetry, true);
2257
2841
  }
2258
2842
  catch (error) {
2843
+ execution.errorClass = pathFreeErrorClass(error);
2259
2844
  if (error instanceof RealtimeAdmissionTimeoutError) {
2260
2845
  execution.admissionTimeout = error;
2261
2846
  }
@@ -2292,9 +2877,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2292
2877
  return {
2293
2878
  result: execution.result ?? (allRefused ? 0 : 1),
2294
2879
  accepted,
2880
+ yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
2295
2881
  coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
2296
2882
  .filter((path) => path.status === "refused")
2297
2883
  .map((path) => path.path)),
2884
+ ...(execution.errorClass ? { errorClass: execution.errorClass } : {}),
2298
2885
  ...(admissionTimeout ? { admissionTimeout } : {}),
2299
2886
  };
2300
2887
  }
@@ -2489,6 +3076,222 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2489
3076
  /** Vanished never-journaled file deletes removed from batches at drain time. */
2490
3077
  let drainedUntrackedDeletes = 0;
2491
3078
  const pendingWatcherPaths = new Map();
3079
+ const durableWatcherPaths = new Map();
3080
+ const durableWatcherPathLimit = 10_000;
3081
+ let durableWatcherOverflowed = false;
3082
+ let durableWatcherPersistTimer;
3083
+ let durableWatcherPersistQueue = Promise.resolve();
3084
+ let durableWatcherPersistGeneration = 0;
3085
+ let durableWatcherPersistedGeneration = 0;
3086
+ const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
3087
+ const persistDurableWatcherPaths = () => {
3088
+ const generation = durableWatcherPersistGeneration;
3089
+ if (generation === durableWatcherPersistedGeneration)
3090
+ return durableWatcherPersistQueue;
3091
+ const snapshot = durableWatcherOverflowed
3092
+ ? { format: 1, hqRoot, overflowed: true }
3093
+ : { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
3094
+ durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
3095
+ try {
3096
+ if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
3097
+ await fs.promises.rm(watcherPushStatePath, { force: true });
3098
+ durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
3099
+ return;
3100
+ }
3101
+ await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
3102
+ const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
3103
+ await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
3104
+ await fs.promises.rename(tempPath, watcherPushStatePath);
3105
+ durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
3106
+ }
3107
+ catch (err) {
3108
+ process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
3109
+ }
3110
+ }).catch((err) => {
3111
+ process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
3112
+ });
3113
+ return durableWatcherPersistQueue;
3114
+ };
3115
+ const scheduleDurableWatcherPersist = () => {
3116
+ durableWatcherPersistGeneration += 1;
3117
+ if (durableWatcherPersistTimer !== undefined)
3118
+ clearTimeout(durableWatcherPersistTimer);
3119
+ durableWatcherPersistTimer = setTimeout(() => {
3120
+ durableWatcherPersistTimer = undefined;
3121
+ void persistDurableWatcherPaths();
3122
+ }, 1_000);
3123
+ durableWatcherPersistTimer.unref?.();
3124
+ };
3125
+ const addDurableWatcherPath = (absolutePath, relativePath) => {
3126
+ if (durableWatcherOverflowed)
3127
+ return;
3128
+ if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
3129
+ durableWatcherPaths.clear();
3130
+ durableWatcherOverflowed = true;
3131
+ }
3132
+ else if (!durableWatcherOverflowed) {
3133
+ durableWatcherPaths.set(absolutePath, relativePath);
3134
+ }
3135
+ scheduleDurableWatcherPersist();
3136
+ };
3137
+ const clearDurableWatcherOverflow = () => {
3138
+ if (!durableWatcherOverflowed)
3139
+ return;
3140
+ durableWatcherOverflowed = false;
3141
+ durableWatcherPaths.clear();
3142
+ for (const [absolutePath, relativePath] of pendingWatcherPaths) {
3143
+ addDurableWatcherPath(absolutePath, relativePath);
3144
+ }
3145
+ scheduleDurableWatcherPersist();
3146
+ };
3147
+ const acknowledgeDurableWatcherUploads = (relativePaths) => {
3148
+ let changed = false;
3149
+ const uploaded = new Set(relativePaths);
3150
+ for (const [absolutePath, relativePath] of durableWatcherPaths) {
3151
+ if (!uploaded.has(relativePath))
3152
+ continue;
3153
+ durableWatcherPaths.delete(absolutePath);
3154
+ changed = true;
3155
+ }
3156
+ if (changed)
3157
+ scheduleDurableWatcherPersist();
3158
+ };
3159
+ const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
3160
+ const stillPending = new Set(pendingWatcherPaths.values());
3161
+ acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
3162
+ };
3163
+ let activePassCount = 0;
3164
+ let inFlightWatcherPushes = 0;
3165
+ const watcherPushStages = new Map();
3166
+ let nextWatcherPushGroupId = 0;
3167
+ const setWatcherPushStage = (groupId, stage) => {
3168
+ const record = watcherPushStages.get(groupId);
3169
+ if (!record || record.stage === stage)
3170
+ return;
3171
+ record.stage = stage;
3172
+ record.stageEnteredAt = Date.now();
3173
+ if (stage === "guarded scheduler admission requested") {
3174
+ record.admissionRequestedAtMonotonicMs = performance.now();
3175
+ record.admissionWaiting = true;
3176
+ }
3177
+ else if (stage === "guarded scheduler admission granted") {
3178
+ record.admissionWaiting = false;
3179
+ }
3180
+ process.stderr.write(`${JSON.stringify({
3181
+ type: "watcher-push-group-stage",
3182
+ diagnostic: true,
3183
+ groupId: record.groupId,
3184
+ route: record.route,
3185
+ paths: record.paths,
3186
+ stage: record.stage,
3187
+ stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
3188
+ })}\n`);
3189
+ };
3190
+ const startWatcherPushStageTracking = (route, paths) => {
3191
+ const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
3192
+ watcherPushStages.set(groupId, {
3193
+ groupId,
3194
+ route,
3195
+ paths,
3196
+ stage: "",
3197
+ stageEnteredAt: Date.now(),
3198
+ admissionRequestedAtMonotonicMs: null,
3199
+ admissionWaiting: false,
3200
+ });
3201
+ return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
3202
+ };
3203
+ const emitWatcherPushInflightStages = () => {
3204
+ if (inFlightWatcherPushes <= 0)
3205
+ return;
3206
+ const now = Date.now();
3207
+ const entries = [...watcherPushStages.values()];
3208
+ const admission = passScheduler.diagnosticAdmissionSnapshot;
3209
+ const coordinatorAdmission = coordinator?.inspect();
3210
+ for (const record of entries) {
3211
+ if (!record.admissionWaiting)
3212
+ continue;
3213
+ const liveStage = resolveWatcherPushWaitStage({
3214
+ capYieldActive: passScheduler.isInCapYield,
3215
+ queueDepth: admission.queueDepth,
3216
+ holders: admission.holders,
3217
+ });
3218
+ if (liveStage !== record.stage)
3219
+ setWatcherPushStage(record.groupId, liveStage);
3220
+ }
3221
+ process.stderr.write(`${JSON.stringify({
3222
+ type: "watcher-push-inflight-stages",
3223
+ diagnostic: true,
3224
+ groups: entries.slice(0, 20).map((record) => {
3225
+ const monotonicNow = performance.now();
3226
+ const current = watcherPushWaitDiagnostics({
3227
+ stage: record.stage,
3228
+ admissionWaiting: record.admissionWaiting,
3229
+ admissionRequestedAtMonotonicMs: record.admissionRequestedAtMonotonicMs,
3230
+ lastAnyAdmissionAtMonotonicMs: admission.lastAnyAdmissionAtMonotonicMs,
3231
+ nowMonotonicMs: monotonicNow,
3232
+ capYieldActive: passScheduler.isInCapYield,
3233
+ queueDepth: admission.queueDepth,
3234
+ holders: admission.holders,
3235
+ });
3236
+ return {
3237
+ route: record.route,
3238
+ paths: record.paths,
3239
+ ...current,
3240
+ ageMs: Math.max(0, now - record.stageEnteredAt),
3241
+ };
3242
+ }),
3243
+ other: Math.max(0, entries.length - 20),
3244
+ realtimeAdmissionQueueDepth: admission.queueDepth,
3245
+ holders: admission.holders,
3246
+ coordinatorRealtimeQueueDepth: coordinatorAdmission?.realtimeQueued ?? 0,
3247
+ coordinatorRealtimeRunning: coordinatorAdmission?.realtimeRunning ?? false,
3248
+ coordinatorRealtimeLanes: coordinatorAdmission?.realtimeLanes ?? {
3249
+ fast: null,
3250
+ slow: null,
3251
+ overflowPush: null,
3252
+ },
3253
+ coordinatorChoreQueueDepth: coordinatorAdmission?.choreQueued ?? 0,
3254
+ coordinatorRealtimeLastAdmissionAgoMs: coordinatorAdmission?.lastRealtimeAdmissionAt === null ||
3255
+ coordinatorAdmission === undefined
3256
+ ? null
3257
+ : Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastRealtimeAdmissionAt),
3258
+ coordinatorChoreLastAdmissionAgoMs: coordinatorAdmission?.lastChoreAdmissionAt === null ||
3259
+ coordinatorAdmission === undefined
3260
+ ? null
3261
+ : Math.max(0, coordinatorAdmission.now - coordinatorAdmission.lastChoreAdmissionAt),
3262
+ })}\n`);
3263
+ };
3264
+ const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
3265
+ watcherPushStagesTimer.unref?.();
3266
+ let pendingReceivers = 0;
3267
+ let lastSyncWorkStateEmitAt = 0;
3268
+ let syncWorkStateTimer;
3269
+ const emitSyncWorkState = (force = false) => {
3270
+ const now = Date.now();
3271
+ const elapsed = now - lastSyncWorkStateEmitAt;
3272
+ if (force || elapsed >= 250) {
3273
+ if (syncWorkStateTimer !== undefined)
3274
+ clearTimeout(syncWorkStateTimer);
3275
+ syncWorkStateTimer = undefined;
3276
+ lastSyncWorkStateEmitAt = now;
3277
+ const snapshot = {
3278
+ passActive: activePassCount > 0,
3279
+ queuedWatcherPushes: pendingWatcherPaths.size,
3280
+ inFlightWatcherPushes,
3281
+ pendingReceivers,
3282
+ };
3283
+ deps.onSyncWorkStateSnapshot?.(snapshot);
3284
+ stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
3285
+ return;
3286
+ }
3287
+ if (syncWorkStateTimer !== undefined)
3288
+ return;
3289
+ syncWorkStateTimer = setTimeout(() => {
3290
+ syncWorkStateTimer = undefined;
3291
+ emitSyncWorkState(true);
3292
+ }, Math.max(1, 250 - elapsed));
3293
+ syncWorkStateTimer.unref?.();
3294
+ };
2492
3295
  const droppedUnresolvedWatcherSlugs = new Set();
2493
3296
  const logDroppedUnresolvedWatcherRoute = (slug) => {
2494
3297
  if (droppedUnresolvedWatcherSlugs.has(slug))
@@ -2531,10 +3334,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2531
3334
  const changes = new Map();
2532
3335
  for (const [absolutePath, relativePath] of batch.paths.entries()) {
2533
3336
  const change = batch.changes?.get(absolutePath);
2534
- const route = routeChangeToTarget(relativePath, change?.kind);
3337
+ let routedRelativePath = relativePath;
3338
+ const absoluteRootRelativePath = path.relative(hqRoot, absolutePath)
3339
+ .split(path.sep)
3340
+ .join("/");
3341
+ if (isSafeRelativePath(absoluteRootRelativePath)) {
3342
+ const absoluteRoute = routeChangeToTarget(absoluteRootRelativePath, change?.kind);
3343
+ const reportedRoute = routeChangeToTarget(relativePath, change?.kind);
3344
+ if (absoluteRoute?.kind === "company" &&
3345
+ (reportedRoute?.kind !== "company" ||
3346
+ reportedRoute.slug.toLowerCase() !== absoluteRoute.slug.toLowerCase())) {
3347
+ routedRelativePath = absoluteRootRelativePath;
3348
+ }
3349
+ }
3350
+ const route = routeChangeToTarget(routedRelativePath, change?.kind);
2535
3351
  if (!route || !isWatchRouteSelected(route, watchRouteSelection))
2536
3352
  continue;
2537
- paths.set(absolutePath, relativePath);
3353
+ paths.set(absolutePath, routedRelativePath);
2538
3354
  if (change)
2539
3355
  changes.set(absolutePath, change);
2540
3356
  }
@@ -2556,6 +3372,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2556
3372
  }
2557
3373
  }
2558
3374
  if (batch) {
3375
+ if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
3376
+ durableWatcherPaths.clear();
3377
+ durableWatcherOverflowed = true;
3378
+ scheduleDurableWatcherPersist();
3379
+ }
2559
3380
  const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
2560
3381
  const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
2561
3382
  return previousSequence === undefined ||
@@ -2564,6 +3385,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2564
3385
  });
2565
3386
  if (entries.length === 0)
2566
3387
  return false;
3388
+ const queueWasEmpty = pendingWatcherPaths.size === 0;
2567
3389
  pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
2568
3390
  pendingWatcherOriginalBatch =
2569
3391
  pendingWatcherPaths.size === 0 &&
@@ -2575,11 +3397,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2575
3397
  : null;
2576
3398
  for (const [absolutePath, relativePath] of entries) {
2577
3399
  pendingWatcherPaths.set(absolutePath, relativePath);
3400
+ addDurableWatcherPath(absolutePath, relativePath);
2578
3401
  const change = batch.changes?.get(absolutePath);
2579
3402
  if (change)
2580
3403
  pendingWatcherChanges.set(absolutePath, change);
2581
3404
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2582
3405
  }
3406
+ emitSyncWorkState(queueWasEmpty);
2583
3407
  if (batch.overflowed) {
2584
3408
  pendingWatcherOverflowed = true;
2585
3409
  pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
@@ -2603,16 +3427,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2603
3427
  // counterpart must be re-derived through the codec — a raw join would
2604
3428
  // point at the impossible colon path on Windows.
2605
3429
  const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
3430
+ const queueWasEmpty = pendingWatcherPaths.size === 0;
2606
3431
  pendingWatcherPaths.set(absolutePath, changedRelPath);
3432
+ addDurableWatcherPath(absolutePath, changedRelPath);
2607
3433
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2608
3434
  // A bare path signal carries no delete intent; drop any change record
2609
3435
  // a prior batch left for this path so the stale intent cannot leak
2610
3436
  // into the coalesced batch (#227).
2611
3437
  pendingWatcherChanges.delete(absolutePath);
3438
+ emitSyncWorkState(queueWasEmpty);
2612
3439
  return true;
2613
3440
  }
2614
3441
  pendingWatcherOriginalBatch = null;
2615
3442
  pendingWatcherBareChange = true;
3443
+ emitSyncWorkState();
2616
3444
  return true;
2617
3445
  };
2618
3446
  const takePendingWatcherChange = () => {
@@ -2657,6 +3485,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2657
3485
  pendingWatcherDroppedRouteHints.clear();
2658
3486
  pendingWatcherDroppedRoutesUnknown = false;
2659
3487
  pendingWatcherBatchGeneration = null;
3488
+ emitSyncWorkState();
2660
3489
  return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
2661
3490
  };
2662
3491
  const restorePendingWatcherPaths = (paths, generation, changes) => {
@@ -2670,12 +3499,54 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2670
3499
  if (pendingWatcherChangeSequences.has(absolutePath))
2671
3500
  continue;
2672
3501
  pendingWatcherPaths.set(absolutePath, relativePath);
3502
+ addDurableWatcherPath(absolutePath, relativePath);
2673
3503
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2674
3504
  const change = changes?.get(absolutePath);
2675
3505
  if (change)
2676
3506
  pendingWatcherChanges.set(absolutePath, change);
2677
3507
  }
3508
+ emitSyncWorkState();
2678
3509
  };
3510
+ if (eventPush) {
3511
+ try {
3512
+ const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
3513
+ if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
3514
+ (!Array.isArray(stored.paths) && stored.overflowed !== true)) {
3515
+ throw new Error("stored watcher push state has an unsupported shape or root");
3516
+ }
3517
+ if (stored.overflowed === true) {
3518
+ durableWatcherOverflowed = true;
3519
+ pendingWatcherOverflowed = true;
3520
+ pendingWatcherWarmupCatchup = true;
3521
+ pendingWatcherDroppedRoutesUnknown = true;
3522
+ pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3523
+ emitSyncWorkState(true);
3524
+ process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
3525
+ }
3526
+ for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
3527
+ if (typeof relativePath !== "string" || relativePath.length === 0)
3528
+ continue;
3529
+ const route = routeChangeToTarget(relativePath);
3530
+ if (!route || !isWatchRouteSelected(route, watchRouteSelection))
3531
+ continue;
3532
+ const absolutePath = localPathForVaultKey(hqRoot, relativePath);
3533
+ addDurableWatcherPath(absolutePath, relativePath);
3534
+ pendingWatcherPaths.set(absolutePath, relativePath);
3535
+ pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
3536
+ }
3537
+ if (pendingWatcherPaths.size > 0) {
3538
+ pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3539
+ pendingWatcherWarmupCatchup = true;
3540
+ emitSyncWorkState(true);
3541
+ process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
3542
+ }
3543
+ }
3544
+ catch (err) {
3545
+ if (!(err instanceof Error && err.code === "ENOENT")) {
3546
+ process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
3547
+ }
3548
+ }
3549
+ }
2679
3550
  const routePushFailureBackoff = new RoutePushFailureBackoff();
2680
3551
  const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
2681
3552
  const routePushRetryTimerHandles = new Map();
@@ -3256,14 +4127,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3256
4127
  const relativePath = route.kind === "personal"
3257
4128
  ? key
3258
4129
  : ["companies", route.slug, key].join("/");
3259
- const localPath = path.resolve(hqRoot, ...relativePath.split("/"));
3260
- const fromRoot = path.relative(hqRoot, localPath);
3261
- const parentPrefix = ["..", path.sep].join("");
3262
- if (fromRoot === ".." ||
3263
- fromRoot.startsWith(parentPrefix) ||
3264
- path.isAbsolute(fromRoot)) {
4130
+ // Journal keys are POSIX relative paths. Reject unsafe segments with
4131
+ // a string-only check before building the absolute path; path.resolve
4132
+ // plus path.relative allocated and normalized every journal row.
4133
+ if (!isSafeRelativePath(relativePath))
3265
4134
  continue;
3266
- }
4135
+ const localPath = path.join(hqRoot, relativePath);
3267
4136
  if (shouldSync(localPath)) {
3268
4137
  paths.push(relativePath);
3269
4138
  continue;
@@ -3428,36 +4297,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3428
4297
  }
3429
4298
  }
3430
4299
  };
3431
- const groupBatchByRoute = (batch) => {
3432
- const grouped = new Map();
3433
- for (const [absolutePath, relativePath] of batch.paths.entries()) {
3434
- const change = batch.changes?.get(absolutePath);
3435
- const route = routeChangeToTarget(relativePath, change?.kind);
3436
- if (!route)
3437
- continue;
3438
- const category = isSessionLogsPushOnlyPath(relativePath) ? "session-log-backup" : "interactive";
3439
- const key = `${routeKey(route)}:${category}`;
3440
- const existing = grouped.get(key) ??
3441
- {
3442
- route,
3443
- paths: new Map(),
3444
- changes: new Map(),
3445
- relPaths: [],
3446
- deleteScopeRoots: [],
3447
- };
3448
- existing.paths.set(absolutePath, relativePath);
3449
- if (change)
3450
- existing.changes.set(absolutePath, change);
3451
- existing.relPaths = [...new Set([...existing.paths.values()])];
3452
- existing.deleteScopeRoots = [
3453
- ...new Set([...existing.changes.values()].flatMap((item) => (item.deleteSnapshots ?? []).flatMap((snapshot) => snapshot.deleteScopeRoot === undefined
3454
- ? []
3455
- : [snapshot.deleteScopeRoot]))),
3456
- ];
3457
- grouped.set(key, existing);
3458
- }
3459
- return grouped;
3460
- };
3461
4300
  /** Resolve deferred delete snapshots immediately before their scoped drain. */
3462
4301
  const resolveDeleteSnapshotCaptures = async (batch) => {
3463
4302
  for (const [absolutePath, change] of batch.changes ?? []) {
@@ -3526,8 +4365,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3526
4365
  // a file and still be refused as stale (a download changed the directory
3527
4366
  // mid-pass); the retry then finds the file already in the cloud and uploads
3528
4367
  // nothing, so without this the file is never announced to peers.
3529
- const uploadedAwaitingPublish = new Set();
3530
- const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
3531
4368
  const rememberUploadedPaths = (result) => {
3532
4369
  if (result === undefined || typeof result === "number")
3533
4370
  return;
@@ -3540,6 +4377,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3540
4377
  const oldest = uploadedAwaitingPublish.values().next().value;
3541
4378
  if (oldest !== undefined)
3542
4379
  uploadedAwaitingPublish.delete(oldest);
4380
+ if (!uploadedAwaitingPublishDropLogged) {
4381
+ uploadedAwaitingPublishDropLogged = true;
4382
+ process.stderr.write(`${JSON.stringify({
4383
+ type: "event-sync-upload-buffer-overflow",
4384
+ cap: UPLOADED_AWAITING_PUBLISH_CAP,
4385
+ action: "drop-oldest",
4386
+ })}\n`);
4387
+ }
3543
4388
  }
3544
4389
  }
3545
4390
  };
@@ -3556,8 +4401,37 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3556
4401
  };
3557
4402
  let emitScopedPushFailureTelemetry = null;
3558
4403
  const runScopedDrain = async (batch, watcherBatchGeneration, _wakeOnPushFailure = false, allowFastRealtimeOvertake = false) => {
3559
- if (batch)
3560
- await resolveDeleteSnapshotCaptures(batch);
4404
+ if (batch) {
4405
+ // Read each routed quarantine shard once for this watcher batch. Filter
4406
+ // before resolving delete snapshots or submitting work to the coordinator
4407
+ // so a quarantined key cannot acquire a new upload/delete intent.
4408
+ const pathsByJournal = new Map();
4409
+ let unresolvedCount = 0;
4410
+ for (const [absolutePath, relativePath] of [...batch.paths]) {
4411
+ const kind = batch.changes?.get(absolutePath)?.kind;
4412
+ const coordinates = journalCoordinates(relativePath, kind);
4413
+ if (!coordinates)
4414
+ continue;
4415
+ let quarantined = pathsByJournal.get(coordinates.slug);
4416
+ if (!quarantined) {
4417
+ quarantined = readQuarantinedJournalPaths(coordinates.slug);
4418
+ pathsByJournal.set(coordinates.slug, quarantined);
4419
+ }
4420
+ if (!quarantined.has(coordinates.key))
4421
+ continue;
4422
+ batch.paths.delete(absolutePath);
4423
+ batch.changes?.delete(absolutePath);
4424
+ unresolvedCount += 1;
4425
+ }
4426
+ if (unresolvedCount > 0) {
4427
+ (deps.stderr ?? process.stderr).write(`${JSON.stringify({
4428
+ type: "journal-quarantine-unresolved",
4429
+ count: unresolvedCount,
4430
+ })}\n`);
4431
+ }
4432
+ if (batch.paths.size > 0)
4433
+ await resolveDeleteSnapshotCaptures(batch);
4434
+ }
3561
4435
  const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
3562
4436
  const skipCompanies = grouped ? resolveSkipCompanies() : null;
3563
4437
  const isPausedRoute = (route) => route.kind === "company" &&
@@ -3582,7 +4456,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3582
4456
  return routeFailure?.message ?? failures[0]?.message ??
3583
4457
  `sync pass exited ${result.exitCode}`;
3584
4458
  };
3585
- const recordPushFailure = (route, exitCode, message) => {
4459
+ const recordPushFailure = (route, exitCode, message, caughtError, caughtErrorClass) => {
3586
4460
  const routeId = routeKey(route);
3587
4461
  const companyRoot = route.kind === "company"
3588
4462
  ? path.join(hqRoot, "companies", route.slug)
@@ -3599,11 +4473,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3599
4473
  nextRetryDelayMs: state.delayMs,
3600
4474
  errorSummary: pushFailureTelemetryLabel(errorText),
3601
4475
  };
3602
- const diagnostic = {
3603
- type: "scoped-push-failure",
3604
- ...event,
3605
- error: errorText,
3606
- };
4476
+ const diagnostic = buildScopedPushFailureDiagnostic(event, errorText, caughtError, caughtErrorClass);
3607
4477
  (deps.stderr ?? process.stderr).write(`${JSON.stringify(diagnostic)}\n`);
3608
4478
  if (deps.emitScopedPushFailureTelemetry) {
3609
4479
  deps.emitScopedPushFailureTelemetry(event);
@@ -3654,19 +4524,86 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3654
4524
  ? localUploadSizeClass(uploadPaths)
3655
4525
  : "small";
3656
4526
  const largeUpload = sizeClass !== "small";
3657
- const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
3658
- const execution = parallelCoordinatorEnabled && !chorePush
3659
- ? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
3660
- const admittedPaths = slice.paths.map((path) => path.canonicalPath);
3661
- const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
3662
- const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
3663
- return runGuarded(scopedArgv, admittedChanges.size > 0
3664
- ? () => prepareWatcherChanges(admittedChanges)
3665
- : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
3666
- }, sizeClass)
3667
- : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
3668
- ? () => prepareWatcherChanges(group.changes)
3669
- : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
4527
+ // With coordinator fencing enabled, even large watcher batches can
4528
+ // use bounded realtime admission. Keeping them as chores would make
4529
+ // them wait behind a long-lived receiver pull's slot indefinitely.
4530
+ const chorePush = isChoreLocalPush(group.relPaths, largeUpload && !parallelCoordinatorEnabled);
4531
+ const { groupId, setStage: watcherPushStage } = startWatcherPushStageTracking(routeId, group.relPaths.length);
4532
+ watcherPushStage("coordinator submit");
4533
+ inFlightWatcherPushes += group.relPaths.length;
4534
+ emitSyncWorkState(true);
4535
+ emitWatcherPushInflightStages();
4536
+ let execution;
4537
+ try {
4538
+ execution = await (parallelCoordinatorEnabled && !chorePush
4539
+ ? runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry) => {
4540
+ watcherPushStage("worker entry");
4541
+ const candidatePaths = slice.paths.map((path) => path.canonicalPath);
4542
+ const changeByRelativePath = new Map([...group.paths.entries()].map(([absolutePath, relativePath]) => [
4543
+ relativePath,
4544
+ group.changes.get(absolutePath),
4545
+ ]));
4546
+ const refusedPaths = new Set();
4547
+ const admittedPaths = [];
4548
+ for (const relativePath of candidatePaths) {
4549
+ const kind = changeByRelativePath.get(relativePath)?.kind;
4550
+ const isUpsert = kind === undefined || kind === "add" || kind === "change";
4551
+ if (isUpsert && await isUnchangedAppliedUpsert(relativePath)) {
4552
+ refusedPaths.add(relativePath);
4553
+ }
4554
+ else {
4555
+ // Admission means this is a local edit (or a delete); it
4556
+ // supersedes any stale receiver echo marker.
4557
+ appliedUpsertHashesForWatcher.delete(relativePath);
4558
+ admittedPaths.push(relativePath);
4559
+ }
4560
+ }
4561
+ if (refusedPaths.size > 0) {
4562
+ const execution = coordinatorExecutions.get(slice.work);
4563
+ if (execution)
4564
+ execution.refusedPaths = refusedPaths;
4565
+ process.stderr.write(`${JSON.stringify({
4566
+ type: "watcher-applied-upsert-echo-suppressed",
4567
+ count: refusedPaths.size,
4568
+ })}\n`);
4569
+ }
4570
+ if (admittedPaths.length === 0) {
4571
+ if (refusedPaths.size > 0 && eventSyncLive && eventSync) {
4572
+ // The emitter's empty-batch outcome is the settlement
4573
+ // signal for this completed echo check. No path is
4574
+ // published, but receivers and diagnostics can still
4575
+ // observe that the watcher drained the batch.
4576
+ eventSync.publishBatch({ paths: new Map() });
4577
+ }
4578
+ return 0;
4579
+ }
4580
+ const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
4581
+ const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
4582
+ return runGuarded(scopedArgv, admittedChanges.size > 0
4583
+ ? () => prepareWatcherChanges(admittedChanges)
4584
+ : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry,
4585
+ // Watcher-originated pushes may share the guarded
4586
+ // realtime slot with an in-flight pull. Coordinator
4587
+ // fencing already serializes overlapping path epochs;
4588
+ // this must not depend on the optional full-pass flag.
4589
+ true, undefined, routeId, watcherPushStage);
4590
+ }, sizeClass)
4591
+ : (() => {
4592
+ watcherPushStage("worker entry");
4593
+ return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
4594
+ ? () => prepareWatcherChanges(group.changes)
4595
+ : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
4596
+ result,
4597
+ accepted: true,
4598
+ coordinatorRefusedPaths: [],
4599
+ }));
4600
+ })());
4601
+ }
4602
+ finally {
4603
+ watcherPushStages.delete(groupId);
4604
+ inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
4605
+ emitSyncWorkState(true);
4606
+ }
3670
4607
  const result = execution.result;
3671
4608
  rememberUploadedPaths(result);
3672
4609
  const pushExit = passExitCode(result);
@@ -3677,6 +4614,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3677
4614
  const refusedPaths = new Set(pushPathResults
3678
4615
  .filter((pathResult) => pathResult.status === "refused")
3679
4616
  .map((pathResult) => pathResult.relativePath));
4617
+ for (const pathResult of pushPathResults) {
4618
+ if (pathResult.status === "accepted" && pathResult.operation !== "delete") {
4619
+ appliedUpsertHashesForWatcher.delete(pathResult.relativePath);
4620
+ }
4621
+ }
3680
4622
  const successfullyDeletedPaths = new Set(pushPathResults
3681
4623
  .filter((pathResult) => pathResult.status === "accepted" && pathResult.operation === "delete")
3682
4624
  .map((pathResult) => pathResult.relativePath));
@@ -3685,6 +4627,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3685
4627
  }
3686
4628
  const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
3687
4629
  relativePath.startsWith(`${refusedPath}/`));
4630
+ acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
3688
4631
  const publishPaths = new Map();
3689
4632
  for (const [absolutePath, relativePath] of group.paths) {
3690
4633
  if (isRefusedPath(relativePath))
@@ -3724,16 +4667,21 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3724
4667
  deferPushAfterAdmissionTimeout(group, routeId);
3725
4668
  }
3726
4669
  else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
4670
+ acknowledgeDurableWatcherUploads(group.relPaths);
3727
4671
  routePushFailureBackoff.succeeded(routeId);
3728
4672
  clearRoutePushRetry(routeId);
3729
4673
  if (group.route.kind === "company") {
4674
+ deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
3730
4675
  logDroppedUnresolvedWatcherRoute(group.route.slug);
3731
4676
  }
3732
4677
  }
3733
4678
  else {
4679
+ // A failure before pass invocation (including journal preparation)
4680
+ // has no durable upload outcome. Keep every path pending so the
4681
+ // next scoped drain retries it after the failed coordinator slice.
3734
4682
  restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
3735
4683
  if (pushExit !== 0) {
3736
- recordPushFailure(group.route, pushExit, pushFailureText(result, group.route));
4684
+ recordPushFailure(group.route, pushExit, pushFailureText(result, group.route), undefined, execution.errorClass);
3737
4685
  }
3738
4686
  else if (_wakeOnPushFailure) {
3739
4687
  interruptWait();
@@ -3745,7 +4693,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3745
4693
  deferPushAfterAdmissionTimeout(group, routeId);
3746
4694
  continue;
3747
4695
  }
3748
- recordPushFailure(group.route, 1, pushFailureText(1, group.route, err));
4696
+ recordPushFailure(group.route, 1, pushFailureText(1, group.route, err), err);
3749
4697
  restorePendingWatcherPaths(group.paths, watcherBatchGeneration, group.changes);
3750
4698
  }
3751
4699
  }
@@ -3788,7 +4736,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3788
4736
  const execution = parallelCoordinatorEnabled
3789
4737
  ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
3790
4738
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
3791
- return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
4739
+ return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route));
3792
4740
  })
3793
4741
  : await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
3794
4742
  if (error instanceof RealtimeAdmissionTimeoutError) {
@@ -3840,6 +4788,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3840
4788
  addRoute(hint);
3841
4789
  const skipCompanies = resolveSkipCompanies();
3842
4790
  if (parallelCoordinatorEnabled) {
4791
+ let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
4792
+ // An overflow whose known touched routes are all explicitly disabled
4793
+ // has no chores to enqueue, so no completion callback can clear it.
4794
+ // Clear only when there was at least one known selected route; an empty
4795
+ // route set still means the overflow was not safely attributable here.
4796
+ if (routes.size > 0 && routesRemaining === 0) {
4797
+ clearDurableWatcherOverflow();
4798
+ return 0;
4799
+ }
3843
4800
  for (const route of routes.values()) {
3844
4801
  if (route.kind === "company" &&
3845
4802
  skipCompanies.has(route.slug.toLowerCase())) {
@@ -3868,6 +4825,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3868
4825
  process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
3869
4826
  interruptWait();
3870
4827
  }
4828
+ else {
4829
+ routesRemaining -= 1;
4830
+ if (routesRemaining === 0)
4831
+ clearDurableWatcherOverflow();
4832
+ }
3871
4833
  });
3872
4834
  }
3873
4835
  return 0;
@@ -3983,6 +4945,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3983
4945
  // churn overflowed the 200-path batch and skipped publishBatch.
3984
4946
  personalMode: true,
3985
4947
  includeCompanyPaths: companiesMode || companyMode,
4948
+ // Inventory filtered candidates so the route's push filter remains
4949
+ // authoritative and can attribute exclusions in warm-up diagnostics.
4950
+ includeFilteredWarmupCandidates: true,
3986
4951
  onDegraded: opts.onDegraded,
3987
4952
  captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
3988
4953
  queueLocalDeleteSnapshots: opts.queueLocalDeleteSnapshots,
@@ -4104,6 +5069,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4104
5069
  }
4105
5070
  }
4106
5071
  watcher.start();
5072
+ let catchupPathStatWarningReported = false;
4107
5073
  const reportWatcherReady = () => {
4108
5074
  if (stopped || watcher === null)
4109
5075
  return;
@@ -4144,14 +5110,202 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4144
5110
  : watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
4145
5111
  if (!catchup || catchup.cancelled || stopped || watcher === null)
4146
5112
  return;
4147
- const queued = addPendingWatcherChange(undefined, catchup.batch, {
5113
+ const skipStore = deps.watcherCatchupSkipStore;
5114
+ const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
5115
+ const eligiblePaths = new Map();
5116
+ const eligibleChanges = new Map();
5117
+ const excludedByReason = {};
5118
+ const excludedByReasonByRoute = {};
5119
+ const personalFilterExclusions = new Map();
5120
+ const continuityPaths = new Set(computeContinuityPointerPaths(hqRoot).map((p) => path.relative(hqRoot, p).split(path.sep).join("/")));
5121
+ const hasAgencyCarveout = computeAgencySyncPaths(hqRoot).length > 0;
5122
+ const hasSessionLogsCarveout = computeSessionLogsSyncPaths(hqRoot).length > 0;
5123
+ const { ignoreFilter, personalPushFilter } = createWatcherWarmupPathFilters(hqRoot, (relativePath, exclusionId) => personalFilterExclusions.set(relativePath, exclusionId));
5124
+ const clearedResolvedCompanies = new Set();
5125
+ const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
5126
+ const candidateRouteCounts = new Map();
5127
+ const candidateRoutePrefixes = new Map();
5128
+ const eligibleRouteCounts = new Map();
5129
+ const eligibleRoutePrefixes = new Map();
5130
+ const backupClass = (relativePath) => {
5131
+ if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
5132
+ return "attioCleanup";
5133
+ if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
5134
+ return "readModelRollout";
5135
+ return undefined;
5136
+ };
5137
+ const exclude = (reason, routeName) => {
5138
+ excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
5139
+ const routeReasons = excludedByReasonByRoute[routeName] ??= {};
5140
+ routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
5141
+ };
5142
+ let filteredCandidateCount = 0;
5143
+ for (const [reason, count] of Object.entries(catchup.filteredWarmupCandidateCounts ?? {})) {
5144
+ filteredCandidateCount += count;
5145
+ excludedByReason[reason] = (excludedByReason[reason] ?? 0) + count;
5146
+ const routeReasons = excludedByReasonByRoute.filtered ??= {};
5147
+ routeReasons[reason] = (routeReasons[reason] ?? 0) + count;
5148
+ }
5149
+ if (filteredCandidateCount > 0)
5150
+ candidateRouteCounts.set("filtered", filteredCandidateCount);
5151
+ let examinedWarmupPaths = 0;
5152
+ for (const [absolutePath, relativePath] of catchup.batch.paths) {
5153
+ examinedWarmupPaths += 1;
5154
+ const warmupYield = yieldWatcherWarmupIfNeeded(examinedWarmupPaths);
5155
+ if (warmupYield)
5156
+ await warmupYield;
5157
+ if (catchup.safeRelativePaths !== true && !isSafeRelativePath(relativePath)) {
5158
+ recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, "unrouted", relativePath);
5159
+ exclude("unsafe-path", "unrouted");
5160
+ continue;
5161
+ }
5162
+ const route = routeChangeToTarget(relativePath);
5163
+ const routeName = route ? routeKey(route) : "unrouted";
5164
+ const summaryRoutePath = route?.kind === "company"
5165
+ ? relativePath.slice(`companies/${route.slug}/`.length)
5166
+ : relativePath;
5167
+ recordWarmupRoutePath(candidateRouteCounts, candidateRoutePrefixes, routeName, summaryRoutePath);
5168
+ const ontologyBackupClass = backupClass(relativePath);
5169
+ if (ontologyBackupClass) {
5170
+ exclude("local-rule:ontology-backup", routeName);
5171
+ ontologyBackupPaths[ontologyBackupClass] += 1;
5172
+ continue;
5173
+ }
5174
+ if (route?.kind === "company") {
5175
+ const resolvedNow = cloudCompanySlugs.has(route.slug) ||
5176
+ skipStore?.hasCurrentGrantCompany(route.slug) ||
5177
+ skipStore?.hasCurrentGrantVersion(route.slug) ||
5178
+ (companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
5179
+ if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
5180
+ skipStore?.clearUnresolvedCompany(route.slug);
5181
+ clearedResolvedCompanies.add(route.slug);
5182
+ }
5183
+ if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
5184
+ exclude("unresolved-company", routeName);
5185
+ continue;
5186
+ }
5187
+ }
5188
+ if (route?.kind === "company" &&
5189
+ !cloudCompanySlugs.has(route.slug) &&
5190
+ !skipStore?.hasCurrentGrantCompany(route.slug) &&
5191
+ !skipStore?.hasCurrentGrantVersion(route.slug) &&
5192
+ !(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
5193
+ exclude("local-only-company", routeName);
5194
+ continue;
5195
+ }
5196
+ const company = route?.kind === "company" ? route.slug : "personal";
5197
+ const routePath = route?.kind === "company"
5198
+ ? relativePath.slice(`companies/${route.slug}/`.length)
5199
+ : relativePath;
5200
+ if (isEphemeralPath(routePath)) {
5201
+ exclude("push-filter:conflict-mirror", routeName);
5202
+ continue;
5203
+ }
5204
+ // The production TreeWatcher collector has already applied this
5205
+ // shared filter stack and reports rejected files as reason counts.
5206
+ // Injected/custom collectors without that proof still evaluate the
5207
+ // route filter here, with personal defaults first for `.env` attribution.
5208
+ const personalDefaultExclusion = catchup.pushFiltersApplied === true
5209
+ ? undefined
5210
+ : route?.kind === "personal"
5211
+ ? matchPersonalVaultExclusion(relativePath, false)
5212
+ : undefined;
5213
+ const isPersonalRoute = route?.kind !== "company";
5214
+ const acceptedByPushFilter = catchup.pushFiltersApplied === true || (!personalDefaultExclusion && (isPersonalRoute
5215
+ ? personalPushFilter(absolutePath, false, relativePath)
5216
+ : ignoreFilter(absolutePath, false, relativePath)));
5217
+ if (!acceptedByPushFilter) {
5218
+ const exclusionId = personalDefaultExclusion?.id ?? (route?.kind === "personal"
5219
+ ? personalFilterExclusions.get(relativePath)
5220
+ : undefined);
5221
+ exclude(exclusionId
5222
+ ? `push-filter:personal:${exclusionId}`
5223
+ : "push-filter:ignore", routeName);
5224
+ continue;
5225
+ }
5226
+ if (isPersonalRoute &&
5227
+ !shouldSkipPersonalWarmupScopeCheck(catchup.pushFiltersApplied === true, route?.kind) &&
5228
+ isPersonalVaultScopeExcluded(relativePath, {
5229
+ continuityPaths,
5230
+ hasAgencyCarveout,
5231
+ hasSessionLogsCarveout,
5232
+ })) {
5233
+ exclude("push-filter:personal:excluded-top-level", routeName);
5234
+ continue;
5235
+ }
5236
+ if (route && isLocallyIneligibleForPush(routePath, {
5237
+ boardClientOwned: true,
5238
+ ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
5239
+ })) {
5240
+ const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
5241
+ exclude(`local-rule:${pathClass}`, routeName);
5242
+ continue;
5243
+ }
5244
+ const knownSymlink = catchup.symlinkPaths?.has(absolutePath) ?? false;
5245
+ const hasSkipRecord = skipStore?.hasRecord(company, routePath) ?? false;
5246
+ try {
5247
+ const stat = !catchup.symlinkPaths || hasSkipRecord
5248
+ ? fs.lstatSync(absolutePath)
5249
+ : undefined;
5250
+ if (route && (knownSymlink || stat?.isSymbolicLink())) {
5251
+ const scopeRoot = route.kind === "company"
5252
+ ? path.join(hqRoot, "companies", route.slug)
5253
+ : relativePath.startsWith("personal/")
5254
+ ? path.join(hqRoot, "personal")
5255
+ : hqRoot;
5256
+ const inspection = inspectLocalSymlinkScope({
5257
+ absoluteLinkPath: absolutePath,
5258
+ linkPath: relativePath,
5259
+ scopeRoot,
5260
+ scope: route.kind === "company"
5261
+ ? { kind: "company", companySlug: route.slug }
5262
+ : { kind: "personal" },
5263
+ });
5264
+ if (inspection.refused) {
5265
+ exclude("symlink", routeName);
5266
+ continue;
5267
+ }
5268
+ }
5269
+ if (hasSkipRecord && stat &&
5270
+ skipStore?.matches(company, routePath, stat)) {
5271
+ exclude("record-match", routeName);
5272
+ continue;
5273
+ }
5274
+ }
5275
+ catch (error) {
5276
+ // Fail open: unreadable state or a filesystem race must not suppress a candidate.
5277
+ if (!catchupPathStatWarningReported) {
5278
+ catchupPathStatWarningReported = true;
5279
+ process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
5280
+ }
5281
+ }
5282
+ eligiblePaths.set(absolutePath, relativePath);
5283
+ recordWarmupRoutePath(eligibleRouteCounts, eligibleRoutePrefixes, routeName, summaryRoutePath);
5284
+ const change = catchup.batch.changes?.get(absolutePath);
5285
+ if (change)
5286
+ eligibleChanges.set(absolutePath, change);
5287
+ }
5288
+ const queued = addPendingWatcherChange(undefined, {
5289
+ paths: eligiblePaths,
5290
+ changes: eligibleChanges,
5291
+ }, {
4148
5292
  preserveChangesAfter: liveChangeSequenceAtWarmupStart,
4149
5293
  });
4150
5294
  pendingWatcherWarmupCatchup ||= queued;
5295
+ const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
4151
5296
  process.stderr.write(JSON.stringify({
4152
5297
  type: "watcher-warmup-catchup",
4153
5298
  reason: "watch-set-ready",
4154
- pathsFound: catchup.batch.paths.size,
5299
+ pathsFound: eligiblePaths.size,
5300
+ candidatePathsByRoute: rankedWarmupCounts(candidateRouteCounts),
5301
+ pathsByRoute: rankedWarmupCounts(eligibleRouteCounts),
5302
+ pathPrefixesByRoute: rankedWarmupPrefixes(candidateRouteCounts, candidateRoutePrefixes, eligibleRouteCounts, eligibleRoutePrefixes),
5303
+ excludedByReason: {
5304
+ ...excludedByReason,
5305
+ ...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
5306
+ },
5307
+ excludedByReasonByRoute,
5308
+ ontologyBackupPaths,
4155
5309
  scannedPaths: catchup.scannedPaths,
4156
5310
  elapsedMs: Date.now() - startedAt,
4157
5311
  queued,
@@ -4218,13 +5372,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4218
5372
  const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
4219
5373
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
4220
5374
  const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
4221
- return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
5375
+ return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal, routeKey(route)));
4222
5376
  });
4223
5377
  const pathCount = events.length;
4224
5378
  // Rethrow the typed refusal so the receiver keeps the lease and
4225
5379
  // retries the whole batch, instead of bisecting a "failed" pull.
4226
5380
  if (execution.admissionTimeout)
4227
5381
  throw execution.admissionTimeout;
5382
+ if (execution.yieldedToPush) {
5383
+ throw new RealtimePushYieldError();
5384
+ }
4228
5385
  const exitCode = passExitCode(execution.result);
4229
5386
  if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
4230
5387
  if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
@@ -4246,74 +5403,82 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4246
5403
  const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
4247
5404
  if (stopped)
4248
5405
  return;
4249
- // The injectable receiver seam and the production SQS receiver both
4250
- // arrive here only after a successful receive. Keep the cadence tight
4251
- // until the receiver later reports an empty outer long-poll.
4252
- onReceiveActivity(true);
4253
- // A received wake/high-water signal means there is bounded work now.
4254
- // It wakes the scheduler but is still funneled through runGuarded below,
4255
- // so it cannot overlap the pass that may currently own the operation
4256
- // lock.
4257
- // Apply the caps once across the whole delivery, then retain the residue
4258
- // in the normal lane. The receiver already coalesces to one latest event
4259
- // per path, so the partition cannot split revisions of one path.
4260
- if (parallelCoordinatorEnabled) {
4261
- const lane = receiverLaneByBatch.get(events);
4262
- if (lane) {
4263
- receiverLaneByBatch.delete(events);
4264
- await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
5406
+ try {
5407
+ // The injectable receiver seam and the production SQS receiver both
5408
+ // arrive here only after a successful receive. Keep the cadence tight
5409
+ // until the receiver later reports an empty outer long-poll.
5410
+ onReceiveActivity(true);
5411
+ pendingReceivers += events.length;
5412
+ emitSyncWorkState();
5413
+ // A received wake/high-water signal means there is bounded work now.
5414
+ // It wakes the scheduler but is still funneled through runGuarded below,
5415
+ // so it cannot overlap the pass that may currently own the operation
5416
+ // lock.
5417
+ // Apply the caps once across the whole delivery, then retain the residue
5418
+ // in the normal lane. The receiver already coalesces to one latest event
5419
+ // per path, so the partition cannot split revisions of one path.
5420
+ if (parallelCoordinatorEnabled) {
5421
+ const lane = receiverLaneByBatch.get(events);
5422
+ if (lane) {
5423
+ receiverLaneByBatch.delete(events);
5424
+ await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
5425
+ return;
5426
+ }
5427
+ // Direct embedders may invoke the engine without first going through
5428
+ // SqsPushReceiver. Preserve that seam while applying the same
5429
+ // delivery-wide partition the receiver would expose.
5430
+ const lanes = partitionByKnownSizes(events, receiverEventSize);
5431
+ await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
5432
+ await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4265
5433
  return;
4266
5434
  }
4267
- // Direct embedders may invoke the engine without first going through
4268
- // SqsPushReceiver. Preserve that seam while applying the same
4269
- // delivery-wide partition the receiver would expose.
4270
5435
  const lanes = partitionByKnownSizes(events, receiverEventSize);
4271
- await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
4272
- await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4273
- return;
4274
- }
4275
- const lanes = partitionByKnownSizes(events, receiverEventSize);
4276
- const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
4277
- const eventsByRoute = new Map();
4278
- for (const event of laneEvents) {
4279
- const route = routeChangeToTarget(event.relativePath);
4280
- // The receiver is an independent ingress from the local watcher.
4281
- // Apply the same route-selection boundary before it can enqueue a
4282
- // fast pull: a --skip-personal runner must acknowledge, not pull,
4283
- // a personal-namespace delivery.
4284
- if (!route || !isWatchRouteSelected(route, watchRouteSelection))
4285
- continue;
4286
- const key = routeKey(route);
4287
- const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
4288
- group.events.push(event);
4289
- if (event.kind === "upsert")
4290
- group.upsertPaths.push(event.relativePath);
4291
- eventsByRoute.set(key, group);
4292
- }
4293
- for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
4294
- const paths = routeEvents.map((event) => event.relativePath);
4295
- const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
4296
- const exitCode = passExitCode(result);
4297
- if (exitCode === UNRESOLVED_COMPANY_EXIT) {
4298
- if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
5436
+ const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
5437
+ const eventsByRoute = new Map();
5438
+ for (const event of laneEvents) {
5439
+ const route = routeChangeToTarget(event.relativePath);
5440
+ // The receiver is an independent ingress from the local watcher.
5441
+ // Apply the same route-selection boundary before it can enqueue a
5442
+ // fast pull: a --skip-personal runner must acknowledge, not pull,
5443
+ // a personal-namespace delivery.
5444
+ if (!route || !isWatchRouteSelected(route, watchRouteSelection))
5445
+ continue;
5446
+ const key = routeKey(route);
5447
+ const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
5448
+ group.events.push(event);
5449
+ if (event.kind === "upsert")
5450
+ group.upsertPaths.push(event.relativePath);
5451
+ eventsByRoute.set(key, group);
5452
+ }
5453
+ for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
5454
+ const paths = routeEvents.map((event) => event.relativePath);
5455
+ const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
5456
+ const exitCode = passExitCode(result);
5457
+ if (exitCode === UNRESOLVED_COMPANY_EXIT) {
5458
+ if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
5459
+ throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
5460
+ }
5461
+ warnUnresolvedReceiverCompany(route, routeEvents.length);
5462
+ await settleEvents?.(routeEvents);
5463
+ continue;
5464
+ }
5465
+ if (exitCode !== 0) {
4299
5466
  throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4300
5467
  }
4301
- warnUnresolvedReceiverCompany(route, routeEvents.length);
4302
5468
  await settleEvents?.(routeEvents);
4303
- continue;
4304
5469
  }
4305
- if (exitCode !== 0) {
4306
- throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4307
- }
4308
- await settleEvents?.(routeEvents);
4309
- }
4310
- };
4311
- // Dispatch realtime work before awaiting its bulk residue. This lets an
4312
- // established slow pass continue while the small scoped pull uses its
4313
- // ref-counted lock carve-out; the residue remains serialized and cannot
4314
- // starve the slow lane.
4315
- await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
4316
- await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
5470
+ };
5471
+ // Dispatch realtime work before awaiting its bulk residue. This lets an
5472
+ // established slow pass continue while the small scoped pull uses its
5473
+ // ref-counted lock carve-out; the residue remains serialized and cannot
5474
+ // starve the slow lane.
5475
+ await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
5476
+ await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
5477
+ }
5478
+ finally {
5479
+ pendingReceivers = Math.max(0, pendingReceivers - events.length);
5480
+ emitSyncWorkState();
5481
+ }
4317
5482
  };
4318
5483
  receiverSyncBatchFn.partition = (events, maxPaths) => {
4319
5484
  const lanes = partitionByKnownSizes(events, receiverEventSize);
@@ -4470,6 +5635,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4470
5635
  if (stopped)
4471
5636
  return shutdownPromise ?? Promise.resolve();
4472
5637
  stopped = true;
5638
+ clearInterval(watcherPushStagesTimer);
5639
+ if (durableWatcherPersistTimer !== undefined)
5640
+ clearTimeout(durableWatcherPersistTimer);
5641
+ durableWatcherPersistTimer = undefined;
5642
+ if (syncWorkStateTimer !== undefined)
5643
+ clearTimeout(syncWorkStateTimer);
5644
+ syncWorkStateTimer = undefined;
5645
+ emitSyncWorkState(true);
4473
5646
  for (const route of routePushRetryTimerHandles.keys()) {
4474
5647
  clearRoutePushRetry(route);
4475
5648
  }
@@ -4499,6 +5672,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4499
5672
  coordinatorDisposal,
4500
5673
  boundedEventSyncStartup,
4501
5674
  eventSyncDisposal,
5675
+ persistDurableWatcherPaths(),
4502
5676
  ]).then(() => undefined);
4503
5677
  try {
4504
5678
  void receiver?.dispose();
@@ -4916,6 +6090,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4916
6090
  lastCompletedFullReconcileWallAt = Date.now();
4917
6091
  fullPassJournalHealthy = execution.accepted &&
4918
6092
  isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
6093
+ if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
6094
+ clearDurableWatcherOverflow();
6095
+ }
6096
+ else if (fullPassJournalHealthy && pending.batch !== null) {
6097
+ acknowledgeDurableWatcherPathsCoveredByFullPass([
6098
+ ...pending.batch.paths.values(),
6099
+ ]);
6100
+ }
4919
6101
  if (fullPassJournalHealthy) {
4920
6102
  fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
4921
6103
  }
@@ -4973,7 +6155,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4973
6155
  const catchupBatch = workingBatch;
4974
6156
  const catchupGeneration = pending.generation;
4975
6157
  void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
4976
- if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
6158
+ if (exit === 0) {
6159
+ clearDurableWatcherOverflow();
6160
+ return;
6161
+ }
6162
+ if (exit === SCOPED_DRAIN_DEFERRED)
4977
6163
  return;
4978
6164
  restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
4979
6165
  interruptWait();
@@ -5081,6 +6267,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5081
6267
  ? exitCode === 0
5082
6268
  : hasCompletedRunnerPass(result));
5083
6269
  if (completedPostRepairPass) {
6270
+ if (!parallelCoordinatorEnabled &&
6271
+ pending.batch?.overflowed === true &&
6272
+ ((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
6273
+ clearDurableWatcherOverflow();
6274
+ }
6275
+ else if (!parallelCoordinatorEnabled &&
6276
+ ranFullReconcile &&
6277
+ fullPassJournalHealthy &&
6278
+ pending.batch !== null) {
6279
+ acknowledgeDurableWatcherPathsCoveredByFullPass([
6280
+ ...pending.batch.paths.values(),
6281
+ ]);
6282
+ }
5084
6283
  // The production adapter returns RunnerPassResult only after fanout,
5085
6284
  // which follows journal repair. Its numeric seam has the same
5086
6285
  // contract: zero means the injected pass completed. This preserves