@indigoai-us/hq-cloud 6.18.49-alpha.7 → 6.18.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/bin/sync-runner-company.d.ts +3 -5
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +13 -22
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-watch-loop.d.ts +2 -12
  6. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.js +112 -675
  8. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  9. package/dist/bin/sync-runner.d.ts +5 -27
  10. package/dist/bin/sync-runner.d.ts.map +1 -1
  11. package/dist/bin/sync-runner.js +15 -133
  12. package/dist/bin/sync-runner.js.map +1 -1
  13. package/dist/cli/rescue-core.d.ts +6 -1
  14. package/dist/cli/rescue-core.d.ts.map +1 -1
  15. package/dist/cli/rescue-core.js +217 -31
  16. package/dist/cli/rescue-core.js.map +1 -1
  17. package/dist/cli/share.d.ts +0 -12
  18. package/dist/cli/share.d.ts.map +1 -1
  19. package/dist/cli/share.js +4 -57
  20. package/dist/cli/share.js.map +1 -1
  21. package/dist/lib/cloud-authoritative.d.ts +2 -9
  22. package/dist/lib/cloud-authoritative.d.ts.map +1 -1
  23. package/dist/lib/cloud-authoritative.js +2 -34
  24. package/dist/lib/cloud-authoritative.js.map +1 -1
  25. package/dist/outposts/session-host-command.d.ts +13 -0
  26. package/dist/outposts/session-host-command.d.ts.map +1 -1
  27. package/dist/outposts/session-host-command.js +34 -17
  28. package/dist/outposts/session-host-command.js.map +1 -1
  29. package/dist/outposts/session-host-connection.d.ts +2 -0
  30. package/dist/outposts/session-host-connection.d.ts.map +1 -1
  31. package/dist/outposts/session-host-connection.js +15 -0
  32. package/dist/outposts/session-host-connection.js.map +1 -1
  33. package/dist/outposts/session-host-server.d.ts +4 -0
  34. package/dist/outposts/session-host-server.d.ts.map +1 -1
  35. package/dist/outposts/session-host-server.js +42 -0
  36. package/dist/outposts/session-host-server.js.map +1 -1
  37. package/dist/session-log-capture.d.ts.map +1 -1
  38. package/dist/session-log-capture.js +140 -0
  39. package/dist/session-log-capture.js.map +1 -1
  40. package/dist/symlink-target-policy.d.ts +0 -11
  41. package/dist/symlink-target-policy.d.ts.map +1 -1
  42. package/dist/symlink-target-policy.js +0 -80
  43. package/dist/symlink-target-policy.js.map +1 -1
  44. package/dist/sync/event-sync.d.ts.map +1 -1
  45. package/dist/sync/event-sync.js +12 -0
  46. package/dist/sync/event-sync.js.map +1 -1
  47. package/dist/sync/parallel-sync-coordinator.d.ts +0 -4
  48. package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
  49. package/dist/sync/parallel-sync-coordinator.js +14 -155
  50. package/dist/sync/parallel-sync-coordinator.js.map +1 -1
  51. package/dist/sync/push-receiver.d.ts +20 -6
  52. package/dist/sync/push-receiver.d.ts.map +1 -1
  53. package/dist/sync/push-receiver.js +160 -34
  54. package/dist/sync/push-receiver.js.map +1 -1
  55. package/dist/watcher.d.ts +3 -3
  56. package/dist/watcher.d.ts.map +1 -1
  57. package/dist/watcher.js +7 -15
  58. package/dist/watcher.js.map +1 -1
  59. package/package.json +2 -1
  60. package/dist/bin/watcher-catchup-skip-store.d.ts +0 -59
  61. package/dist/bin/watcher-catchup-skip-store.d.ts.map +0 -1
  62. package/dist/bin/watcher-catchup-skip-store.js +0 -278
  63. package/dist/bin/watcher-catchup-skip-store.js.map +0 -1
@@ -1,7 +1,6 @@
1
1
  import * as os from "os";
2
2
  import * as path from "path";
3
3
  import * as fs from "fs";
4
- import { createHash } from "node:crypto";
5
4
  import { MULTIPART_PART_SIZE_BYTES } from "../sync/multipart-uploader.js";
6
5
  import * as v8 from "node:v8";
7
6
  import { performance } from "node:perf_hooks";
@@ -16,17 +15,13 @@ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
16
15
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
17
16
  import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
18
17
  import { localPathForVaultKey } from "../local-path-codec.js";
19
- import { isEphemeralPath } from "../cli/share.js";
20
- import { cloudAuthoritativePathClass, isLocallyIneligibleForPush, } from "../lib/cloud-authoritative.js";
21
- import { inspectLocalSymlinkScope } from "../symlink-target-policy.js";
22
18
  import { shouldRecordLocalDeleteIntent } from "../local-replacement.js";
23
19
  import { createIgnoreFilter } from "../ignore.js";
24
- import { wrapFilterWithPersonalVaultDefaults } from "../personal-vault-exclusions.js";
25
20
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
26
21
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
27
22
  import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, isCorruptJournalJson, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, recoverCachedAreaJournalMirrorIfNeeded, setAreaLedgerRecoveryHqRootForProcess, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, readJournalScopedAsync, applyJournalDeltaAsync, } from "../journal.js";
28
- import { isPersonalVaultDeletionExcluded, readCloudBackedSlugsFromManifest, } from "../personal-vault.js";
29
- import { NoopPushReceiver, ReceiverDispatchError, RealtimePushYieldError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
23
+ import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
24
+ import { NoopPushReceiver, ReceiverDispatchError, isReceiverTransportFailure, } from "../sync/push-receiver.js";
30
25
  import { CONTENT_HASH_PATTERN } from "../sync/push-event.js";
31
26
  import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
32
27
  import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
@@ -268,35 +263,12 @@ export class CooperativePassScheduler {
268
263
  get hasActiveSlowRealtime() {
269
264
  return this.active?.workClass === "realtime" && this.active.lane === "slow";
270
265
  }
271
- get isInCapYield() {
272
- return this.capYieldDepth > 0;
273
- }
274
266
  /** Includes the running slow/fast lanes and hand-off occupants; it never changes their cap. */
275
267
  get activeRealtimeWorkerCount() {
276
268
  return ((this.active?.workClass === "realtime" ? 1 : 0) +
277
269
  (this.activeFastRealtime === null ? 0 : 1) +
278
270
  this.realtimeOccupants.size);
279
271
  }
280
- get diagnosticAdmissionSnapshot() {
281
- const active = [
282
- this.active,
283
- this.activeFastRealtime,
284
- ...this.realtimeOccupants,
285
- this.detachedChore,
286
- ].filter((entry) => entry !== null);
287
- const holders = active.map((entry) => ({
288
- kind: entry.transferDirection === "pull"
289
- ? "pull"
290
- : entry.workClass === "chore"
291
- ? "chore"
292
- : "push",
293
- scope: entry.label.slice(0, 160),
294
- }));
295
- return {
296
- queueDepth: this.pending.filter((entry) => entry.workClass === "realtime").length,
297
- holders: holders.slice(0, 20),
298
- };
299
- }
300
272
  pendingLaneCount(lane) {
301
273
  return this.pending.filter((entry) => entry.lane === lane).length;
302
274
  }
@@ -1320,53 +1292,6 @@ export function isChoreLocalPush(paths, largeUpload) {
1320
1292
  }
1321
1293
  return count > 0;
1322
1294
  }
1323
- /** Keep warm-up route telemetry useful on large checkouts without unbounded logs. */
1324
- export function watcherWarmupPathsByRoute(relativePaths) {
1325
- const counts = new Map();
1326
- for (const relativePath of relativePaths) {
1327
- const route = routeChangeToTarget(relativePath);
1328
- const key = route ? routeKey(route) : "unrouted";
1329
- counts.set(key, (counts.get(key) ?? 0) + 1);
1330
- }
1331
- const ranked = [...counts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1332
- const result = {};
1333
- for (const [key, count] of ranked.slice(0, 20))
1334
- result[key] = count;
1335
- const other = ranked.slice(20).reduce((sum, [, count]) => sum + count, 0);
1336
- if (other > 0)
1337
- result.other = other;
1338
- return result;
1339
- }
1340
- export function watcherWarmupPrefixesByRoute(relativePaths) {
1341
- const counts = new Map();
1342
- for (const relativePath of relativePaths) {
1343
- const route = routeChangeToTarget(relativePath);
1344
- const key = route ? routeKey(route) : "unrouted";
1345
- const routePath = route?.kind === "company"
1346
- ? relativePath.slice(`companies/${route.slug}/`.length)
1347
- : relativePath;
1348
- const prefix = routePath.split("/").slice(0, 2).join("/") || "(root)";
1349
- const routeCounts = counts.get(key) ?? new Map();
1350
- routeCounts.set(prefix, (routeCounts.get(prefix) ?? 0) + 1);
1351
- counts.set(key, routeCounts);
1352
- }
1353
- const result = {};
1354
- for (const [route, routeCounts] of [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)).slice(0, 20)) {
1355
- const ranked = [...routeCounts.entries()].sort(([leftKey, leftCount], [rightKey, rightCount]) => rightCount - leftCount || leftKey.localeCompare(rightKey));
1356
- const top = {};
1357
- for (const [prefix, count] of ranked.slice(0, 5))
1358
- top[prefix] = count;
1359
- const other = ranked.slice(5).reduce((sum, [, count]) => sum + count, 0);
1360
- if (other > 0)
1361
- top.other = other;
1362
- result[route] = top;
1363
- }
1364
- const otherRoutes = [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)).slice(20);
1365
- if (otherRoutes.length > 0) {
1366
- result.other = { other: otherRoutes.reduce((sum, [, routeCounts]) => sum + [...routeCounts.values()].reduce((routeSum, count) => routeSum + count, 0), 0) };
1367
- }
1368
- return result;
1369
- }
1370
1295
  /** A receiver event needs a known small upsert size before it can go realtime. */
1371
1296
  export function isFastReceiverBatch(eventCount) {
1372
1297
  return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
@@ -1620,9 +1545,14 @@ function unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)
1620
1545
  receiveCounts.every((count) => count !== undefined && Number.isFinite(count) && count >= 2);
1621
1546
  });
1622
1547
  }
1623
- function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
1548
+ export function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode) {
1624
1549
  const exitCode = passExitCode(outcome);
1625
1550
  const result = typeof outcome === "number" ? undefined : outcome.result;
1551
+ const httpStatus = [
1552
+ ...(result?.errors ?? []),
1553
+ ...(result?.transient ?? []),
1554
+ ...(result?.companyFailures ?? []),
1555
+ ].find((failure) => failure.httpStatus !== undefined)?.httpStatus;
1626
1556
  const diagnostics = [
1627
1557
  ...(result?.errors ?? []).map(({ company, message }) => `${company}: ${message}`),
1628
1558
  ...(result?.transient ?? []).map(({ company, message }) => `${company}: ${message}`),
@@ -1630,11 +1560,15 @@ function targetedReceiverDispatchError(outcome, accepted, authRequiredExitCode)
1630
1560
  ];
1631
1561
  const failureReason = `targeted pull failed with exit code ${exitCode}, accepted=${accepted}` +
1632
1562
  (diagnostics.length === 0 ? "" : `: ${[...new Set(diagnostics)].join("; ")}`);
1633
- const isTransportFailure = exitCode === authRequiredExitCode ||
1634
- exitCode === TRANSIENT_NETWORK_EXIT ||
1635
- (result?.transient?.length ?? 0) > 0 ||
1636
- isReceiverTransportFailure(failureReason);
1637
- return new ReceiverDispatchError(isTransportFailure ? "transport" : "path-specific", failureReason);
1563
+ const exitKind = exitCode === authRequiredExitCode
1564
+ ? "auth_required"
1565
+ : exitCode === TRANSIENT_NETWORK_EXIT || (result?.transient?.length ?? 0) > 0
1566
+ ? "transient_network"
1567
+ : undefined;
1568
+ const errorCode = ["ENOSPC", "EDQUOT", "EACCES", "EPERM"].find((code) => diagnostics.some((diagnostic) => new RegExp(`\\b${code}\\b`).test(diagnostic)));
1569
+ const transportTextMatch = exitKind === undefined && isReceiverTransportFailure(failureReason);
1570
+ const isTransportFailure = exitKind !== undefined || transportTextMatch;
1571
+ return new ReceiverDispatchError(isTransportFailure ? "transport" : "path-specific", failureReason, { exitCode, errorCode, exitKind, transportTextMatch, httpStatus, rawExit: typeof outcome === "number" });
1638
1572
  }
1639
1573
  /**
1640
1574
  * Exit code for one scoped/catch-up route. A 404 company route returns
@@ -1829,10 +1763,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1829
1763
  let recycleRequested = null;
1830
1764
  const pendingAppliedUpserts = new Map();
1831
1765
  const pendingAppliedDeletes = new Set();
1832
- const uploadedAwaitingPublish = new Set();
1833
- const UPLOADED_AWAITING_PUBLISH_BATCH_SIZE = 1_000;
1834
- const UPLOADED_AWAITING_PUBLISH_CAP = 10_000;
1835
- let uploadedAwaitingPublishDropLogged = false;
1836
1766
  const rememberAppliedUpsert = (relativePath, contentHash) => {
1837
1767
  pendingAppliedDeletes.delete(relativePath);
1838
1768
  if (eventSyncLive && eventSync) {
@@ -1863,23 +1793,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1863
1793
  pendingAppliedDeletes.delete(oldest);
1864
1794
  }
1865
1795
  };
1866
- const flushPendingUploadEvents = () => {
1867
- if (!eventSyncLive || !eventSync || uploadedAwaitingPublish.size === 0)
1868
- return;
1869
- while (uploadedAwaitingPublish.size > 0) {
1870
- const uploadedPaths = [...uploadedAwaitingPublish].slice(0, UPLOADED_AWAITING_PUBLISH_BATCH_SIZE);
1871
- const paths = new Map(uploadedPaths.map((relativePath) => [
1872
- localPathForVaultKey(hqRoot, relativePath),
1873
- relativePath,
1874
- ]));
1875
- const expanded = expandDirectoryPublishPaths(paths, uploadedPaths);
1876
- if (expanded.size > 0)
1877
- eventSync.publishBatch({ paths: expanded });
1878
- for (const relativePath of uploadedPaths) {
1879
- uploadedAwaitingPublish.delete(relativePath);
1880
- }
1881
- }
1882
- };
1883
1796
  const flushPendingAppliedUpserts = () => {
1884
1797
  if (!eventSyncLive || !eventSync)
1885
1798
  return;
@@ -1891,7 +1804,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1891
1804
  eventSync.markContentDeleted?.(relativePath);
1892
1805
  }
1893
1806
  pendingAppliedDeletes.clear();
1894
- flushPendingUploadEvents();
1895
1807
  };
1896
1808
  // `activeSlowPass` is assigned before its task has necessarily acquired the
1897
1809
  // cross-process lock (it may be waiting behind a rescue or a second runner).
@@ -1942,7 +1854,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1942
1854
  refresh.unref?.();
1943
1855
  }
1944
1856
  const passScheduler = new CooperativePassScheduler(monotonicNow, CHORE_STARVATION_CAP_MS, { caps: fullPassCaps });
1945
- const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal, watcherPushStage) => {
1857
+ const runPassWithLock = async (passArgvForRun, prepare, lane = "slow", workClass = "chore", checkpoint, telemetry, pullSignal, receiverSignal) => {
1946
1858
  const passCheckpoint = pullSignal || receiverSignal
1947
1859
  ? Object.assign(async () => checkpoint?.(), {
1948
1860
  shouldYieldToRealtime: checkpoint?.shouldYieldToRealtime,
@@ -1972,43 +1884,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1972
1884
  }
1973
1885
  };
1974
1886
  const invokePass = () => {
1975
- watcherPushStage?.("pass invocation");
1976
- activePassCount += 1;
1977
- emitSyncWorkState(true);
1978
- const trackPass = (pass) => {
1979
- const complete = () => {
1980
- watcherPushStage?.("pass return");
1981
- activePassCount = Math.max(0, activePassCount - 1);
1982
- emitSyncWorkState(true);
1983
- };
1984
- // Observe settlement without returning a chained promise to the pass
1985
- // scheduler. The scheduler's synchronous test seam and watcher wake
1986
- // ordering depend on the original pass promise's completion turn.
1987
- void pass.then(complete, complete);
1988
- return pass;
1989
- };
1990
- try {
1991
- if (!runPassSupportsCooperativeCheckpoint) {
1992
- if (!runPassSupportsQmdLifecycle)
1993
- return trackPass(runPass(passArgvForRun));
1994
- return trackPass(runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs));
1995
- }
1996
- // This callback is emitted by the production runner after journal
1997
- // repair/seed and before fanout. It starts ingress early, but never
1998
- // admits a second durable writer: queued realtime still waits for the
1999
- // existing scheduler's cooperative hand-off checkpoint.
2000
- const onJournalsReady = () => activateEventPushSurfaces();
2001
- if (!runPassSupportsQmdLifecycle) {
2002
- return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass));
2003
- }
2004
- return trackPass(runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes));
1887
+ if (!runPassSupportsCooperativeCheckpoint) {
1888
+ if (!runPassSupportsQmdLifecycle)
1889
+ return runPass(passArgvForRun);
1890
+ return runPass(passArgvForRun, undefined, undefined, undefined, undefined, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs);
2005
1891
  }
2006
- catch (error) {
2007
- watcherPushStage?.("pass return");
2008
- activePassCount = Math.max(0, activePassCount - 1);
2009
- emitSyncWorkState(true);
2010
- throw error;
1892
+ // This callback is emitted by the production runner after journal
1893
+ // repair/seed and before fanout. It starts ingress early, but never
1894
+ // admits a second durable writer: queued realtime still waits for the
1895
+ // existing scheduler's cooperative hand-off checkpoint.
1896
+ const onJournalsReady = () => activateEventPushSurfaces();
1897
+ if (!runPassSupportsQmdLifecycle) {
1898
+ return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass);
2011
1899
  }
1900
+ return runPass(passArgvForRun, passCheckpoint, onJournalsReady, recordDurableAppliedUpserts, workClass, qmdShutdownController.signal, writeMaintenanceProgress, idleHeartbeatIntervalMs, recordDurableAppliedDeletes);
2012
1901
  };
2013
1902
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
2014
1903
  // admitting a pass and immediately after it returns. A pass that runs a long
@@ -2023,11 +1912,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2023
1912
  return;
2024
1913
  recordDurableAppliedUpserts(outcome.result?.appliedUpserts ?? []);
2025
1914
  recordDurableAppliedDeletes(outcome.result?.appliedDeletes ?? []);
2026
- rememberUploadedPaths(outcome);
2027
- if (typeof outcome !== "number") {
2028
- acknowledgeDurableWatcherUploads(outcome.result?.uploadedPaths ?? []);
2029
- }
2030
- flushPendingUploadEvents();
2031
1915
  };
2032
1916
  // Pre-admit boundary observation.
2033
1917
  heapCheckpoint?.();
@@ -2278,25 +2162,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2278
2162
  // watcher fast path explicitly opts into its established scoped exception.
2279
2163
  return passScheduler.submit(task, workClass, lane, allowFastRealtimeOvertake, signal, resourceCap, allowFlaggedRealtimeHandoff, transferDirection, transferSizeClass, holderLabel);
2280
2164
  };
2281
- const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass, watcherPushStage) => {
2282
- if (watcherPushStage) {
2283
- watcherPushStage("guarded scheduler admission requested");
2284
- const admission = passScheduler.diagnosticAdmissionSnapshot;
2285
- if (passScheduler.isInCapYield)
2286
- watcherPushStage("CPU/heap cap wait");
2287
- else if (admission.holders.some((holder) => holder.kind === "pull")) {
2288
- watcherPushStage("pull-yield wait");
2289
- }
2290
- }
2165
+ const runGuarded = (passArgvForRun, prepare, lane = "slow", workClass = "chore", allowFastRealtimeOvertake = false, signal, resourceCap = false, telemetry, allowConcurrentHandoff = false, receiverSignal, holderLabel = workClass === "chore" && resourceCap ? "full-reconcile" : workClass) => {
2291
2166
  telemetry?.mark("operationGate");
2292
2167
  const admissionSignal = signal && receiverSignal
2293
2168
  ? AbortSignal.any([signal, receiverSignal])
2294
2169
  : signal ?? receiverSignal;
2295
2170
  return runGuardedTask((context) => {
2296
- watcherPushStage?.("guarded scheduler admission granted");
2297
2171
  telemetry?.mark("appendLock");
2298
2172
  telemetry?.setOccupancy(passScheduler.activeRealtimeWorkerCount);
2299
- return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal, watcherPushStage);
2173
+ return runPassWithLock(passArgvForRun, prepare, lane, workClass, workClass === "chore" ? context.checkpoint : undefined, telemetry, context.signal, receiverSignal);
2300
2174
  }, lane, workClass, allowFastRealtimeOvertake, admissionSignal, resourceCap, allowConcurrentHandoff, telemetry?.direction, telemetry?.sizeClass, holderLabel);
2301
2175
  };
2302
2176
  /*
@@ -2418,7 +2292,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2418
2292
  return {
2419
2293
  result: execution.result ?? (allRefused ? 0 : 1),
2420
2294
  accepted,
2421
- yieldedToPush: outcome.paths.some((path) => path.reason === "yielded to queued watcher push"),
2422
2295
  coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
2423
2296
  .filter((path) => path.status === "refused")
2424
2297
  .map((path) => path.path)),
@@ -2616,172 +2489,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2616
2489
  /** Vanished never-journaled file deletes removed from batches at drain time. */
2617
2490
  let drainedUntrackedDeletes = 0;
2618
2491
  const pendingWatcherPaths = new Map();
2619
- const durableWatcherPaths = new Map();
2620
- const durableWatcherPathLimit = 10_000;
2621
- let durableWatcherOverflowed = false;
2622
- let durableWatcherPersistTimer;
2623
- let durableWatcherPersistQueue = Promise.resolve();
2624
- let durableWatcherPersistGeneration = 0;
2625
- let durableWatcherPersistedGeneration = 0;
2626
- const watcherPushStatePath = path.join(getStateDir(), "sync-runner", `watcher-push-${createHash("sha256").update(hqRoot).digest("hex").slice(0, 20)}.json`);
2627
- const persistDurableWatcherPaths = () => {
2628
- const generation = durableWatcherPersistGeneration;
2629
- if (generation === durableWatcherPersistedGeneration)
2630
- return durableWatcherPersistQueue;
2631
- const snapshot = durableWatcherOverflowed
2632
- ? { format: 1, hqRoot, overflowed: true }
2633
- : { format: 1, hqRoot, paths: [...durableWatcherPaths.values()] };
2634
- durableWatcherPersistQueue = durableWatcherPersistQueue.then(async () => {
2635
- try {
2636
- if (!durableWatcherOverflowed && durableWatcherPaths.size === 0) {
2637
- await fs.promises.rm(watcherPushStatePath, { force: true });
2638
- durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
2639
- return;
2640
- }
2641
- await fs.promises.mkdir(path.dirname(watcherPushStatePath), { recursive: true, mode: 0o700 });
2642
- const tempPath = `${watcherPushStatePath}.${process.pid}.tmp`;
2643
- await fs.promises.writeFile(tempPath, JSON.stringify(snapshot), { mode: 0o600 });
2644
- await fs.promises.rename(tempPath, watcherPushStatePath);
2645
- durableWatcherPersistedGeneration = Math.max(durableWatcherPersistedGeneration, generation);
2646
- }
2647
- catch (err) {
2648
- process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
2649
- }
2650
- }).catch((err) => {
2651
- process.stderr.write(`hq-sync-runner: watcher push state persistence failed: ${describeError(err)}\n`);
2652
- });
2653
- return durableWatcherPersistQueue;
2654
- };
2655
- const scheduleDurableWatcherPersist = () => {
2656
- durableWatcherPersistGeneration += 1;
2657
- if (durableWatcherPersistTimer !== undefined)
2658
- clearTimeout(durableWatcherPersistTimer);
2659
- durableWatcherPersistTimer = setTimeout(() => {
2660
- durableWatcherPersistTimer = undefined;
2661
- void persistDurableWatcherPaths();
2662
- }, 1_000);
2663
- durableWatcherPersistTimer.unref?.();
2664
- };
2665
- const addDurableWatcherPath = (absolutePath, relativePath) => {
2666
- if (durableWatcherOverflowed)
2667
- return;
2668
- if (!durableWatcherPaths.has(absolutePath) && durableWatcherPaths.size >= durableWatcherPathLimit) {
2669
- durableWatcherPaths.clear();
2670
- durableWatcherOverflowed = true;
2671
- }
2672
- else if (!durableWatcherOverflowed) {
2673
- durableWatcherPaths.set(absolutePath, relativePath);
2674
- }
2675
- scheduleDurableWatcherPersist();
2676
- };
2677
- const clearDurableWatcherOverflow = () => {
2678
- if (!durableWatcherOverflowed)
2679
- return;
2680
- durableWatcherOverflowed = false;
2681
- durableWatcherPaths.clear();
2682
- for (const [absolutePath, relativePath] of pendingWatcherPaths) {
2683
- addDurableWatcherPath(absolutePath, relativePath);
2684
- }
2685
- scheduleDurableWatcherPersist();
2686
- };
2687
- const acknowledgeDurableWatcherUploads = (relativePaths) => {
2688
- let changed = false;
2689
- const uploaded = new Set(relativePaths);
2690
- for (const [absolutePath, relativePath] of durableWatcherPaths) {
2691
- if (!uploaded.has(relativePath))
2692
- continue;
2693
- durableWatcherPaths.delete(absolutePath);
2694
- changed = true;
2695
- }
2696
- if (changed)
2697
- scheduleDurableWatcherPersist();
2698
- };
2699
- const acknowledgeDurableWatcherPathsCoveredByFullPass = (relativePaths) => {
2700
- const stillPending = new Set(pendingWatcherPaths.values());
2701
- acknowledgeDurableWatcherUploads(relativePaths.filter((relativePath) => !stillPending.has(relativePath)));
2702
- };
2703
- let activePassCount = 0;
2704
- let inFlightWatcherPushes = 0;
2705
- const watcherPushStages = new Map();
2706
- let nextWatcherPushGroupId = 0;
2707
- const setWatcherPushStage = (groupId, stage) => {
2708
- const record = watcherPushStages.get(groupId);
2709
- if (!record || record.stage === stage)
2710
- return;
2711
- record.stage = stage;
2712
- record.stageEnteredAt = Date.now();
2713
- process.stderr.write(`${JSON.stringify({
2714
- type: "watcher-push-group-stage",
2715
- diagnostic: true,
2716
- groupId: record.groupId,
2717
- route: record.route,
2718
- paths: record.paths,
2719
- stage: record.stage,
2720
- stageEnteredAt: new Date(record.stageEnteredAt).toISOString(),
2721
- })}\n`);
2722
- };
2723
- const startWatcherPushStageTracking = (route, paths) => {
2724
- const groupId = `watcher-push-${++nextWatcherPushGroupId}`;
2725
- watcherPushStages.set(groupId, {
2726
- groupId,
2727
- route,
2728
- paths,
2729
- stage: "",
2730
- stageEnteredAt: Date.now(),
2731
- });
2732
- return { groupId, setStage: (stage) => setWatcherPushStage(groupId, stage) };
2733
- };
2734
- const emitWatcherPushInflightStages = () => {
2735
- if (inFlightWatcherPushes <= 0)
2736
- return;
2737
- const now = Date.now();
2738
- const entries = [...watcherPushStages.values()];
2739
- const admission = passScheduler.diagnosticAdmissionSnapshot;
2740
- process.stderr.write(`${JSON.stringify({
2741
- type: "watcher-push-inflight-stages",
2742
- diagnostic: true,
2743
- groups: entries.slice(0, 20).map((record) => ({
2744
- route: record.route,
2745
- paths: record.paths,
2746
- stage: record.stage,
2747
- ageMs: Math.max(0, now - record.stageEnteredAt),
2748
- })),
2749
- other: Math.max(0, entries.length - 20),
2750
- realtimeAdmissionQueueDepth: admission.queueDepth,
2751
- holders: admission.holders,
2752
- })}\n`);
2753
- };
2754
- const watcherPushStagesTimer = setInterval(emitWatcherPushInflightStages, Math.max(1, deps.watcherPushDiagnosticsIntervalMs ?? 60_000));
2755
- watcherPushStagesTimer.unref?.();
2756
- let pendingReceivers = 0;
2757
- let lastSyncWorkStateEmitAt = 0;
2758
- let syncWorkStateTimer;
2759
- const emitSyncWorkState = (force = false) => {
2760
- const now = Date.now();
2761
- const elapsed = now - lastSyncWorkStateEmitAt;
2762
- if (force || elapsed >= 250) {
2763
- if (syncWorkStateTimer !== undefined)
2764
- clearTimeout(syncWorkStateTimer);
2765
- syncWorkStateTimer = undefined;
2766
- lastSyncWorkStateEmitAt = now;
2767
- const snapshot = {
2768
- passActive: activePassCount > 0,
2769
- queuedWatcherPushes: pendingWatcherPaths.size,
2770
- inFlightWatcherPushes,
2771
- pendingReceivers,
2772
- };
2773
- deps.onSyncWorkStateSnapshot?.(snapshot);
2774
- stdout.write(`${JSON.stringify({ type: "sync-work-state", ...snapshot })}\n`);
2775
- return;
2776
- }
2777
- if (syncWorkStateTimer !== undefined)
2778
- return;
2779
- syncWorkStateTimer = setTimeout(() => {
2780
- syncWorkStateTimer = undefined;
2781
- emitSyncWorkState(true);
2782
- }, Math.max(1, 250 - elapsed));
2783
- syncWorkStateTimer.unref?.();
2784
- };
2785
2492
  const droppedUnresolvedWatcherSlugs = new Set();
2786
2493
  const logDroppedUnresolvedWatcherRoute = (slug) => {
2787
2494
  if (droppedUnresolvedWatcherSlugs.has(slug))
@@ -2849,11 +2556,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2849
2556
  }
2850
2557
  }
2851
2558
  if (batch) {
2852
- if (batch.overflowed || batch.paths.size > durableWatcherPathLimit) {
2853
- durableWatcherPaths.clear();
2854
- durableWatcherOverflowed = true;
2855
- scheduleDurableWatcherPersist();
2856
- }
2857
2559
  const entries = [...batch.paths.entries()].filter(([absolutePath]) => {
2858
2560
  const previousSequence = pendingWatcherChangeSequences.get(absolutePath);
2859
2561
  return previousSequence === undefined ||
@@ -2862,7 +2564,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2862
2564
  });
2863
2565
  if (entries.length === 0)
2864
2566
  return false;
2865
- const queueWasEmpty = pendingWatcherPaths.size === 0;
2866
2567
  pendingWatcherBatchGeneration ??= ++nextWatcherBatchGeneration;
2867
2568
  pendingWatcherOriginalBatch =
2868
2569
  pendingWatcherPaths.size === 0 &&
@@ -2874,13 +2575,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2874
2575
  : null;
2875
2576
  for (const [absolutePath, relativePath] of entries) {
2876
2577
  pendingWatcherPaths.set(absolutePath, relativePath);
2877
- addDurableWatcherPath(absolutePath, relativePath);
2878
2578
  const change = batch.changes?.get(absolutePath);
2879
2579
  if (change)
2880
2580
  pendingWatcherChanges.set(absolutePath, change);
2881
2581
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2882
2582
  }
2883
- emitSyncWorkState(queueWasEmpty);
2884
2583
  if (batch.overflowed) {
2885
2584
  pendingWatcherOverflowed = true;
2886
2585
  pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
@@ -2904,20 +2603,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2904
2603
  // counterpart must be re-derived through the codec — a raw join would
2905
2604
  // point at the impossible colon path on Windows.
2906
2605
  const absolutePath = localPathForVaultKey(hqRoot, changedRelPath);
2907
- const queueWasEmpty = pendingWatcherPaths.size === 0;
2908
2606
  pendingWatcherPaths.set(absolutePath, changedRelPath);
2909
- addDurableWatcherPath(absolutePath, changedRelPath);
2910
2607
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2911
2608
  // A bare path signal carries no delete intent; drop any change record
2912
2609
  // a prior batch left for this path so the stale intent cannot leak
2913
2610
  // into the coalesced batch (#227).
2914
2611
  pendingWatcherChanges.delete(absolutePath);
2915
- emitSyncWorkState(queueWasEmpty);
2916
2612
  return true;
2917
2613
  }
2918
2614
  pendingWatcherOriginalBatch = null;
2919
2615
  pendingWatcherBareChange = true;
2920
- emitSyncWorkState();
2921
2616
  return true;
2922
2617
  };
2923
2618
  const takePendingWatcherChange = () => {
@@ -2962,7 +2657,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2962
2657
  pendingWatcherDroppedRouteHints.clear();
2963
2658
  pendingWatcherDroppedRoutesUnknown = false;
2964
2659
  pendingWatcherBatchGeneration = null;
2965
- emitSyncWorkState();
2966
2660
  return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
2967
2661
  };
2968
2662
  const restorePendingWatcherPaths = (paths, generation, changes) => {
@@ -2976,54 +2670,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2976
2670
  if (pendingWatcherChangeSequences.has(absolutePath))
2977
2671
  continue;
2978
2672
  pendingWatcherPaths.set(absolutePath, relativePath);
2979
- addDurableWatcherPath(absolutePath, relativePath);
2980
2673
  pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
2981
2674
  const change = changes?.get(absolutePath);
2982
2675
  if (change)
2983
2676
  pendingWatcherChanges.set(absolutePath, change);
2984
2677
  }
2985
- emitSyncWorkState();
2986
2678
  };
2987
- if (eventPush) {
2988
- try {
2989
- const stored = JSON.parse(fs.readFileSync(watcherPushStatePath, "utf8"));
2990
- if (stored.format !== 1 || stored.hqRoot !== hqRoot ||
2991
- (!Array.isArray(stored.paths) && stored.overflowed !== true)) {
2992
- throw new Error("stored watcher push state has an unsupported shape or root");
2993
- }
2994
- if (stored.overflowed === true) {
2995
- durableWatcherOverflowed = true;
2996
- pendingWatcherOverflowed = true;
2997
- pendingWatcherWarmupCatchup = true;
2998
- pendingWatcherDroppedRoutesUnknown = true;
2999
- pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3000
- emitSyncWorkState(true);
3001
- process.stderr.write("hq-sync-runner: restored overflowed watcher push state; scheduling full warm-up catch-up\n");
3002
- }
3003
- for (const relativePath of Array.isArray(stored.paths) ? stored.paths : []) {
3004
- if (typeof relativePath !== "string" || relativePath.length === 0)
3005
- continue;
3006
- const route = routeChangeToTarget(relativePath);
3007
- if (!route || !isWatchRouteSelected(route, watchRouteSelection))
3008
- continue;
3009
- const absolutePath = localPathForVaultKey(hqRoot, relativePath);
3010
- addDurableWatcherPath(absolutePath, relativePath);
3011
- pendingWatcherPaths.set(absolutePath, relativePath);
3012
- pendingWatcherChangeSequences.set(absolutePath, ++nextPendingWatcherChangeSequence);
3013
- }
3014
- if (pendingWatcherPaths.size > 0) {
3015
- pendingWatcherBatchGeneration = ++nextWatcherBatchGeneration;
3016
- pendingWatcherWarmupCatchup = true;
3017
- emitSyncWorkState(true);
3018
- process.stderr.write(`hq-sync-runner: restored ${pendingWatcherPaths.size} watcher push path(s) from prior shutdown\n`);
3019
- }
3020
- }
3021
- catch (err) {
3022
- if (!(err instanceof Error && err.code === "ENOENT")) {
3023
- process.stderr.write(`hq-sync-runner: watcher push state recovery failed: ${describeError(err)}\n`);
3024
- }
3025
- }
3026
- }
3027
2679
  const routePushFailureBackoff = new RoutePushFailureBackoff();
3028
2680
  const pushRetryScheduler = deps.pushRetryScheduler ?? systemPushRetryScheduler;
3029
2681
  const routePushRetryTimerHandles = new Map();
@@ -3874,6 +3526,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3874
3526
  // a file and still be refused as stale (a download changed the directory
3875
3527
  // mid-pass); the retry then finds the file already in the cloud and uploads
3876
3528
  // nothing, so without this the file is never announced to peers.
3529
+ const uploadedAwaitingPublish = new Set();
3530
+ const UPLOADED_AWAITING_PUBLISH_CAP = 1_000;
3877
3531
  const rememberUploadedPaths = (result) => {
3878
3532
  if (result === undefined || typeof result === "number")
3879
3533
  return;
@@ -3886,14 +3540,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3886
3540
  const oldest = uploadedAwaitingPublish.values().next().value;
3887
3541
  if (oldest !== undefined)
3888
3542
  uploadedAwaitingPublish.delete(oldest);
3889
- if (!uploadedAwaitingPublishDropLogged) {
3890
- uploadedAwaitingPublishDropLogged = true;
3891
- process.stderr.write(`${JSON.stringify({
3892
- type: "event-sync-upload-buffer-overflow",
3893
- cap: UPLOADED_AWAITING_PUBLISH_CAP,
3894
- action: "drop-oldest",
3895
- })}\n`);
3896
- }
3897
3543
  }
3898
3544
  }
3899
3545
  };
@@ -4009,39 +3655,18 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4009
3655
  : "small";
4010
3656
  const largeUpload = sizeClass !== "small";
4011
3657
  const chorePush = isChoreLocalPush(group.relPaths, largeUpload);
4012
- const { groupId, setStage: watcherPushStage } = startWatcherPushStageTracking(routeId, group.relPaths.length);
4013
- watcherPushStage("coordinator submit");
4014
- inFlightWatcherPushes += group.relPaths.length;
4015
- emitSyncWorkState(true);
4016
- emitWatcherPushInflightStages();
4017
- let execution;
4018
- try {
4019
- execution = await (parallelCoordinatorEnabled && !chorePush
4020
- ? runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
4021
- watcherPushStage("worker entry");
4022
- const admittedPaths = slice.paths.map((path) => path.canonicalPath);
4023
- const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
4024
- const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
4025
- return runGuarded(scopedArgv, admittedChanges.size > 0
4026
- ? () => prepareWatcherChanges(admittedChanges)
4027
- : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced, undefined, routeId, watcherPushStage);
4028
- }, sizeClass)
4029
- : (() => {
4030
- watcherPushStage("worker entry");
4031
- return runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
4032
- ? () => prepareWatcherChanges(group.changes)
4033
- : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake, undefined, false, undefined, false, undefined, routeId, watcherPushStage).then((result) => ({
4034
- result,
4035
- accepted: true,
4036
- coordinatorRefusedPaths: [],
4037
- }));
4038
- })());
4039
- }
4040
- finally {
4041
- watcherPushStages.delete(groupId);
4042
- inFlightWatcherPushes = Math.max(0, inFlightWatcherPushes - group.relPaths.length);
4043
- emitSyncWorkState(true);
4044
- }
3658
+ const execution = parallelCoordinatorEnabled && !chorePush
3659
+ ? await runRealtimePass("watcher", "push", group.relPaths, lane, async (slice, telemetry, coordinatorFenced) => {
3660
+ const admittedPaths = slice.paths.map((path) => path.canonicalPath);
3661
+ const admittedChanges = changesForAdmittedPaths(group, admittedPaths);
3662
+ const scopedArgv = buildScopedPushArgv(group.route, admittedPaths, passArgv, group.deleteScopeRoots);
3663
+ return runGuarded(scopedArgv, admittedChanges.size > 0
3664
+ ? () => prepareWatcherChanges(admittedChanges)
3665
+ : undefined, lane, "realtime", allowFastRealtimeOvertake, slice.signal, false, telemetry, coordinatorFenced);
3666
+ }, sizeClass)
3667
+ : { result: await runGuarded(buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots), group.changes.size > 0
3668
+ ? () => prepareWatcherChanges(group.changes)
3669
+ : undefined, lane, chorePush ? "chore" : "realtime", allowFastRealtimeOvertake), accepted: true, coordinatorRefusedPaths: [] };
4045
3670
  const result = execution.result;
4046
3671
  rememberUploadedPaths(result);
4047
3672
  const pushExit = passExitCode(result);
@@ -4060,7 +3685,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4060
3685
  }
4061
3686
  const isRefusedPath = (relativePath) => [...refusedPaths].some((refusedPath) => relativePath === refusedPath ||
4062
3687
  relativePath.startsWith(`${refusedPath}/`));
4063
- acknowledgeDurableWatcherUploads(group.relPaths.filter((relativePath) => !isRefusedPath(relativePath)));
4064
3688
  const publishPaths = new Map();
4065
3689
  for (const [absolutePath, relativePath] of group.paths) {
4066
3690
  if (isRefusedPath(relativePath))
@@ -4100,11 +3724,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4100
3724
  deferPushAfterAdmissionTimeout(group, routeId);
4101
3725
  }
4102
3726
  else if (pushExit === UNRESOLVED_COMPANY_EXIT) {
4103
- acknowledgeDurableWatcherUploads(group.relPaths);
4104
3727
  routePushFailureBackoff.succeeded(routeId);
4105
3728
  clearRoutePushRetry(routeId);
4106
3729
  if (group.route.kind === "company") {
4107
- deps.watcherCatchupSkipStore?.recordUnresolvedCompany(group.route.slug);
4108
3730
  logDroppedUnresolvedWatcherRoute(group.route.slug);
4109
3731
  }
4110
3732
  }
@@ -4166,7 +3788,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4166
3788
  const execution = parallelCoordinatorEnabled
4167
3789
  ? await runRealtimePass("watcher", "pull", group.relPaths, lane, (slice, telemetry, coordinatorFenced) => {
4168
3790
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
4169
- return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced, undefined, routeKey(group.route));
3791
+ return runGuarded(buildTargetedPullArgv(group.route, passArgv, slicePaths, exactFileUpserts.filter((relativePath) => slicePaths.includes(relativePath))), undefined, lane, "realtime", false, slice.signal, false, telemetry, coordinatorFenced);
4170
3792
  })
4171
3793
  : await runGuarded(pullArgv, undefined, lane, "realtime").then((result) => ({ result, accepted: true }), (error) => {
4172
3794
  if (error instanceof RealtimeAdmissionTimeoutError) {
@@ -4218,7 +3840,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4218
3840
  addRoute(hint);
4219
3841
  const skipCompanies = resolveSkipCompanies();
4220
3842
  if (parallelCoordinatorEnabled) {
4221
- let routesRemaining = [...routes.values()].filter((route) => route.kind !== "company" || !skipCompanies.has(route.slug.toLowerCase())).length;
4222
3843
  for (const route of routes.values()) {
4223
3844
  if (route.kind === "company" &&
4224
3845
  skipCompanies.has(route.slug.toLowerCase())) {
@@ -4247,11 +3868,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4247
3868
  process.stderr.write(`hq-sync-runner: backlog chore for ${routeKey(route)} failed; will retry on cadence\n`);
4248
3869
  interruptWait();
4249
3870
  }
4250
- else {
4251
- routesRemaining -= 1;
4252
- if (routesRemaining === 0)
4253
- clearDurableWatcherOverflow();
4254
- }
4255
3871
  });
4256
3872
  }
4257
3873
  return 0;
@@ -4488,7 +4104,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4488
4104
  }
4489
4105
  }
4490
4106
  watcher.start();
4491
- let catchupPathStatWarningReported = false;
4492
4107
  const reportWatcherReady = () => {
4493
4108
  if (stopped || watcher === null)
4494
4109
  return;
@@ -4529,147 +4144,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4529
4144
  : watcher?.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, journalPaths);
4530
4145
  if (!catchup || catchup.cancelled || stopped || watcher === null)
4531
4146
  return;
4532
- const skipStore = deps.watcherCatchupSkipStore;
4533
- const cloudCompanySlugs = readCloudBackedSlugsFromManifest(hqRoot);
4534
- const eligiblePaths = new Map();
4535
- const eligibleChanges = new Map();
4536
- const excludedByReason = {};
4537
- const excludedByReasonByRoute = {};
4538
- const ignoreFilter = createIgnoreFilter(hqRoot);
4539
- const personalFilterExclusions = new Map();
4540
- const personalPushFilter = wrapFilterWithPersonalVaultDefaults(ignoreFilter, hqRoot, (relativePath, match) => personalFilterExclusions.set(relativePath, match.id));
4541
- const clearedResolvedCompanies = new Set();
4542
- const ontologyBackupPaths = { attioCleanup: 0, readModelRollout: 0 };
4543
- const backupClass = (relativePath) => {
4544
- if (/\/ontology\/_attio-cleanup-backup\//.test(`/${relativePath}`))
4545
- return "attioCleanup";
4546
- if (/\/ontology\/_read-model-rollout-backup\//.test(`/${relativePath}`))
4547
- return "readModelRollout";
4548
- return undefined;
4549
- };
4550
- const exclude = (reason, routeName) => {
4551
- excludedByReason[reason] = (excludedByReason[reason] ?? 0) + 1;
4552
- const routeReasons = excludedByReasonByRoute[routeName] ??= {};
4553
- routeReasons[reason] = (routeReasons[reason] ?? 0) + 1;
4554
- };
4555
- for (const [absolutePath, relativePath] of catchup.batch.paths) {
4556
- const route = routeChangeToTarget(relativePath);
4557
- const routeName = route ? routeKey(route) : "unrouted";
4558
- const ontologyBackupClass = backupClass(relativePath);
4559
- if (ontologyBackupClass) {
4560
- exclude("local-rule:ontology-backup", routeName);
4561
- ontologyBackupPaths[ontologyBackupClass] += 1;
4562
- continue;
4563
- }
4564
- if (route?.kind === "company") {
4565
- const resolvedNow = cloudCompanySlugs.has(route.slug) ||
4566
- skipStore?.hasCurrentGrantCompany(route.slug) ||
4567
- skipStore?.hasCurrentGrantVersion(route.slug) ||
4568
- (companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase());
4569
- if (resolvedNow && !clearedResolvedCompanies.has(route.slug)) {
4570
- skipStore?.clearUnresolvedCompany(route.slug);
4571
- clearedResolvedCompanies.add(route.slug);
4572
- }
4573
- if (!resolvedNow && skipStore?.hasFreshUnresolvedCompany(route.slug)) {
4574
- exclude("unresolved-company", routeName);
4575
- continue;
4576
- }
4577
- }
4578
- if (route?.kind === "company" &&
4579
- !cloudCompanySlugs.has(route.slug) &&
4580
- !skipStore?.hasCurrentGrantCompany(route.slug) &&
4581
- !skipStore?.hasCurrentGrantVersion(route.slug) &&
4582
- !(companyMode && companySlug?.trim().toLowerCase() === route.slug.trim().toLowerCase())) {
4583
- exclude("local-only-company", routeName);
4584
- continue;
4585
- }
4586
- const company = route?.kind === "company" ? route.slug : "personal";
4587
- const routePath = route?.kind === "company"
4588
- ? relativePath.slice(`companies/${route.slug}/`.length)
4589
- : relativePath;
4590
- if (isEphemeralPath(routePath)) {
4591
- exclude("push-filter:conflict-mirror", routeName);
4592
- continue;
4593
- }
4594
- const acceptedByPushFilter = route?.kind === "personal"
4595
- ? personalPushFilter(absolutePath, false)
4596
- : ignoreFilter(absolutePath, false);
4597
- if (!acceptedByPushFilter) {
4598
- const exclusionId = route?.kind === "personal"
4599
- ? personalFilterExclusions.get(relativePath)
4600
- : undefined;
4601
- exclude(exclusionId
4602
- ? `push-filter:personal:${exclusionId}`
4603
- : "push-filter:ignore", routeName);
4604
- continue;
4605
- }
4606
- if (route && isLocallyIneligibleForPush(routePath, {
4607
- boardClientOwned: true,
4608
- ontologyEntityPush: route.kind === "company" && skipStore?.allowsOntologyEntityPush(company),
4609
- })) {
4610
- const pathClass = cloudAuthoritativePathClass(routePath) ?? "other";
4611
- exclude(`local-rule:${pathClass}`, routeName);
4612
- continue;
4613
- }
4614
- try {
4615
- const stat = fs.lstatSync(absolutePath);
4616
- if (route && stat.isSymbolicLink()) {
4617
- const scopeRoot = route.kind === "company"
4618
- ? path.join(hqRoot, "companies", route.slug)
4619
- : relativePath.startsWith("personal/")
4620
- ? path.join(hqRoot, "personal")
4621
- : hqRoot;
4622
- const inspection = inspectLocalSymlinkScope({
4623
- absoluteLinkPath: absolutePath,
4624
- linkPath: relativePath,
4625
- scopeRoot,
4626
- scope: route.kind === "company"
4627
- ? { kind: "company", companySlug: route.slug }
4628
- : { kind: "personal" },
4629
- });
4630
- if (inspection.refused) {
4631
- exclude("symlink", routeName);
4632
- continue;
4633
- }
4634
- }
4635
- if (skipStore?.matches(company, routePath, stat)) {
4636
- exclude("record-match", routeName);
4637
- continue;
4638
- }
4639
- }
4640
- catch (error) {
4641
- // Fail open: unreadable state or a filesystem race must not suppress a candidate.
4642
- if (!catchupPathStatWarningReported) {
4643
- catchupPathStatWarningReported = true;
4644
- process.stderr.write(`hq-sync-runner: warm-up skip stat failed open: ${describeError(error)}\n`);
4645
- }
4646
- }
4647
- eligiblePaths.set(absolutePath, relativePath);
4648
- const change = catchup.batch.changes?.get(absolutePath);
4649
- if (change)
4650
- eligibleChanges.set(absolutePath, change);
4651
- }
4652
- const queued = addPendingWatcherChange(undefined, {
4653
- paths: eligiblePaths,
4654
- changes: eligibleChanges,
4655
- }, {
4147
+ const queued = addPendingWatcherChange(undefined, catchup.batch, {
4656
4148
  preserveChangesAfter: liveChangeSequenceAtWarmupStart,
4657
4149
  });
4658
4150
  pendingWatcherWarmupCatchup ||= queued;
4659
- const capEvicted = skipStore?.takeCapEvictedCount() ?? 0;
4660
4151
  process.stderr.write(JSON.stringify({
4661
4152
  type: "watcher-warmup-catchup",
4662
4153
  reason: "watch-set-ready",
4663
- pathsFound: eligiblePaths.size,
4664
- candidatePathsByRoute: watcherWarmupPathsByRoute(catchup.batch.paths.values()),
4665
- pathsByRoute: watcherWarmupPathsByRoute(eligiblePaths.values()),
4666
- pathPrefixesByRoute: watcherWarmupPrefixesByRoute(catchup.batch.paths.values()),
4667
- excludedByReason: {
4668
- ...excludedByReason,
4669
- ...(capEvicted > 0 ? { "cap-evicted": capEvicted } : {}),
4670
- },
4671
- excludedByReasonByRoute,
4672
- ontologyBackupPaths,
4154
+ pathsFound: catchup.batch.paths.size,
4673
4155
  scannedPaths: catchup.scannedPaths,
4674
4156
  elapsedMs: Date.now() - startedAt,
4675
4157
  queued,
@@ -4736,16 +4218,13 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4736
4218
  const execution = await runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => {
4737
4219
  const slicePaths = slice.paths.map((path) => path.canonicalPath);
4738
4220
  const exactFileUpserts = exactUpsertPathsForCoordinatorSlice(events, slicePaths);
4739
- return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal, routeKey(route)));
4221
+ return withScopeContentHashesFile(route, events, slicePaths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, slicePaths, exactFileUpserts, contentHashesFile), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced, receiverSignal));
4740
4222
  });
4741
4223
  const pathCount = events.length;
4742
4224
  // Rethrow the typed refusal so the receiver keeps the lease and
4743
4225
  // retries the whole batch, instead of bisecting a "failed" pull.
4744
4226
  if (execution.admissionTimeout)
4745
4227
  throw execution.admissionTimeout;
4746
- if (execution.yieldedToPush) {
4747
- throw new RealtimePushYieldError();
4748
- }
4749
4228
  const exitCode = passExitCode(execution.result);
4750
4229
  if (shouldAcknowledgeParallelUnresolvedCompany(execution.accepted, exitCode)) {
4751
4230
  if (!unresolvedReceiverMessagesWereRedelivered(events, receiveCountsByPath)) {
@@ -4767,82 +4246,74 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4767
4246
  const receiverSyncBatchFn = async ({ events, signal, approximateReceiveCountsByPath, settleEvents, }) => {
4768
4247
  if (stopped)
4769
4248
  return;
4770
- try {
4771
- // The injectable receiver seam and the production SQS receiver both
4772
- // arrive here only after a successful receive. Keep the cadence tight
4773
- // until the receiver later reports an empty outer long-poll.
4774
- onReceiveActivity(true);
4775
- pendingReceivers += events.length;
4776
- emitSyncWorkState();
4777
- // A received wake/high-water signal means there is bounded work now.
4778
- // It wakes the scheduler but is still funneled through runGuarded below,
4779
- // so it cannot overlap the pass that may currently own the operation
4780
- // lock.
4781
- // Apply the caps once across the whole delivery, then retain the residue
4782
- // in the normal lane. The receiver already coalesces to one latest event
4783
- // per path, so the partition cannot split revisions of one path.
4784
- if (parallelCoordinatorEnabled) {
4785
- const lane = receiverLaneByBatch.get(events);
4786
- if (lane) {
4787
- receiverLaneByBatch.delete(events);
4788
- await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
4789
- return;
4790
- }
4791
- // Direct embedders may invoke the engine without first going through
4792
- // SqsPushReceiver. Preserve that seam while applying the same
4793
- // delivery-wide partition the receiver would expose.
4794
- const lanes = partitionByKnownSizes(events, receiverEventSize);
4795
- await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
4796
- await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4249
+ // The injectable receiver seam and the production SQS receiver both
4250
+ // arrive here only after a successful receive. Keep the cadence tight
4251
+ // until the receiver later reports an empty outer long-poll.
4252
+ onReceiveActivity(true);
4253
+ // A received wake/high-water signal means there is bounded work now.
4254
+ // It wakes the scheduler but is still funneled through runGuarded below,
4255
+ // so it cannot overlap the pass that may currently own the operation
4256
+ // lock.
4257
+ // Apply the caps once across the whole delivery, then retain the residue
4258
+ // in the normal lane. The receiver already coalesces to one latest event
4259
+ // per path, so the partition cannot split revisions of one path.
4260
+ if (parallelCoordinatorEnabled) {
4261
+ const lane = receiverLaneByBatch.get(events);
4262
+ if (lane) {
4263
+ receiverLaneByBatch.delete(events);
4264
+ await drainParallelReceiverLane(lane, events, approximateReceiveCountsByPath, settleEvents, signal);
4797
4265
  return;
4798
4266
  }
4267
+ // Direct embedders may invoke the engine without first going through
4268
+ // SqsPushReceiver. Preserve that seam while applying the same
4269
+ // delivery-wide partition the receiver would expose.
4799
4270
  const lanes = partitionByKnownSizes(events, receiverEventSize);
4800
- const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
4801
- const eventsByRoute = new Map();
4802
- for (const event of laneEvents) {
4803
- const route = routeChangeToTarget(event.relativePath);
4804
- // The receiver is an independent ingress from the local watcher.
4805
- // Apply the same route-selection boundary before it can enqueue a
4806
- // fast pull: a --skip-personal runner must acknowledge, not pull,
4807
- // a personal-namespace delivery.
4808
- if (!route || !isWatchRouteSelected(route, watchRouteSelection))
4809
- continue;
4810
- const key = routeKey(route);
4811
- const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
4812
- group.events.push(event);
4813
- if (event.kind === "upsert")
4814
- group.upsertPaths.push(event.relativePath);
4815
- eventsByRoute.set(key, group);
4816
- }
4817
- for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
4818
- const paths = routeEvents.map((event) => event.relativePath);
4819
- const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
4820
- const exitCode = passExitCode(result);
4821
- if (exitCode === UNRESOLVED_COMPANY_EXIT) {
4822
- if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
4823
- throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4824
- }
4825
- warnUnresolvedReceiverCompany(route, routeEvents.length);
4826
- await settleEvents?.(routeEvents);
4827
- continue;
4828
- }
4829
- if (exitCode !== 0) {
4271
+ await drainParallelReceiverLane("fast", lanes.fast, approximateReceiveCountsByPath, settleEvents, signal);
4272
+ await drainParallelReceiverLane("slow", lanes.slow, approximateReceiveCountsByPath, settleEvents, signal);
4273
+ return;
4274
+ }
4275
+ const lanes = partitionByKnownSizes(events, receiverEventSize);
4276
+ const drainLane = async (lane, laneEvents, receiveCountsByPath, receiverSignal) => {
4277
+ const eventsByRoute = new Map();
4278
+ for (const event of laneEvents) {
4279
+ const route = routeChangeToTarget(event.relativePath);
4280
+ // The receiver is an independent ingress from the local watcher.
4281
+ // Apply the same route-selection boundary before it can enqueue a
4282
+ // fast pull: a --skip-personal runner must acknowledge, not pull,
4283
+ // a personal-namespace delivery.
4284
+ if (!route || !isWatchRouteSelected(route, watchRouteSelection))
4285
+ continue;
4286
+ const key = routeKey(route);
4287
+ const group = eventsByRoute.get(key) ?? { route, events: [], upsertPaths: [] };
4288
+ group.events.push(event);
4289
+ if (event.kind === "upsert")
4290
+ group.upsertPaths.push(event.relativePath);
4291
+ eventsByRoute.set(key, group);
4292
+ }
4293
+ for (const { route, events: routeEvents, upsertPaths } of eventsByRoute.values()) {
4294
+ const paths = routeEvents.map((event) => event.relativePath);
4295
+ const result = await withScopeContentHashesFile(route, routeEvents, paths, (contentHashesFile) => runGuarded(buildTargetedPullArgv(route, passArgv, paths, upsertPaths, contentHashesFile), undefined, lane, "realtime", false, receiverSignal));
4296
+ const exitCode = passExitCode(result);
4297
+ if (exitCode === UNRESOLVED_COMPANY_EXIT) {
4298
+ if (!unresolvedReceiverMessagesWereRedelivered(routeEvents, receiveCountsByPath)) {
4830
4299
  throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4831
4300
  }
4301
+ warnUnresolvedReceiverCompany(route, routeEvents.length);
4832
4302
  await settleEvents?.(routeEvents);
4303
+ continue;
4833
4304
  }
4834
- };
4835
- // Dispatch realtime work before awaiting its bulk residue. This lets an
4836
- // established slow pass continue while the small scoped pull uses its
4837
- // ref-counted lock carve-out; the residue remains serialized and cannot
4838
- // starve the slow lane.
4839
- await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
4840
- await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
4841
- }
4842
- finally {
4843
- pendingReceivers = Math.max(0, pendingReceivers - events.length);
4844
- emitSyncWorkState();
4845
- }
4305
+ if (exitCode !== 0) {
4306
+ throw targetedReceiverDispatchError(result, true, runtime.authRequiredPassExit);
4307
+ }
4308
+ await settleEvents?.(routeEvents);
4309
+ }
4310
+ };
4311
+ // Dispatch realtime work before awaiting its bulk residue. This lets an
4312
+ // established slow pass continue while the small scoped pull uses its
4313
+ // ref-counted lock carve-out; the residue remains serialized and cannot
4314
+ // starve the slow lane.
4315
+ await drainLane("fast", lanes.fast, approximateReceiveCountsByPath, signal);
4316
+ await drainLane("slow", lanes.slow, approximateReceiveCountsByPath, signal);
4846
4317
  };
4847
4318
  receiverSyncBatchFn.partition = (events, maxPaths) => {
4848
4319
  const lanes = partitionByKnownSizes(events, receiverEventSize);
@@ -4999,14 +4470,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
4999
4470
  if (stopped)
5000
4471
  return shutdownPromise ?? Promise.resolve();
5001
4472
  stopped = true;
5002
- clearInterval(watcherPushStagesTimer);
5003
- if (durableWatcherPersistTimer !== undefined)
5004
- clearTimeout(durableWatcherPersistTimer);
5005
- durableWatcherPersistTimer = undefined;
5006
- if (syncWorkStateTimer !== undefined)
5007
- clearTimeout(syncWorkStateTimer);
5008
- syncWorkStateTimer = undefined;
5009
- emitSyncWorkState(true);
5010
4473
  for (const route of routePushRetryTimerHandles.keys()) {
5011
4474
  clearRoutePushRetry(route);
5012
4475
  }
@@ -5036,7 +4499,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5036
4499
  coordinatorDisposal,
5037
4500
  boundedEventSyncStartup,
5038
4501
  eventSyncDisposal,
5039
- persistDurableWatcherPaths(),
5040
4502
  ]).then(() => undefined);
5041
4503
  try {
5042
4504
  void receiver?.dispose();
@@ -5454,14 +4916,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5454
4916
  lastCompletedFullReconcileWallAt = Date.now();
5455
4917
  fullPassJournalHealthy = execution.accepted &&
5456
4918
  isFullPassJournalHealthy(exit, typeof execution.result === "number" ? undefined : execution.result.result?.partial);
5457
- if (fullPassJournalHealthy && pending.batch?.overflowed === true) {
5458
- clearDurableWatcherOverflow();
5459
- }
5460
- else if (fullPassJournalHealthy && pending.batch !== null) {
5461
- acknowledgeDurableWatcherPathsCoveredByFullPass([
5462
- ...pending.batch.paths.values(),
5463
- ]);
5464
- }
5465
4919
  if (fullPassJournalHealthy) {
5466
4920
  fullPassContinuity.markFullPassCompleted(lastCompletedFullReconcileAt);
5467
4921
  }
@@ -5519,11 +4973,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5519
4973
  const catchupBatch = workingBatch;
5520
4974
  const catchupGeneration = pending.generation;
5521
4975
  void runScopedDrain(catchupBatch, catchupGeneration, true).then((exit) => {
5522
- if (exit === 0) {
5523
- clearDurableWatcherOverflow();
5524
- return;
5525
- }
5526
- if (exit === SCOPED_DRAIN_DEFERRED)
4976
+ if (exit === 0 || exit === SCOPED_DRAIN_DEFERRED)
5527
4977
  return;
5528
4978
  restorePendingWatcherPaths(catchupBatch.paths, catchupGeneration);
5529
4979
  interruptWait();
@@ -5631,19 +5081,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
5631
5081
  ? exitCode === 0
5632
5082
  : hasCompletedRunnerPass(result));
5633
5083
  if (completedPostRepairPass) {
5634
- if (!parallelCoordinatorEnabled &&
5635
- pending.batch?.overflowed === true &&
5636
- ((ranFullReconcile && fullPassJournalHealthy) || (!watcherWarmupCatchup && exitCode === 0))) {
5637
- clearDurableWatcherOverflow();
5638
- }
5639
- else if (!parallelCoordinatorEnabled &&
5640
- ranFullReconcile &&
5641
- fullPassJournalHealthy &&
5642
- pending.batch !== null) {
5643
- acknowledgeDurableWatcherPathsCoveredByFullPass([
5644
- ...pending.batch.paths.values(),
5645
- ]);
5646
- }
5647
5084
  // The production adapter returns RunnerPassResult only after fanout,
5648
5085
  // which follows journal repair. Its numeric seam has the same
5649
5086
  // contract: zero means the injected pass completed. This preserves