@indigoai-us/hq-cloud 6.16.82-alpha.0 → 6.16.84-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
  2. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-watch-loop.js +118 -71
  4. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  5. package/dist/bin/sync-runner.d.ts +2 -0
  6. package/dist/bin/sync-runner.d.ts.map +1 -1
  7. package/dist/bin/sync-runner.js.map +1 -1
  8. package/dist/bin/sync-runner.test.js +507 -248
  9. package/dist/bin/sync-runner.test.js.map +1 -1
  10. package/dist/cli/share.d.ts.map +1 -1
  11. package/dist/cli/share.js +50 -9
  12. package/dist/cli/share.js.map +1 -1
  13. package/dist/cli/share.test.js +224 -1
  14. package/dist/cli/share.test.js.map +1 -1
  15. package/dist/lib/machine-id.d.ts +7 -0
  16. package/dist/lib/machine-id.d.ts.map +1 -1
  17. package/dist/lib/machine-id.js +54 -7
  18. package/dist/lib/machine-id.js.map +1 -1
  19. package/dist/lib/machine-id.test.js +20 -1
  20. package/dist/lib/machine-id.test.js.map +1 -1
  21. package/dist/object-io.d.ts +2 -0
  22. package/dist/object-io.d.ts.map +1 -1
  23. package/dist/object-io.js +78 -27
  24. package/dist/object-io.js.map +1 -1
  25. package/dist/object-io.test.js +107 -2
  26. package/dist/object-io.test.js.map +1 -1
  27. package/dist/sync/event-sync.d.ts +15 -7
  28. package/dist/sync/event-sync.d.ts.map +1 -1
  29. package/dist/sync/event-sync.js +148 -15
  30. package/dist/sync/event-sync.js.map +1 -1
  31. package/dist/sync/event-sync.test.js +321 -2
  32. package/dist/sync/event-sync.test.js.map +1 -1
  33. package/dist/sync/full-pass-interval.d.ts +25 -0
  34. package/dist/sync/full-pass-interval.d.ts.map +1 -0
  35. package/dist/sync/full-pass-interval.js +53 -0
  36. package/dist/sync/full-pass-interval.js.map +1 -0
  37. package/dist/sync/full-pass-interval.test.d.ts +2 -0
  38. package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
  39. package/dist/sync/full-pass-interval.test.js +21 -0
  40. package/dist/sync/full-pass-interval.test.js.map +1 -0
  41. package/dist/sync/push-receiver.d.ts +0 -14
  42. package/dist/sync/push-receiver.d.ts.map +1 -1
  43. package/dist/sync/push-receiver.js +12 -49
  44. package/dist/sync/push-receiver.js.map +1 -1
  45. package/dist/sync/push-receiver.test.js +7 -171
  46. package/dist/sync/push-receiver.test.js.map +1 -1
  47. package/dist/sync/push-transport.d.ts +15 -0
  48. package/dist/sync/push-transport.d.ts.map +1 -1
  49. package/dist/sync/push-transport.js +27 -0
  50. package/dist/sync/push-transport.js.map +1 -1
  51. package/dist/sync/push-transport.test.js +14 -1
  52. package/dist/sync/push-transport.test.js.map +1 -1
  53. package/package.json +1 -1
@@ -240,26 +240,15 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
240
240
  remainder: TreeChangeBatch | null;
241
241
  };
242
242
  /**
243
- * Elapsed-time cadence for scheduled full reconciles: one hour by default,
244
- * measured from the last full pass (its dispatch or its completion, whichever
245
- * is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
246
- * 10_800_000 for three hours); an unset or invalid value falls back to the
247
- * hour.
248
- *
249
- * The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
250
- * elapsed-time floor. Ticks fire on every watcher batch, not only on the
251
- * remote poll, so on an active vault a new full pass started 4-8 minutes —
252
- * once observed 21 seconds — after the previous one finished, and each pass
253
- * on a real install takes ~10 minutes of listing and HEAD traffic. The runner
254
- * was effectively reconciling continuously. Full reconcile is the correctness
255
- * backstop behind event-push (a missed watcher event, a new directory the
256
- * native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
257
- * and bare-wake triggers still force it immediately when live events cannot
258
- * be trusted, so an hour is the cadence of the backstop, not of realtime.
243
+ * A failed watch pass is still retried in-process, but even an explicitly
244
+ * tiny poll interval must not turn a broken dependency into a hot loop.
245
+ * Three failures in thirty minutes escalate from this floor to exponential
246
+ * backoff; success or a quiet window starts a fresh budget.
259
247
  */
260
- export declare const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
261
- export declare const DEFAULT_FULL_RECONCILE_MS: number | undefined;
262
- export declare function resolveFullReconcileMs(env?: Record<string, string | undefined>): number | undefined;
248
+ export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
249
+ export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
250
+ export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
251
+ export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
263
252
  /** Initial poll-only recovery delay after a watcher conclusively stands down. */
264
253
  export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
265
254
  /** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
@@ -290,6 +279,20 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
290
279
  nextDueAt: number;
291
280
  delayMs: number;
292
281
  };
282
+ /**
283
+ * Bounds in-process retries after a completed watch pass returned non-zero.
284
+ *
285
+ * This is deliberately separate from the adaptive poll cadence: cadence
286
+ * controls normal remote work, whereas this budget prevents a configured
287
+ * short poll from continuously re-running a pass that is already known to
288
+ * have failed. The caller supplies the monotonic scheduler clock so tests do
289
+ * not need wall-clock waits.
290
+ */
291
+ export declare class WatchPassExitRetryBudget {
292
+ private lastFailureAt;
293
+ private failures;
294
+ record(exitCode: number, now: number): number;
295
+ }
293
296
  export interface ParsedLoopArgs {
294
297
  hqRoot: string;
295
298
  lockTimeoutSec?: number;
@@ -1 +1 @@
1
- {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAcvB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA01GjB"}
1
+ {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAE1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAeD;;;;;GAKG;AACH,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,eAAO,MAAM,uCAAuC,IAAI,CAAC;AACzD,eAAO,MAAM,gCAAgC,QAAS,CAAC;AACvD,eAAO,MAAM,4BAA4B,QAAc,CAAC;AAGxD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAK;IAErB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;CAgC9C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA04GjB"}
@@ -11,7 +11,7 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
11
11
  import { describeError } from "../lib/describe-error.js";
12
12
  import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
13
13
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
14
- import { getOrCreateMachineId } from "../lib/machine-id.js";
14
+ import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
15
15
  import { localPathForVaultKey } from "../local-path-codec.js";
16
16
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
17
17
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
@@ -27,12 +27,12 @@ import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPre
27
27
  import { setRunnerEmitObserver } from "./sync-runner-events.js";
28
28
  import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
29
29
  import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
30
+ import { createHqFlagsIntervalSource, } from "../sync/full-pass-interval.js";
30
31
  import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
31
32
  import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
32
33
  import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
33
34
  import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
34
35
  import { resolveSkipCompanies } from "./sync-runner-planning.js";
35
- const FULL_RECONCILE_EVERY_TICKS = 10;
36
36
  /** A deferred full chore must get the next admission after this long. */
37
37
  export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
38
38
  /**
@@ -857,36 +857,6 @@ export function takeEnumeratedBatchChunk(batch, limit) {
857
857
  remainder: asBatch(remainderPaths, splitChanges(remainderPaths)),
858
858
  };
859
859
  }
860
- /**
861
- * Elapsed-time cadence for scheduled full reconciles: one hour by default,
862
- * measured from the last full pass (its dispatch or its completion, whichever
863
- * is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
864
- * 10_800_000 for three hours); an unset or invalid value falls back to the
865
- * hour.
866
- *
867
- * The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
868
- * elapsed-time floor. Ticks fire on every watcher batch, not only on the
869
- * remote poll, so on an active vault a new full pass started 4-8 minutes —
870
- * once observed 21 seconds — after the previous one finished, and each pass
871
- * on a real install takes ~10 minutes of listing and HEAD traffic. The runner
872
- * was effectively reconciling continuously. Full reconcile is the correctness
873
- * backstop behind event-push (a missed watcher event, a new directory the
874
- * native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
875
- * and bare-wake triggers still force it immediately when live events cannot
876
- * be trusted, so an hour is the cadence of the backstop, not of realtime.
877
- */
878
- export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
879
- export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
880
- export function resolveFullReconcileMs(env = process.env) {
881
- const raw = env[FULL_RECONCILE_MS_ENV];
882
- if (raw === undefined || raw.trim() === "")
883
- return DEFAULT_FULL_RECONCILE_MS;
884
- const parsed = Number(raw);
885
- if (!Number.isFinite(parsed) || parsed <= 0)
886
- return DEFAULT_FULL_RECONCILE_MS;
887
- const intervalMs = Math.trunc(parsed);
888
- return intervalMs > 0 ? intervalMs : DEFAULT_FULL_RECONCILE_MS;
889
- }
890
860
  /**
891
861
  * Adaptive remote-poll backoff bounds (whole-box CPU aware). When
892
862
  * `--poll-remote-ms` is NOT passed, the watch loop scales its poll cadence to
@@ -900,6 +870,16 @@ export function resolveFullReconcileMs(env = process.env) {
900
870
  const ADAPTIVE_POLL_FLOOR_MS = 60_000;
901
871
  const ADAPTIVE_POLL_CEIL_MS = 600_000;
902
872
  const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
873
+ /**
874
+ * A failed watch pass is still retried in-process, but even an explicitly
875
+ * tiny poll interval must not turn a broken dependency into a hot loop.
876
+ * Three failures in thirty minutes escalate from this floor to exponential
877
+ * backoff; success or a quiet window starts a fresh budget.
878
+ */
879
+ export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
880
+ export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
881
+ export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
882
+ export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
903
883
  /** Covers coarse timestamp filesystems and scheduling just before watcher start. */
904
884
  const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
905
885
  /** Initial poll-only recovery delay after a watcher conclusively stands down. */
@@ -949,6 +929,41 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
949
929
  nextDueAt = now + intervalMs;
950
930
  return { nextDueAt, delayMs: intervalMs };
951
931
  }
932
+ /**
933
+ * Bounds in-process retries after a completed watch pass returned non-zero.
934
+ *
935
+ * This is deliberately separate from the adaptive poll cadence: cadence
936
+ * controls normal remote work, whereas this budget prevents a configured
937
+ * short poll from continuously re-running a pass that is already known to
938
+ * have failed. The caller supplies the monotonic scheduler clock so tests do
939
+ * not need wall-clock waits.
940
+ */
941
+ export class WatchPassExitRetryBudget {
942
+ lastFailureAt = null;
943
+ failures = 0;
944
+ record(exitCode, now) {
945
+ if (exitCode === 0) {
946
+ this.lastFailureAt = null;
947
+ this.failures = 0;
948
+ return 0;
949
+ }
950
+ if (this.lastFailureAt === null ||
951
+ now < this.lastFailureAt ||
952
+ now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
953
+ this.lastFailureAt = now;
954
+ this.failures = 0;
955
+ }
956
+ this.failures += 1;
957
+ this.lastFailureAt = now;
958
+ if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
959
+ // Preserve the established normal-cadence retry for isolated failures.
960
+ // The bounded cooldown starts only after repeated exits in the window.
961
+ return 0;
962
+ }
963
+ const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
964
+ return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
965
+ }
966
+ }
952
967
  /**
953
968
  * Select journal entries covered by one watcher delete. Exact file unlinks
954
969
  * deliberately avoid enumeration; only directory unlinks expand descendants.
@@ -1070,7 +1085,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1070
1085
  const sampleLoadAvg = deps.sampleLoadAvg ?? (() => os.loadavg()[0]);
1071
1086
  const cpuCount = os.cpus().length;
1072
1087
  const eventPush = argv.includes("--event-push");
1073
- const fullReconcileIntervalMs = resolveFullReconcileMs();
1088
+ const fullPassIntervalSource = deps.fullPassIntervalSource ??
1089
+ createHqFlagsIntervalSource({
1090
+ endpoint: process.env.HQ_FLAGS_API_URL,
1091
+ getToken: runtime.defaultGetAccessToken,
1092
+ });
1093
+ let fullReconcileIntervalMs = fullPassIntervalSource.read();
1074
1094
  const companiesMode = argv.includes("--companies");
1075
1095
  const companyIdx = argv.indexOf("--company");
1076
1096
  const companyMode = companyIdx >= 0;
@@ -1240,11 +1260,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1240
1260
  flags: flagSource,
1241
1261
  platform: process.platform,
1242
1262
  });
1243
- if (!deps.fullPassCaps && process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
1263
+ if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
1244
1264
  const refresh = setTimeout(() => {
1245
- void flagSource.refresh?.().then((value) => {
1246
- if (value)
1247
- fullPassCaps.applyFlags(value);
1265
+ void Promise.all([
1266
+ deps.fullPassCaps ? Promise.resolve(null) : flagSource.refresh?.(),
1267
+ deps.fullPassIntervalSource ? Promise.resolve(null) : fullPassIntervalSource.refresh?.(),
1268
+ ]).then(([capFlags, intervalMs]) => {
1269
+ if (capFlags)
1270
+ fullPassCaps.applyFlags(capFlags);
1271
+ if (intervalMs)
1272
+ fullReconcileIntervalMs = intervalMs;
1248
1273
  });
1249
1274
  }, 0);
1250
1275
  refresh.unref?.();
@@ -1461,8 +1486,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1461
1486
  resolveInterruptWait = null;
1462
1487
  resolve?.();
1463
1488
  };
1464
- const waitForNextPass = async (delayMs) => {
1465
- if (interruptGeneration !== observedInterruptGeneration) {
1489
+ const waitForNextPass = async (delayMs, interruptible = true) => {
1490
+ if (interruptible && interruptGeneration !== observedInterruptGeneration) {
1466
1491
  observedInterruptGeneration = interruptGeneration;
1467
1492
  return;
1468
1493
  }
@@ -1477,9 +1502,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1477
1502
  }, idleHeartbeatIntervalMs);
1478
1503
  heartbeatTimer.unref?.();
1479
1504
  try {
1480
- await Promise.race([
1481
- sleep(delayMs),
1482
- new Promise((resolve) => {
1505
+ const waits = [sleep(delayMs), stoppedSignal];
1506
+ if (interruptible) {
1507
+ waits.push(new Promise((resolve) => {
1483
1508
  resolveInterruptWait = resolve;
1484
1509
  // Do not lose an event that arrived between the first generation
1485
1510
  // check and registration of this wait.
@@ -1487,9 +1512,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1487
1512
  resolveInterruptWait = null;
1488
1513
  resolve();
1489
1514
  }
1490
- }),
1491
- stoppedSignal,
1492
- ]);
1515
+ }));
1516
+ }
1517
+ await Promise.race(waits);
1493
1518
  }
1494
1519
  finally {
1495
1520
  clearInterval(heartbeatTimer);
@@ -1576,7 +1601,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1576
1601
  let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
1577
1602
  const runCoordinatorWork = async (work, task, options = {}) => {
1578
1603
  const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
1579
- ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
1604
+ ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr, monotonicNow)
1580
1605
  : undefined;
1581
1606
  if (!coordinator) {
1582
1607
  return {
@@ -1605,10 +1630,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1605
1630
  const outcome = await coordinator.submit(work);
1606
1631
  const executorSucceeded = execution.result !== undefined &&
1607
1632
  coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
1608
- // A refusal without this execution's durable pull is only a coordinator
1609
- // scheduling result. It cannot prove that the receipt's bytes reached
1610
- // disk, so the SQS bridge must leave it retryable for a replacement pull.
1611
- const accepted = executorSucceeded && outcome.paths.every((path) => path.status === "durable" || path.status === "refused");
1633
+ const allRefused = outcome.paths.length > 0 && outcome.paths.every((path) => path.status === "refused");
1634
+ const accepted = executorSucceeded
1635
+ ? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
1636
+ : allRefused;
1612
1637
  if (telemetry) {
1613
1638
  const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
1614
1639
  ? "durable"
@@ -1618,7 +1643,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1618
1643
  telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
1619
1644
  }
1620
1645
  return {
1621
- result: execution.result ?? 1,
1646
+ result: execution.result ?? (allRefused ? 0 : 1),
1622
1647
  accepted,
1623
1648
  coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
1624
1649
  .filter((path) => path.status === "refused")
@@ -3218,11 +3243,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3218
3243
  authToken: getAccessToken,
3219
3244
  region: runtime.region,
3220
3245
  });
3246
+ const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
3221
3247
  const handles = await startES({
3222
3248
  hqRoot,
3223
3249
  apiUrl: runtime.apiUrl,
3224
3250
  authToken: getAccessToken,
3225
- deviceId: getOrCreateMachineId(hqRoot),
3251
+ deviceId: deviceIdentity.machineId,
3252
+ deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
3253
+ deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
3226
3254
  resolveTenantId: async () => {
3227
3255
  const persons = await eventSyncClient.entity.listByType("person");
3228
3256
  const pick = pickCanonicalPersonEntity(persons);
@@ -3238,18 +3266,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3238
3266
  publishedContentHashStateDir: getStateDir(),
3239
3267
  publishedContentHashLookup,
3240
3268
  onReceiveActivity,
3241
- onReceiptOutcome: (receipt) => {
3242
- // This is a bounded diagnostic trace, deliberately separate from
3243
- // low-cardinality metrics. It includes only the irreversible hash
3244
- // of the event path and the closed receipt outcome.
3245
- process.stderr.write(`${JSON.stringify({
3246
- event: "sync.realtime.receipt",
3247
- schemaVersion: 1,
3248
- source: "receiver",
3249
- direction: "pull",
3250
- ...receipt,
3251
- })}\n`);
3252
- },
3253
3269
  });
3254
3270
  if (!handles)
3255
3271
  return;
@@ -3525,6 +3541,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3525
3541
  // The initial reconcile is due immediately. On-time passes advance from this
3526
3542
  // deadline; overruns restart their cooldown from completion.
3527
3543
  let nextDueAt = monotonicNow();
3544
+ const passExitRetryBudget = new WatchPassExitRetryBudget();
3545
+ let passExitRetryNotBefore = null;
3546
+ const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
3547
+ const delayMs = passExitRetryBudget.record(exitCode, now);
3548
+ passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
3549
+ if (delayMs > 0) {
3550
+ process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
3551
+ }
3552
+ };
3528
3553
  let pollTick = 0;
3529
3554
  let lastCompletedFullReconcileAt = null;
3530
3555
  // Anchor for the elapsed-time cadence: the most recent full reconcile that
@@ -3554,6 +3579,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3554
3579
  return RUNNER_HEAP_RECYCLE_EXIT;
3555
3580
  if (coordinatorAuthRequired)
3556
3581
  return runtime.authRequiredPassExit;
3582
+ if (passExitRetryNotBefore !== null) {
3583
+ const retryNow = monotonicNow();
3584
+ if (retryNow < passExitRetryNotBefore) {
3585
+ // A watcher/receiver interrupt must not turn a known failing pass
3586
+ // back into a tight loop. Pending work remains queued and runs when
3587
+ // this non-interruptible cooldown has elapsed.
3588
+ await waitForNextPass(passExitRetryNotBefore - retryNow, false);
3589
+ continue;
3590
+ }
3591
+ passExitRetryNotBefore = null;
3592
+ }
3557
3593
  pollTick++;
3558
3594
  areaAutoMigration.onTick();
3559
3595
  const pending = takePendingWatcherChange();
@@ -3585,11 +3621,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3585
3621
  const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
3586
3622
  const coverageStale = rootCoverageStale;
3587
3623
  rootCoverageStale = false;
3588
- const scheduledInterval = fullReconcileIntervalMs === undefined
3589
- ? pollTick % FULL_RECONCILE_EVERY_TICKS === 0
3590
- : lastFullReconcileAt !== null &&
3591
- cadenceNow !== null &&
3592
- cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
3624
+ const scheduledInterval = lastFullReconcileAt !== null &&
3625
+ cadenceNow !== null &&
3626
+ cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
3593
3627
  const triggers = {
3594
3628
  "event-push-surfaces-inactive": eventPushSurfacesInactive,
3595
3629
  "first-tick": firstTick,
@@ -3649,6 +3683,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3649
3683
  };
3650
3684
  let result;
3651
3685
  let ranFullReconcile = false;
3686
+ let ranWatchPass = false;
3652
3687
  let enqueuedFullChore = false;
3653
3688
  if (shouldRunFull) {
3654
3689
  emitDecision("full", reasons);
@@ -3668,6 +3703,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3668
3703
  interruptWait();
3669
3704
  return;
3670
3705
  }
3706
+ recordWatchPassExit(exit);
3707
+ if (exit !== 0)
3708
+ interruptWait();
3671
3709
  // Preserve the numeric success seam accepted by the prior
3672
3710
  // coordinator while also accepting a completed RunnerPassOutcome
3673
3711
  // (including a partial per-company pass) that proves journal
@@ -3712,6 +3750,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3712
3750
  }
3713
3751
  else {
3714
3752
  ranFullReconcile = true;
3753
+ ranWatchPass = true;
3715
3754
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3716
3755
  }
3717
3756
  }
@@ -3741,6 +3780,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3741
3780
  result = 0;
3742
3781
  }
3743
3782
  else {
3783
+ ranWatchPass = true;
3744
3784
  result = overflowWithKnownRoutes
3745
3785
  ? await runOverflowRouteDrain(workingBatch)
3746
3786
  : await runScopedDrain(workingBatch, pending.generation);
@@ -3758,6 +3798,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3758
3798
  process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
3759
3799
  `falling back to a full reconcile\n`);
3760
3800
  ranFullReconcile = true;
3801
+ ranWatchPass = true;
3761
3802
  emitDecision("full", ["targeted-pass-failed"]);
3762
3803
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3763
3804
  }
@@ -3903,6 +3944,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3903
3944
  realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
3904
3945
  }
3905
3946
  const now = monotonicNow();
3947
+ if (ranWatchPass)
3948
+ recordWatchPassExit(exitCode, now);
3906
3949
  if (ranFullReconcile) {
3907
3950
  // The awaited full pass finished on this tick; reuse the tick's clock
3908
3951
  // sample so the cadence anchor never costs a second read.
@@ -3922,10 +3965,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3922
3965
  ? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
3923
3966
  : advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
3924
3967
  nextDueAt = schedule.nextDueAt;
3968
+ const retryDelayMs = passExitRetryNotBefore === null
3969
+ ? 0
3970
+ : Math.max(0, passExitRetryNotBefore - now);
3971
+ const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
3925
3972
  const untilHighWaterMs = realtimeState.nextHighWaterAt === null
3926
- ? schedule.delayMs
3973
+ ? nextPassDelayMs
3927
3974
  : Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
3928
- await waitForNextPass(Math.min(schedule.delayMs, untilHighWaterMs));
3975
+ await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
3929
3976
  }
3930
3977
  return 0;
3931
3978
  }