@indigoai-us/hq-cloud 6.16.83-alpha.0 → 6.16.85-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
  2. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-watch-loop.js +127 -61
  4. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  5. package/dist/bin/sync-runner.d.ts +2 -0
  6. package/dist/bin/sync-runner.d.ts.map +1 -1
  7. package/dist/bin/sync-runner.js.map +1 -1
  8. package/dist/bin/sync-runner.test.js +641 -296
  9. package/dist/bin/sync-runner.test.js.map +1 -1
  10. package/dist/cli/share.d.ts.map +1 -1
  11. package/dist/cli/share.js +50 -9
  12. package/dist/cli/share.js.map +1 -1
  13. package/dist/cli/share.test.js +224 -1
  14. package/dist/cli/share.test.js.map +1 -1
  15. package/dist/lib/machine-id.d.ts +7 -0
  16. package/dist/lib/machine-id.d.ts.map +1 -1
  17. package/dist/lib/machine-id.js +54 -7
  18. package/dist/lib/machine-id.js.map +1 -1
  19. package/dist/lib/machine-id.test.js +20 -1
  20. package/dist/lib/machine-id.test.js.map +1 -1
  21. package/dist/object-io.d.ts +2 -0
  22. package/dist/object-io.d.ts.map +1 -1
  23. package/dist/object-io.js +78 -27
  24. package/dist/object-io.js.map +1 -1
  25. package/dist/object-io.test.js +107 -2
  26. package/dist/object-io.test.js.map +1 -1
  27. package/dist/sync/event-sync.d.ts +14 -4
  28. package/dist/sync/event-sync.d.ts.map +1 -1
  29. package/dist/sync/event-sync.js +148 -14
  30. package/dist/sync/event-sync.js.map +1 -1
  31. package/dist/sync/event-sync.test.js +321 -2
  32. package/dist/sync/event-sync.test.js.map +1 -1
  33. package/dist/sync/full-pass-interval.d.ts +25 -0
  34. package/dist/sync/full-pass-interval.d.ts.map +1 -0
  35. package/dist/sync/full-pass-interval.js +53 -0
  36. package/dist/sync/full-pass-interval.js.map +1 -0
  37. package/dist/sync/full-pass-interval.test.d.ts +2 -0
  38. package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
  39. package/dist/sync/full-pass-interval.test.js +21 -0
  40. package/dist/sync/full-pass-interval.test.js.map +1 -0
  41. package/dist/sync/parallel-sync-coordinator.d.ts +1 -0
  42. package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
  43. package/dist/sync/parallel-sync-coordinator.js +39 -8
  44. package/dist/sync/parallel-sync-coordinator.js.map +1 -1
  45. package/dist/sync/parallel-sync-coordinator.test.js +33 -1
  46. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  47. package/dist/sync/push-receiver.d.ts +3 -1
  48. package/dist/sync/push-receiver.d.ts.map +1 -1
  49. package/dist/sync/push-receiver.js +45 -9
  50. package/dist/sync/push-receiver.js.map +1 -1
  51. package/dist/sync/push-receiver.test.js +70 -0
  52. package/dist/sync/push-receiver.test.js.map +1 -1
  53. package/dist/sync/push-transport.d.ts +15 -0
  54. package/dist/sync/push-transport.d.ts.map +1 -1
  55. package/dist/sync/push-transport.js +27 -0
  56. package/dist/sync/push-transport.js.map +1 -1
  57. package/dist/sync/push-transport.test.js +14 -1
  58. package/dist/sync/push-transport.test.js.map +1 -1
  59. package/dist/sync/sync-work.d.ts +5 -0
  60. package/dist/sync/sync-work.d.ts.map +1 -1
  61. package/dist/sync/sync-work.js.map +1 -1
  62. package/package.json +1 -1
@@ -240,26 +240,15 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
240
240
  remainder: TreeChangeBatch | null;
241
241
  };
242
242
  /**
243
- * Elapsed-time cadence for scheduled full reconciles: one hour by default,
244
- * measured from the last full pass (its dispatch or its completion, whichever
245
- * is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
246
- * 10_800_000 for three hours); an unset or invalid value falls back to the
247
- * hour.
248
- *
249
- * The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
250
- * elapsed-time floor. Ticks fire on every watcher batch, not only on the
251
- * remote poll, so on an active vault a new full pass started 4-8 minutes —
252
- * once observed 21 seconds — after the previous one finished, and each pass
253
- * on a real install takes ~10 minutes of listing and HEAD traffic. The runner
254
- * was effectively reconciling continuously. Full reconcile is the correctness
255
- * backstop behind event-push (a missed watcher event, a new directory the
256
- * native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
257
- * and bare-wake triggers still force it immediately when live events cannot
258
- * be trusted, so an hour is the cadence of the backstop, not of realtime.
243
+ * A failed watch pass is still retried in-process, but even an explicitly
244
+ * tiny poll interval must not turn a broken dependency into a hot loop.
245
+ * Three failures in thirty minutes escalate from this floor to exponential
246
+ * backoff; success or a quiet window starts a fresh budget.
259
247
  */
260
- export declare const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
261
- export declare const DEFAULT_FULL_RECONCILE_MS: number | undefined;
262
- export declare function resolveFullReconcileMs(env?: Record<string, string | undefined>): number | undefined;
248
+ export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
249
+ export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
250
+ export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
251
+ export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
263
252
  /** Initial poll-only recovery delay after a watcher conclusively stands down. */
264
253
  export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
265
254
  /** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
@@ -290,6 +279,20 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
290
279
  nextDueAt: number;
291
280
  delayMs: number;
292
281
  };
282
+ /**
283
+ * Bounds in-process retries after a completed watch pass returned non-zero.
284
+ *
285
+ * This is deliberately separate from the adaptive poll cadence: cadence
286
+ * controls normal remote work, whereas this budget prevents a configured
287
+ * short poll from continuously re-running a pass that is already known to
288
+ * have failed. The caller supplies the monotonic scheduler clock so tests do
289
+ * not need wall-clock waits.
290
+ */
291
+ export declare class WatchPassExitRetryBudget {
292
+ private lastFailureAt;
293
+ private failures;
294
+ record(exitCode: number, now: number): number;
295
+ }
293
296
  export interface ParsedLoopArgs {
294
297
  hqRoot: string;
295
298
  lockTimeoutSec?: number;
@@ -1 +1 @@
1
- {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAcvB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA01GjB"}
1
+ {"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAcvB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAE1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAeD;;;;;GAKG;AACH,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,eAAO,MAAM,uCAAuC,IAAI,CAAC;AACzD,eAAO,MAAM,gCAAgC,QAAS,CAAC;AACvD,eAAO,MAAM,4BAA4B,QAAc,CAAC;AAGxD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAK;IAErB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;CAgC9C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA+5GjB"}
@@ -11,7 +11,7 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
11
11
  import { describeError } from "../lib/describe-error.js";
12
12
  import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
13
13
  import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
14
- import { getOrCreateMachineId } from "../lib/machine-id.js";
14
+ import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
15
15
  import { localPathForVaultKey } from "../local-path-codec.js";
16
16
  import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
17
17
  import { recoverHqRootLocalState } from "../hq-root-recovery.js";
@@ -27,12 +27,12 @@ import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPre
27
27
  import { setRunnerEmitObserver } from "./sync-runner-events.js";
28
28
  import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
29
29
  import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
30
+ import { createHqFlagsIntervalSource, } from "../sync/full-pass-interval.js";
30
31
  import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
31
32
  import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
32
33
  import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
33
34
  import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
34
35
  import { resolveSkipCompanies } from "./sync-runner-planning.js";
35
- const FULL_RECONCILE_EVERY_TICKS = 10;
36
36
  /** A deferred full chore must get the next admission after this long. */
37
37
  export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
38
38
  /**
@@ -857,36 +857,6 @@ export function takeEnumeratedBatchChunk(batch, limit) {
857
857
  remainder: asBatch(remainderPaths, splitChanges(remainderPaths)),
858
858
  };
859
859
  }
860
- /**
861
- * Elapsed-time cadence for scheduled full reconciles: one hour by default,
862
- * measured from the last full pass (its dispatch or its completion, whichever
863
- * is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
864
- * 10_800_000 for three hours); an unset or invalid value falls back to the
865
- * hour.
866
- *
867
- * The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
868
- * elapsed-time floor. Ticks fire on every watcher batch, not only on the
869
- * remote poll, so on an active vault a new full pass started 4-8 minutes —
870
- * once observed 21 seconds — after the previous one finished, and each pass
871
- * on a real install takes ~10 minutes of listing and HEAD traffic. The runner
872
- * was effectively reconciling continuously. Full reconcile is the correctness
873
- * backstop behind event-push (a missed watcher event, a new directory the
874
- * native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
875
- * and bare-wake triggers still force it immediately when live events cannot
876
- * be trusted, so an hour is the cadence of the backstop, not of realtime.
877
- */
878
- export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
879
- export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
880
- export function resolveFullReconcileMs(env = process.env) {
881
- const raw = env[FULL_RECONCILE_MS_ENV];
882
- if (raw === undefined || raw.trim() === "")
883
- return DEFAULT_FULL_RECONCILE_MS;
884
- const parsed = Number(raw);
885
- if (!Number.isFinite(parsed) || parsed <= 0)
886
- return DEFAULT_FULL_RECONCILE_MS;
887
- const intervalMs = Math.trunc(parsed);
888
- return intervalMs > 0 ? intervalMs : DEFAULT_FULL_RECONCILE_MS;
889
- }
890
860
  /**
891
861
  * Adaptive remote-poll backoff bounds (whole-box CPU aware). When
892
862
  * `--poll-remote-ms` is NOT passed, the watch loop scales its poll cadence to
@@ -900,6 +870,16 @@ export function resolveFullReconcileMs(env = process.env) {
900
870
  const ADAPTIVE_POLL_FLOOR_MS = 60_000;
901
871
  const ADAPTIVE_POLL_CEIL_MS = 600_000;
902
872
  const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
873
+ /**
874
+ * A failed watch pass is still retried in-process, but even an explicitly
875
+ * tiny poll interval must not turn a broken dependency into a hot loop.
876
+ * Three failures in thirty minutes escalate from this floor to exponential
877
+ * backoff; success or a quiet window starts a fresh budget.
878
+ */
879
+ export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
880
+ export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
881
+ export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
882
+ export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
903
883
  /** Covers coarse timestamp filesystems and scheduling just before watcher start. */
904
884
  const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
905
885
  /** Initial poll-only recovery delay after a watcher conclusively stands down. */
@@ -949,6 +929,41 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
949
929
  nextDueAt = now + intervalMs;
950
930
  return { nextDueAt, delayMs: intervalMs };
951
931
  }
932
+ /**
933
+ * Bounds in-process retries after a completed watch pass returned non-zero.
934
+ *
935
+ * This is deliberately separate from the adaptive poll cadence: cadence
936
+ * controls normal remote work, whereas this budget prevents a configured
937
+ * short poll from continuously re-running a pass that is already known to
938
+ * have failed. The caller supplies the monotonic scheduler clock so tests do
939
+ * not need wall-clock waits.
940
+ */
941
+ export class WatchPassExitRetryBudget {
942
+ lastFailureAt = null;
943
+ failures = 0;
944
+ record(exitCode, now) {
945
+ if (exitCode === 0) {
946
+ this.lastFailureAt = null;
947
+ this.failures = 0;
948
+ return 0;
949
+ }
950
+ if (this.lastFailureAt === null ||
951
+ now < this.lastFailureAt ||
952
+ now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
953
+ this.lastFailureAt = now;
954
+ this.failures = 0;
955
+ }
956
+ this.failures += 1;
957
+ this.lastFailureAt = now;
958
+ if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
959
+ // Preserve the established normal-cadence retry for isolated failures.
960
+ // The bounded cooldown starts only after repeated exits in the window.
961
+ return 0;
962
+ }
963
+ const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
964
+ return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
965
+ }
966
+ }
952
967
  /**
953
968
  * Select journal entries covered by one watcher delete. Exact file unlinks
954
969
  * deliberately avoid enumeration; only directory unlinks expand descendants.
@@ -1070,7 +1085,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1070
1085
  const sampleLoadAvg = deps.sampleLoadAvg ?? (() => os.loadavg()[0]);
1071
1086
  const cpuCount = os.cpus().length;
1072
1087
  const eventPush = argv.includes("--event-push");
1073
- const fullReconcileIntervalMs = resolveFullReconcileMs();
1088
+ const fullPassIntervalSource = deps.fullPassIntervalSource ??
1089
+ createHqFlagsIntervalSource({
1090
+ endpoint: process.env.HQ_FLAGS_API_URL,
1091
+ getToken: runtime.defaultGetAccessToken,
1092
+ });
1093
+ let fullReconcileIntervalMs = fullPassIntervalSource.read();
1074
1094
  const companiesMode = argv.includes("--companies");
1075
1095
  const companyIdx = argv.indexOf("--company");
1076
1096
  const companyMode = companyIdx >= 0;
@@ -1240,11 +1260,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1240
1260
  flags: flagSource,
1241
1261
  platform: process.platform,
1242
1262
  });
1243
- if (!deps.fullPassCaps && process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
1263
+ if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
1244
1264
  const refresh = setTimeout(() => {
1245
- void flagSource.refresh?.().then((value) => {
1246
- if (value)
1247
- fullPassCaps.applyFlags(value);
1265
+ void Promise.all([
1266
+ deps.fullPassCaps ? Promise.resolve(null) : flagSource.refresh?.(),
1267
+ deps.fullPassIntervalSource ? Promise.resolve(null) : fullPassIntervalSource.refresh?.(),
1268
+ ]).then(([capFlags, intervalMs]) => {
1269
+ if (capFlags)
1270
+ fullPassCaps.applyFlags(capFlags);
1271
+ if (intervalMs)
1272
+ fullReconcileIntervalMs = intervalMs;
1248
1273
  });
1249
1274
  }, 0);
1250
1275
  refresh.unref?.();
@@ -1461,8 +1486,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1461
1486
  resolveInterruptWait = null;
1462
1487
  resolve?.();
1463
1488
  };
1464
- const waitForNextPass = async (delayMs) => {
1465
- if (interruptGeneration !== observedInterruptGeneration) {
1489
+ const waitForNextPass = async (delayMs, interruptible = true) => {
1490
+ if (interruptible && interruptGeneration !== observedInterruptGeneration) {
1466
1491
  observedInterruptGeneration = interruptGeneration;
1467
1492
  return;
1468
1493
  }
@@ -1477,9 +1502,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1477
1502
  }, idleHeartbeatIntervalMs);
1478
1503
  heartbeatTimer.unref?.();
1479
1504
  try {
1480
- await Promise.race([
1481
- sleep(delayMs),
1482
- new Promise((resolve) => {
1505
+ const waits = [sleep(delayMs), stoppedSignal];
1506
+ if (interruptible) {
1507
+ waits.push(new Promise((resolve) => {
1483
1508
  resolveInterruptWait = resolve;
1484
1509
  // Do not lose an event that arrived between the first generation
1485
1510
  // check and registration of this wait.
@@ -1487,9 +1512,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1487
1512
  resolveInterruptWait = null;
1488
1513
  resolve();
1489
1514
  }
1490
- }),
1491
- stoppedSignal,
1492
- ]);
1515
+ }));
1516
+ }
1517
+ await Promise.race(waits);
1493
1518
  }
1494
1519
  finally {
1495
1520
  clearInterval(heartbeatTimer);
@@ -1576,7 +1601,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1576
1601
  let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
1577
1602
  const runCoordinatorWork = async (work, task, options = {}) => {
1578
1603
  const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
1579
- ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
1604
+ ? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr, monotonicNow)
1580
1605
  : undefined;
1581
1606
  if (!coordinator) {
1582
1607
  return {
@@ -1605,10 +1630,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1605
1630
  const outcome = await coordinator.submit(work);
1606
1631
  const executorSucceeded = execution.result !== undefined &&
1607
1632
  coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
1608
- // A refusal without this execution's durable pull is only a coordinator
1609
- // scheduling result. It cannot prove that the receipt's bytes reached
1610
- // disk, so the SQS bridge must leave it retryable for a replacement pull.
1611
- const accepted = executorSucceeded && outcome.paths.every((path) => path.status === "durable" || path.status === "refused");
1633
+ // A receiver refusal belongs to a newer replacement pull. Even when
1634
+ // this slice durably completed its other paths, the SQS receipt batch
1635
+ // cannot be acknowledged until every path is durable. Watcher work
1636
+ // retains its subset-publish contract and filters refused paths below.
1637
+ const requireEveryPathDurable = work.class === "realtime" &&
1638
+ work.source === "receiver" &&
1639
+ work.direction === "pull";
1640
+ const accepted = executorSucceeded && outcome.paths.every((path) => requireEveryPathDurable
1641
+ ? path.status === "durable"
1642
+ : path.status === "durable" || path.status === "refused");
1612
1643
  if (telemetry) {
1613
1644
  const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
1614
1645
  ? "durable"
@@ -1638,16 +1669,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1638
1669
  }
1639
1670
  };
1640
1671
  let nextCoordinatorCausality = 0;
1641
- const realtimeWork = (source, direction, paths, lane) => ({
1672
+ const realtimeWork = (source, direction, paths, lane, sequenceNumberByPath) => ({
1642
1673
  class: "realtime",
1643
1674
  source,
1644
1675
  direction,
1645
1676
  lane,
1646
1677
  paths: normalizeCoordinatorPaths(paths),
1678
+ ...(sequenceNumberByPath === undefined ? {} : { sequenceNumberByPath }),
1647
1679
  causality: `${source}-${++nextCoordinatorCausality}`,
1648
1680
  });
1649
1681
  const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
1650
- const runRealtimePass = (source, direction, paths, lane, task) => runCoordinatorWork(realtimeWork(source, direction, paths, lane), task);
1682
+ const runRealtimePass = (source, direction, paths, lane, task, sequenceNumberByPath) => runCoordinatorWork(realtimeWork(source, direction, paths, lane, sequenceNumberByPath), task);
1651
1683
  const enqueueChore = (kind, task, onSettled, options = { wholeMutation: true }) => {
1652
1684
  void runCoordinatorWork(choreWork(kind), task, options).then(onSettled).catch((error) => {
1653
1685
  process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
@@ -3066,7 +3098,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3066
3098
  // Submit every route in this lane before awaiting any result. In
3067
3099
  // particular, a later route's fast receipt must never wait for an
3068
3100
  // earlier route's slow pull to finish.
3069
- const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced))));
3101
+ const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced), new Map(events.map((event) => [event.relativePath, event.sequenceNumber])))));
3070
3102
  for (const execution of completed) {
3071
3103
  if (!execution.accepted || passExitCode(execution.result) !== 0) {
3072
3104
  throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
@@ -3218,11 +3250,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3218
3250
  authToken: getAccessToken,
3219
3251
  region: runtime.region,
3220
3252
  });
3253
+ const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
3221
3254
  const handles = await startES({
3222
3255
  hqRoot,
3223
3256
  apiUrl: runtime.apiUrl,
3224
3257
  authToken: getAccessToken,
3225
- deviceId: getOrCreateMachineId(hqRoot),
3258
+ deviceId: deviceIdentity.machineId,
3259
+ deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
3260
+ deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
3226
3261
  resolveTenantId: async () => {
3227
3262
  const persons = await eventSyncClient.entity.listByType("person");
3228
3263
  const pick = pickCanonicalPersonEntity(persons);
@@ -3525,6 +3560,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3525
3560
  // The initial reconcile is due immediately. On-time passes advance from this
3526
3561
  // deadline; overruns restart their cooldown from completion.
3527
3562
  let nextDueAt = monotonicNow();
3563
+ const passExitRetryBudget = new WatchPassExitRetryBudget();
3564
+ let passExitRetryNotBefore = null;
3565
+ const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
3566
+ const delayMs = passExitRetryBudget.record(exitCode, now);
3567
+ passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
3568
+ if (delayMs > 0) {
3569
+ process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
3570
+ }
3571
+ };
3528
3572
  let pollTick = 0;
3529
3573
  let lastCompletedFullReconcileAt = null;
3530
3574
  // Anchor for the elapsed-time cadence: the most recent full reconcile that
@@ -3554,6 +3598,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3554
3598
  return RUNNER_HEAP_RECYCLE_EXIT;
3555
3599
  if (coordinatorAuthRequired)
3556
3600
  return runtime.authRequiredPassExit;
3601
+ if (passExitRetryNotBefore !== null) {
3602
+ const retryNow = monotonicNow();
3603
+ if (retryNow < passExitRetryNotBefore) {
3604
+ // A watcher/receiver interrupt must not turn a known failing pass
3605
+ // back into a tight loop. Pending work remains queued and runs when
3606
+ // this non-interruptible cooldown has elapsed.
3607
+ await waitForNextPass(passExitRetryNotBefore - retryNow, false);
3608
+ continue;
3609
+ }
3610
+ passExitRetryNotBefore = null;
3611
+ }
3557
3612
  pollTick++;
3558
3613
  areaAutoMigration.onTick();
3559
3614
  const pending = takePendingWatcherChange();
@@ -3585,11 +3640,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3585
3640
  const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
3586
3641
  const coverageStale = rootCoverageStale;
3587
3642
  rootCoverageStale = false;
3588
- const scheduledInterval = fullReconcileIntervalMs === undefined
3589
- ? pollTick % FULL_RECONCILE_EVERY_TICKS === 0
3590
- : lastFullReconcileAt !== null &&
3591
- cadenceNow !== null &&
3592
- cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
3643
+ const scheduledInterval = lastFullReconcileAt !== null &&
3644
+ cadenceNow !== null &&
3645
+ cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
3593
3646
  const triggers = {
3594
3647
  "event-push-surfaces-inactive": eventPushSurfacesInactive,
3595
3648
  "first-tick": firstTick,
@@ -3649,6 +3702,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3649
3702
  };
3650
3703
  let result;
3651
3704
  let ranFullReconcile = false;
3705
+ let ranWatchPass = false;
3652
3706
  let enqueuedFullChore = false;
3653
3707
  if (shouldRunFull) {
3654
3708
  emitDecision("full", reasons);
@@ -3668,6 +3722,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3668
3722
  interruptWait();
3669
3723
  return;
3670
3724
  }
3725
+ recordWatchPassExit(exit);
3726
+ if (exit !== 0)
3727
+ interruptWait();
3671
3728
  // Preserve the numeric success seam accepted by the prior
3672
3729
  // coordinator while also accepting a completed RunnerPassOutcome
3673
3730
  // (including a partial per-company pass) that proves journal
@@ -3712,6 +3769,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3712
3769
  }
3713
3770
  else {
3714
3771
  ranFullReconcile = true;
3772
+ ranWatchPass = true;
3715
3773
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3716
3774
  }
3717
3775
  }
@@ -3741,6 +3799,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3741
3799
  result = 0;
3742
3800
  }
3743
3801
  else {
3802
+ ranWatchPass = true;
3744
3803
  result = overflowWithKnownRoutes
3745
3804
  ? await runOverflowRouteDrain(workingBatch)
3746
3805
  : await runScopedDrain(workingBatch, pending.generation);
@@ -3758,6 +3817,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3758
3817
  process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
3759
3818
  `falling back to a full reconcile\n`);
3760
3819
  ranFullReconcile = true;
3820
+ ranWatchPass = true;
3761
3821
  emitDecision("full", ["targeted-pass-failed"]);
3762
3822
  result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
3763
3823
  }
@@ -3903,6 +3963,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3903
3963
  realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
3904
3964
  }
3905
3965
  const now = monotonicNow();
3966
+ if (ranWatchPass)
3967
+ recordWatchPassExit(exitCode, now);
3906
3968
  if (ranFullReconcile) {
3907
3969
  // The awaited full pass finished on this tick; reuse the tick's clock
3908
3970
  // sample so the cadence anchor never costs a second read.
@@ -3922,10 +3984,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
3922
3984
  ? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
3923
3985
  : advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
3924
3986
  nextDueAt = schedule.nextDueAt;
3987
+ const retryDelayMs = passExitRetryNotBefore === null
3988
+ ? 0
3989
+ : Math.max(0, passExitRetryNotBefore - now);
3990
+ const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
3925
3991
  const untilHighWaterMs = realtimeState.nextHighWaterAt === null
3926
- ? schedule.delayMs
3992
+ ? nextPassDelayMs
3927
3993
  : Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
3928
- await waitForNextPass(Math.min(schedule.delayMs, untilHighWaterMs));
3994
+ await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
3929
3995
  }
3930
3996
  return 0;
3931
3997
  }