@indigoai-us/hq-cloud 6.16.83-alpha.0 → 6.16.85-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +127 -61
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +641 -296
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +50 -9
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +224 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +78 -27
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +107 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +148 -14
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +321 -2
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/full-pass-interval.d.ts +25 -0
- package/dist/sync/full-pass-interval.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.js +53 -0
- package/dist/sync/full-pass-interval.js.map +1 -0
- package/dist/sync/full-pass-interval.test.d.ts +2 -0
- package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.test.js +21 -0
- package/dist/sync/full-pass-interval.test.js.map +1 -0
- package/dist/sync/parallel-sync-coordinator.d.ts +1 -0
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +39 -8
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +33 -1
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +3 -1
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -9
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +70 -0
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync/sync-work.d.ts +5 -0
- package/dist/sync/sync-work.d.ts.map +1 -1
- package/dist/sync/sync-work.js.map +1 -1
- package/package.json +1 -1
|
@@ -240,26 +240,15 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
|
|
|
240
240
|
remainder: TreeChangeBatch | null;
|
|
241
241
|
};
|
|
242
242
|
/**
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
* hour.
|
|
248
|
-
*
|
|
249
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
250
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
251
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
252
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
253
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
254
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
255
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
256
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
257
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
258
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
243
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
244
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
245
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
246
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
259
247
|
*/
|
|
260
|
-
export declare const
|
|
261
|
-
export declare const
|
|
262
|
-
export declare
|
|
248
|
+
export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
|
|
249
|
+
export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
250
|
+
export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
|
|
251
|
+
export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
|
|
263
252
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
264
253
|
export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
|
|
265
254
|
/** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
|
|
@@ -290,6 +279,20 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
|
|
|
290
279
|
nextDueAt: number;
|
|
291
280
|
delayMs: number;
|
|
292
281
|
};
|
|
282
|
+
/**
|
|
283
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
284
|
+
*
|
|
285
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
286
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
287
|
+
* short poll from continuously re-running a pass that is already known to
|
|
288
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
289
|
+
* not need wall-clock waits.
|
|
290
|
+
*/
|
|
291
|
+
export declare class WatchPassExitRetryBudget {
|
|
292
|
+
private lastFailureAt;
|
|
293
|
+
private failures;
|
|
294
|
+
record(exitCode: number, now: number): number;
|
|
295
|
+
}
|
|
293
296
|
export interface ParsedLoopArgs {
|
|
294
297
|
hqRoot: string;
|
|
295
298
|
lockTimeoutSec?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAcvB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAcvB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAE1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAeD;;;;;GAKG;AACH,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,eAAO,MAAM,uCAAuC,IAAI,CAAC;AACzD,eAAO,MAAM,gCAAgC,QAAS,CAAC;AACvD,eAAO,MAAM,4BAA4B,QAAc,CAAC;AAGxD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAK;IAErB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;CAgC9C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA+5GjB"}
|
|
@@ -11,7 +11,7 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
|
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
13
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
|
-
import {
|
|
14
|
+
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
@@ -27,12 +27,12 @@ import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPre
|
|
|
27
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
28
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
29
|
import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
|
|
30
|
+
import { createHqFlagsIntervalSource, } from "../sync/full-pass-interval.js";
|
|
30
31
|
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
31
32
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
32
33
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
33
34
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
34
35
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
35
|
-
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
36
36
|
/** A deferred full chore must get the next admission after this long. */
|
|
37
37
|
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
38
38
|
/**
|
|
@@ -857,36 +857,6 @@ export function takeEnumeratedBatchChunk(batch, limit) {
|
|
|
857
857
|
remainder: asBatch(remainderPaths, splitChanges(remainderPaths)),
|
|
858
858
|
};
|
|
859
859
|
}
|
|
860
|
-
/**
|
|
861
|
-
* Elapsed-time cadence for scheduled full reconciles: one hour by default,
|
|
862
|
-
* measured from the last full pass (its dispatch or its completion, whichever
|
|
863
|
-
* is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
|
|
864
|
-
* 10_800_000 for three hours); an unset or invalid value falls back to the
|
|
865
|
-
* hour.
|
|
866
|
-
*
|
|
867
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
868
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
869
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
870
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
871
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
872
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
873
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
874
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
875
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
876
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
877
|
-
*/
|
|
878
|
-
export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
879
|
-
export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
|
|
880
|
-
export function resolveFullReconcileMs(env = process.env) {
|
|
881
|
-
const raw = env[FULL_RECONCILE_MS_ENV];
|
|
882
|
-
if (raw === undefined || raw.trim() === "")
|
|
883
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
884
|
-
const parsed = Number(raw);
|
|
885
|
-
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
886
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
887
|
-
const intervalMs = Math.trunc(parsed);
|
|
888
|
-
return intervalMs > 0 ? intervalMs : DEFAULT_FULL_RECONCILE_MS;
|
|
889
|
-
}
|
|
890
860
|
/**
|
|
891
861
|
* Adaptive remote-poll backoff bounds (whole-box CPU aware). When
|
|
892
862
|
* `--poll-remote-ms` is NOT passed, the watch loop scales its poll cadence to
|
|
@@ -900,6 +870,16 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
900
870
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
901
871
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
902
872
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
873
|
+
/**
|
|
874
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
875
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
876
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
877
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
878
|
+
*/
|
|
879
|
+
export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
|
|
880
|
+
export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
881
|
+
export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
|
|
882
|
+
export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
|
|
903
883
|
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
904
884
|
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
905
885
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
@@ -949,6 +929,41 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
949
929
|
nextDueAt = now + intervalMs;
|
|
950
930
|
return { nextDueAt, delayMs: intervalMs };
|
|
951
931
|
}
|
|
932
|
+
/**
|
|
933
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
934
|
+
*
|
|
935
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
936
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
937
|
+
* short poll from continuously re-running a pass that is already known to
|
|
938
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
939
|
+
* not need wall-clock waits.
|
|
940
|
+
*/
|
|
941
|
+
export class WatchPassExitRetryBudget {
|
|
942
|
+
lastFailureAt = null;
|
|
943
|
+
failures = 0;
|
|
944
|
+
record(exitCode, now) {
|
|
945
|
+
if (exitCode === 0) {
|
|
946
|
+
this.lastFailureAt = null;
|
|
947
|
+
this.failures = 0;
|
|
948
|
+
return 0;
|
|
949
|
+
}
|
|
950
|
+
if (this.lastFailureAt === null ||
|
|
951
|
+
now < this.lastFailureAt ||
|
|
952
|
+
now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
|
|
953
|
+
this.lastFailureAt = now;
|
|
954
|
+
this.failures = 0;
|
|
955
|
+
}
|
|
956
|
+
this.failures += 1;
|
|
957
|
+
this.lastFailureAt = now;
|
|
958
|
+
if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
|
|
959
|
+
// Preserve the established normal-cadence retry for isolated failures.
|
|
960
|
+
// The bounded cooldown starts only after repeated exits in the window.
|
|
961
|
+
return 0;
|
|
962
|
+
}
|
|
963
|
+
const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
|
|
964
|
+
return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
952
967
|
/**
|
|
953
968
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
954
969
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1070,7 +1085,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1070
1085
|
const sampleLoadAvg = deps.sampleLoadAvg ?? (() => os.loadavg()[0]);
|
|
1071
1086
|
const cpuCount = os.cpus().length;
|
|
1072
1087
|
const eventPush = argv.includes("--event-push");
|
|
1073
|
-
const
|
|
1088
|
+
const fullPassIntervalSource = deps.fullPassIntervalSource ??
|
|
1089
|
+
createHqFlagsIntervalSource({
|
|
1090
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
1091
|
+
getToken: runtime.defaultGetAccessToken,
|
|
1092
|
+
});
|
|
1093
|
+
let fullReconcileIntervalMs = fullPassIntervalSource.read();
|
|
1074
1094
|
const companiesMode = argv.includes("--companies");
|
|
1075
1095
|
const companyIdx = argv.indexOf("--company");
|
|
1076
1096
|
const companyMode = companyIdx >= 0;
|
|
@@ -1240,11 +1260,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1240
1260
|
flags: flagSource,
|
|
1241
1261
|
platform: process.platform,
|
|
1242
1262
|
});
|
|
1243
|
-
if (
|
|
1263
|
+
if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
1244
1264
|
const refresh = setTimeout(() => {
|
|
1245
|
-
void
|
|
1246
|
-
|
|
1247
|
-
|
|
1265
|
+
void Promise.all([
|
|
1266
|
+
deps.fullPassCaps ? Promise.resolve(null) : flagSource.refresh?.(),
|
|
1267
|
+
deps.fullPassIntervalSource ? Promise.resolve(null) : fullPassIntervalSource.refresh?.(),
|
|
1268
|
+
]).then(([capFlags, intervalMs]) => {
|
|
1269
|
+
if (capFlags)
|
|
1270
|
+
fullPassCaps.applyFlags(capFlags);
|
|
1271
|
+
if (intervalMs)
|
|
1272
|
+
fullReconcileIntervalMs = intervalMs;
|
|
1248
1273
|
});
|
|
1249
1274
|
}, 0);
|
|
1250
1275
|
refresh.unref?.();
|
|
@@ -1461,8 +1486,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1461
1486
|
resolveInterruptWait = null;
|
|
1462
1487
|
resolve?.();
|
|
1463
1488
|
};
|
|
1464
|
-
const waitForNextPass = async (delayMs) => {
|
|
1465
|
-
if (interruptGeneration !== observedInterruptGeneration) {
|
|
1489
|
+
const waitForNextPass = async (delayMs, interruptible = true) => {
|
|
1490
|
+
if (interruptible && interruptGeneration !== observedInterruptGeneration) {
|
|
1466
1491
|
observedInterruptGeneration = interruptGeneration;
|
|
1467
1492
|
return;
|
|
1468
1493
|
}
|
|
@@ -1477,9 +1502,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1477
1502
|
}, idleHeartbeatIntervalMs);
|
|
1478
1503
|
heartbeatTimer.unref?.();
|
|
1479
1504
|
try {
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
new Promise((resolve) => {
|
|
1505
|
+
const waits = [sleep(delayMs), stoppedSignal];
|
|
1506
|
+
if (interruptible) {
|
|
1507
|
+
waits.push(new Promise((resolve) => {
|
|
1483
1508
|
resolveInterruptWait = resolve;
|
|
1484
1509
|
// Do not lose an event that arrived between the first generation
|
|
1485
1510
|
// check and registration of this wait.
|
|
@@ -1487,9 +1512,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1487
1512
|
resolveInterruptWait = null;
|
|
1488
1513
|
resolve();
|
|
1489
1514
|
}
|
|
1490
|
-
})
|
|
1491
|
-
|
|
1492
|
-
|
|
1515
|
+
}));
|
|
1516
|
+
}
|
|
1517
|
+
await Promise.race(waits);
|
|
1493
1518
|
}
|
|
1494
1519
|
finally {
|
|
1495
1520
|
clearInterval(heartbeatTimer);
|
|
@@ -1576,7 +1601,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1576
1601
|
let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
|
|
1577
1602
|
const runCoordinatorWork = async (work, task, options = {}) => {
|
|
1578
1603
|
const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
|
|
1579
|
-
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
|
|
1604
|
+
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr, monotonicNow)
|
|
1580
1605
|
: undefined;
|
|
1581
1606
|
if (!coordinator) {
|
|
1582
1607
|
return {
|
|
@@ -1605,10 +1630,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1605
1630
|
const outcome = await coordinator.submit(work);
|
|
1606
1631
|
const executorSucceeded = execution.result !== undefined &&
|
|
1607
1632
|
coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
|
|
1608
|
-
// A refusal
|
|
1609
|
-
//
|
|
1610
|
-
//
|
|
1611
|
-
|
|
1633
|
+
// A receiver refusal belongs to a newer replacement pull. Even when
|
|
1634
|
+
// this slice durably completed its other paths, the SQS receipt batch
|
|
1635
|
+
// cannot be acknowledged until every path is durable. Watcher work
|
|
1636
|
+
// retains its subset-publish contract and filters refused paths below.
|
|
1637
|
+
const requireEveryPathDurable = work.class === "realtime" &&
|
|
1638
|
+
work.source === "receiver" &&
|
|
1639
|
+
work.direction === "pull";
|
|
1640
|
+
const accepted = executorSucceeded && outcome.paths.every((path) => requireEveryPathDurable
|
|
1641
|
+
? path.status === "durable"
|
|
1642
|
+
: path.status === "durable" || path.status === "refused");
|
|
1612
1643
|
if (telemetry) {
|
|
1613
1644
|
const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
|
|
1614
1645
|
? "durable"
|
|
@@ -1638,16 +1669,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1638
1669
|
}
|
|
1639
1670
|
};
|
|
1640
1671
|
let nextCoordinatorCausality = 0;
|
|
1641
|
-
const realtimeWork = (source, direction, paths, lane) => ({
|
|
1672
|
+
const realtimeWork = (source, direction, paths, lane, sequenceNumberByPath) => ({
|
|
1642
1673
|
class: "realtime",
|
|
1643
1674
|
source,
|
|
1644
1675
|
direction,
|
|
1645
1676
|
lane,
|
|
1646
1677
|
paths: normalizeCoordinatorPaths(paths),
|
|
1678
|
+
...(sequenceNumberByPath === undefined ? {} : { sequenceNumberByPath }),
|
|
1647
1679
|
causality: `${source}-${++nextCoordinatorCausality}`,
|
|
1648
1680
|
});
|
|
1649
1681
|
const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
|
|
1650
|
-
const runRealtimePass = (source, direction, paths, lane, task) => runCoordinatorWork(realtimeWork(source, direction, paths, lane), task);
|
|
1682
|
+
const runRealtimePass = (source, direction, paths, lane, task, sequenceNumberByPath) => runCoordinatorWork(realtimeWork(source, direction, paths, lane, sequenceNumberByPath), task);
|
|
1651
1683
|
const enqueueChore = (kind, task, onSettled, options = { wholeMutation: true }) => {
|
|
1652
1684
|
void runCoordinatorWork(choreWork(kind), task, options).then(onSettled).catch((error) => {
|
|
1653
1685
|
process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
|
|
@@ -3066,7 +3098,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3066
3098
|
// Submit every route in this lane before awaiting any result. In
|
|
3067
3099
|
// particular, a later route's fast receipt must never wait for an
|
|
3068
3100
|
// earlier route's slow pull to finish.
|
|
3069
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced))));
|
|
3101
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced), new Map(events.map((event) => [event.relativePath, event.sequenceNumber])))));
|
|
3070
3102
|
for (const execution of completed) {
|
|
3071
3103
|
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
3072
3104
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
@@ -3218,11 +3250,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3218
3250
|
authToken: getAccessToken,
|
|
3219
3251
|
region: runtime.region,
|
|
3220
3252
|
});
|
|
3253
|
+
const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
|
|
3221
3254
|
const handles = await startES({
|
|
3222
3255
|
hqRoot,
|
|
3223
3256
|
apiUrl: runtime.apiUrl,
|
|
3224
3257
|
authToken: getAccessToken,
|
|
3225
|
-
deviceId:
|
|
3258
|
+
deviceId: deviceIdentity.machineId,
|
|
3259
|
+
deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
|
|
3260
|
+
deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
|
|
3226
3261
|
resolveTenantId: async () => {
|
|
3227
3262
|
const persons = await eventSyncClient.entity.listByType("person");
|
|
3228
3263
|
const pick = pickCanonicalPersonEntity(persons);
|
|
@@ -3525,6 +3560,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3525
3560
|
// The initial reconcile is due immediately. On-time passes advance from this
|
|
3526
3561
|
// deadline; overruns restart their cooldown from completion.
|
|
3527
3562
|
let nextDueAt = monotonicNow();
|
|
3563
|
+
const passExitRetryBudget = new WatchPassExitRetryBudget();
|
|
3564
|
+
let passExitRetryNotBefore = null;
|
|
3565
|
+
const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
|
|
3566
|
+
const delayMs = passExitRetryBudget.record(exitCode, now);
|
|
3567
|
+
passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
|
|
3568
|
+
if (delayMs > 0) {
|
|
3569
|
+
process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
|
|
3570
|
+
}
|
|
3571
|
+
};
|
|
3528
3572
|
let pollTick = 0;
|
|
3529
3573
|
let lastCompletedFullReconcileAt = null;
|
|
3530
3574
|
// Anchor for the elapsed-time cadence: the most recent full reconcile that
|
|
@@ -3554,6 +3598,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3554
3598
|
return RUNNER_HEAP_RECYCLE_EXIT;
|
|
3555
3599
|
if (coordinatorAuthRequired)
|
|
3556
3600
|
return runtime.authRequiredPassExit;
|
|
3601
|
+
if (passExitRetryNotBefore !== null) {
|
|
3602
|
+
const retryNow = monotonicNow();
|
|
3603
|
+
if (retryNow < passExitRetryNotBefore) {
|
|
3604
|
+
// A watcher/receiver interrupt must not turn a known failing pass
|
|
3605
|
+
// back into a tight loop. Pending work remains queued and runs when
|
|
3606
|
+
// this non-interruptible cooldown has elapsed.
|
|
3607
|
+
await waitForNextPass(passExitRetryNotBefore - retryNow, false);
|
|
3608
|
+
continue;
|
|
3609
|
+
}
|
|
3610
|
+
passExitRetryNotBefore = null;
|
|
3611
|
+
}
|
|
3557
3612
|
pollTick++;
|
|
3558
3613
|
areaAutoMigration.onTick();
|
|
3559
3614
|
const pending = takePendingWatcherChange();
|
|
@@ -3585,11 +3640,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3585
3640
|
const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
|
|
3586
3641
|
const coverageStale = rootCoverageStale;
|
|
3587
3642
|
rootCoverageStale = false;
|
|
3588
|
-
const scheduledInterval =
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
cadenceNow !== null &&
|
|
3592
|
-
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3643
|
+
const scheduledInterval = lastFullReconcileAt !== null &&
|
|
3644
|
+
cadenceNow !== null &&
|
|
3645
|
+
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3593
3646
|
const triggers = {
|
|
3594
3647
|
"event-push-surfaces-inactive": eventPushSurfacesInactive,
|
|
3595
3648
|
"first-tick": firstTick,
|
|
@@ -3649,6 +3702,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3649
3702
|
};
|
|
3650
3703
|
let result;
|
|
3651
3704
|
let ranFullReconcile = false;
|
|
3705
|
+
let ranWatchPass = false;
|
|
3652
3706
|
let enqueuedFullChore = false;
|
|
3653
3707
|
if (shouldRunFull) {
|
|
3654
3708
|
emitDecision("full", reasons);
|
|
@@ -3668,6 +3722,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3668
3722
|
interruptWait();
|
|
3669
3723
|
return;
|
|
3670
3724
|
}
|
|
3725
|
+
recordWatchPassExit(exit);
|
|
3726
|
+
if (exit !== 0)
|
|
3727
|
+
interruptWait();
|
|
3671
3728
|
// Preserve the numeric success seam accepted by the prior
|
|
3672
3729
|
// coordinator while also accepting a completed RunnerPassOutcome
|
|
3673
3730
|
// (including a partial per-company pass) that proves journal
|
|
@@ -3712,6 +3769,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3712
3769
|
}
|
|
3713
3770
|
else {
|
|
3714
3771
|
ranFullReconcile = true;
|
|
3772
|
+
ranWatchPass = true;
|
|
3715
3773
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3716
3774
|
}
|
|
3717
3775
|
}
|
|
@@ -3741,6 +3799,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3741
3799
|
result = 0;
|
|
3742
3800
|
}
|
|
3743
3801
|
else {
|
|
3802
|
+
ranWatchPass = true;
|
|
3744
3803
|
result = overflowWithKnownRoutes
|
|
3745
3804
|
? await runOverflowRouteDrain(workingBatch)
|
|
3746
3805
|
: await runScopedDrain(workingBatch, pending.generation);
|
|
@@ -3758,6 +3817,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3758
3817
|
process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
|
|
3759
3818
|
`falling back to a full reconcile\n`);
|
|
3760
3819
|
ranFullReconcile = true;
|
|
3820
|
+
ranWatchPass = true;
|
|
3761
3821
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3762
3822
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3763
3823
|
}
|
|
@@ -3903,6 +3963,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3903
3963
|
realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
|
|
3904
3964
|
}
|
|
3905
3965
|
const now = monotonicNow();
|
|
3966
|
+
if (ranWatchPass)
|
|
3967
|
+
recordWatchPassExit(exitCode, now);
|
|
3906
3968
|
if (ranFullReconcile) {
|
|
3907
3969
|
// The awaited full pass finished on this tick; reuse the tick's clock
|
|
3908
3970
|
// sample so the cadence anchor never costs a second read.
|
|
@@ -3922,10 +3984,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3922
3984
|
? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
|
|
3923
3985
|
: advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
|
|
3924
3986
|
nextDueAt = schedule.nextDueAt;
|
|
3987
|
+
const retryDelayMs = passExitRetryNotBefore === null
|
|
3988
|
+
? 0
|
|
3989
|
+
: Math.max(0, passExitRetryNotBefore - now);
|
|
3990
|
+
const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
|
|
3925
3991
|
const untilHighWaterMs = realtimeState.nextHighWaterAt === null
|
|
3926
|
-
?
|
|
3992
|
+
? nextPassDelayMs
|
|
3927
3993
|
: Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
|
|
3928
|
-
await waitForNextPass(Math.min(
|
|
3994
|
+
await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
|
|
3929
3995
|
}
|
|
3930
3996
|
return 0;
|
|
3931
3997
|
}
|