@indigoai-us/hq-cloud 6.16.82-alpha.0 → 6.16.84-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +118 -71
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +507 -248
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +50 -9
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +224 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +78 -27
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +107 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +15 -7
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +148 -15
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +321 -2
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/full-pass-interval.d.ts +25 -0
- package/dist/sync/full-pass-interval.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.js +53 -0
- package/dist/sync/full-pass-interval.js.map +1 -0
- package/dist/sync/full-pass-interval.test.d.ts +2 -0
- package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.test.js +21 -0
- package/dist/sync/full-pass-interval.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +0 -14
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +12 -49
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +7 -171
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -240,26 +240,15 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
|
|
|
240
240
|
remainder: TreeChangeBatch | null;
|
|
241
241
|
};
|
|
242
242
|
/**
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
* hour.
|
|
248
|
-
*
|
|
249
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
250
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
251
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
252
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
253
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
254
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
255
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
256
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
257
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
258
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
243
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
244
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
245
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
246
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
259
247
|
*/
|
|
260
|
-
export declare const
|
|
261
|
-
export declare const
|
|
262
|
-
export declare
|
|
248
|
+
export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
|
|
249
|
+
export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
250
|
+
export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
|
|
251
|
+
export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
|
|
263
252
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
264
253
|
export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
|
|
265
254
|
/** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
|
|
@@ -290,6 +279,20 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
|
|
|
290
279
|
nextDueAt: number;
|
|
291
280
|
delayMs: number;
|
|
292
281
|
};
|
|
282
|
+
/**
|
|
283
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
284
|
+
*
|
|
285
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
286
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
287
|
+
* short poll from continuously re-running a pass that is already known to
|
|
288
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
289
|
+
* not need wall-clock waits.
|
|
290
|
+
*/
|
|
291
|
+
export declare class WatchPassExitRetryBudget {
|
|
292
|
+
private lastFailureAt;
|
|
293
|
+
private failures;
|
|
294
|
+
record(exitCode: number, now: number): number;
|
|
295
|
+
}
|
|
293
296
|
export interface ParsedLoopArgs {
|
|
294
297
|
hqRoot: string;
|
|
295
298
|
lockTimeoutSec?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAE1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAeD;;;;;GAKG;AACH,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,eAAO,MAAM,uCAAuC,IAAI,CAAC;AACzD,eAAO,MAAM,gCAAgC,QAAS,CAAC;AACvD,eAAO,MAAM,4BAA4B,QAAc,CAAC;AAGxD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAK;IAErB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;CAgC9C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA04GjB"}
|
|
@@ -11,7 +11,7 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
|
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
13
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
|
-
import {
|
|
14
|
+
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
@@ -27,12 +27,12 @@ import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPre
|
|
|
27
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
28
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
29
|
import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
|
|
30
|
+
import { createHqFlagsIntervalSource, } from "../sync/full-pass-interval.js";
|
|
30
31
|
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
31
32
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
32
33
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
33
34
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
34
35
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
35
|
-
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
36
36
|
/** A deferred full chore must get the next admission after this long. */
|
|
37
37
|
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
38
38
|
/**
|
|
@@ -857,36 +857,6 @@ export function takeEnumeratedBatchChunk(batch, limit) {
|
|
|
857
857
|
remainder: asBatch(remainderPaths, splitChanges(remainderPaths)),
|
|
858
858
|
};
|
|
859
859
|
}
|
|
860
|
-
/**
|
|
861
|
-
* Elapsed-time cadence for scheduled full reconciles: one hour by default,
|
|
862
|
-
* measured from the last full pass (its dispatch or its completion, whichever
|
|
863
|
-
* is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
|
|
864
|
-
* 10_800_000 for three hours); an unset or invalid value falls back to the
|
|
865
|
-
* hour.
|
|
866
|
-
*
|
|
867
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
868
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
869
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
870
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
871
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
872
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
873
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
874
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
875
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
876
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
877
|
-
*/
|
|
878
|
-
export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
879
|
-
export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
|
|
880
|
-
export function resolveFullReconcileMs(env = process.env) {
|
|
881
|
-
const raw = env[FULL_RECONCILE_MS_ENV];
|
|
882
|
-
if (raw === undefined || raw.trim() === "")
|
|
883
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
884
|
-
const parsed = Number(raw);
|
|
885
|
-
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
886
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
887
|
-
const intervalMs = Math.trunc(parsed);
|
|
888
|
-
return intervalMs > 0 ? intervalMs : DEFAULT_FULL_RECONCILE_MS;
|
|
889
|
-
}
|
|
890
860
|
/**
|
|
891
861
|
* Adaptive remote-poll backoff bounds (whole-box CPU aware). When
|
|
892
862
|
* `--poll-remote-ms` is NOT passed, the watch loop scales its poll cadence to
|
|
@@ -900,6 +870,16 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
900
870
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
901
871
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
902
872
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
873
|
+
/**
|
|
874
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
875
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
876
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
877
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
878
|
+
*/
|
|
879
|
+
export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
|
|
880
|
+
export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
881
|
+
export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
|
|
882
|
+
export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
|
|
903
883
|
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
904
884
|
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
905
885
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
@@ -949,6 +929,41 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
949
929
|
nextDueAt = now + intervalMs;
|
|
950
930
|
return { nextDueAt, delayMs: intervalMs };
|
|
951
931
|
}
|
|
932
|
+
/**
|
|
933
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
934
|
+
*
|
|
935
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
936
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
937
|
+
* short poll from continuously re-running a pass that is already known to
|
|
938
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
939
|
+
* not need wall-clock waits.
|
|
940
|
+
*/
|
|
941
|
+
export class WatchPassExitRetryBudget {
|
|
942
|
+
lastFailureAt = null;
|
|
943
|
+
failures = 0;
|
|
944
|
+
record(exitCode, now) {
|
|
945
|
+
if (exitCode === 0) {
|
|
946
|
+
this.lastFailureAt = null;
|
|
947
|
+
this.failures = 0;
|
|
948
|
+
return 0;
|
|
949
|
+
}
|
|
950
|
+
if (this.lastFailureAt === null ||
|
|
951
|
+
now < this.lastFailureAt ||
|
|
952
|
+
now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
|
|
953
|
+
this.lastFailureAt = now;
|
|
954
|
+
this.failures = 0;
|
|
955
|
+
}
|
|
956
|
+
this.failures += 1;
|
|
957
|
+
this.lastFailureAt = now;
|
|
958
|
+
if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
|
|
959
|
+
// Preserve the established normal-cadence retry for isolated failures.
|
|
960
|
+
// The bounded cooldown starts only after repeated exits in the window.
|
|
961
|
+
return 0;
|
|
962
|
+
}
|
|
963
|
+
const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
|
|
964
|
+
return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
952
967
|
/**
|
|
953
968
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
954
969
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1070,7 +1085,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1070
1085
|
const sampleLoadAvg = deps.sampleLoadAvg ?? (() => os.loadavg()[0]);
|
|
1071
1086
|
const cpuCount = os.cpus().length;
|
|
1072
1087
|
const eventPush = argv.includes("--event-push");
|
|
1073
|
-
const
|
|
1088
|
+
const fullPassIntervalSource = deps.fullPassIntervalSource ??
|
|
1089
|
+
createHqFlagsIntervalSource({
|
|
1090
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
1091
|
+
getToken: runtime.defaultGetAccessToken,
|
|
1092
|
+
});
|
|
1093
|
+
let fullReconcileIntervalMs = fullPassIntervalSource.read();
|
|
1074
1094
|
const companiesMode = argv.includes("--companies");
|
|
1075
1095
|
const companyIdx = argv.indexOf("--company");
|
|
1076
1096
|
const companyMode = companyIdx >= 0;
|
|
@@ -1240,11 +1260,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1240
1260
|
flags: flagSource,
|
|
1241
1261
|
platform: process.platform,
|
|
1242
1262
|
});
|
|
1243
|
-
if (
|
|
1263
|
+
if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
1244
1264
|
const refresh = setTimeout(() => {
|
|
1245
|
-
void
|
|
1246
|
-
|
|
1247
|
-
|
|
1265
|
+
void Promise.all([
|
|
1266
|
+
deps.fullPassCaps ? Promise.resolve(null) : flagSource.refresh?.(),
|
|
1267
|
+
deps.fullPassIntervalSource ? Promise.resolve(null) : fullPassIntervalSource.refresh?.(),
|
|
1268
|
+
]).then(([capFlags, intervalMs]) => {
|
|
1269
|
+
if (capFlags)
|
|
1270
|
+
fullPassCaps.applyFlags(capFlags);
|
|
1271
|
+
if (intervalMs)
|
|
1272
|
+
fullReconcileIntervalMs = intervalMs;
|
|
1248
1273
|
});
|
|
1249
1274
|
}, 0);
|
|
1250
1275
|
refresh.unref?.();
|
|
@@ -1461,8 +1486,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1461
1486
|
resolveInterruptWait = null;
|
|
1462
1487
|
resolve?.();
|
|
1463
1488
|
};
|
|
1464
|
-
const waitForNextPass = async (delayMs) => {
|
|
1465
|
-
if (interruptGeneration !== observedInterruptGeneration) {
|
|
1489
|
+
const waitForNextPass = async (delayMs, interruptible = true) => {
|
|
1490
|
+
if (interruptible && interruptGeneration !== observedInterruptGeneration) {
|
|
1466
1491
|
observedInterruptGeneration = interruptGeneration;
|
|
1467
1492
|
return;
|
|
1468
1493
|
}
|
|
@@ -1477,9 +1502,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1477
1502
|
}, idleHeartbeatIntervalMs);
|
|
1478
1503
|
heartbeatTimer.unref?.();
|
|
1479
1504
|
try {
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
new Promise((resolve) => {
|
|
1505
|
+
const waits = [sleep(delayMs), stoppedSignal];
|
|
1506
|
+
if (interruptible) {
|
|
1507
|
+
waits.push(new Promise((resolve) => {
|
|
1483
1508
|
resolveInterruptWait = resolve;
|
|
1484
1509
|
// Do not lose an event that arrived between the first generation
|
|
1485
1510
|
// check and registration of this wait.
|
|
@@ -1487,9 +1512,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1487
1512
|
resolveInterruptWait = null;
|
|
1488
1513
|
resolve();
|
|
1489
1514
|
}
|
|
1490
|
-
})
|
|
1491
|
-
|
|
1492
|
-
|
|
1515
|
+
}));
|
|
1516
|
+
}
|
|
1517
|
+
await Promise.race(waits);
|
|
1493
1518
|
}
|
|
1494
1519
|
finally {
|
|
1495
1520
|
clearInterval(heartbeatTimer);
|
|
@@ -1576,7 +1601,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1576
1601
|
let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
|
|
1577
1602
|
const runCoordinatorWork = async (work, task, options = {}) => {
|
|
1578
1603
|
const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
|
|
1579
|
-
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
|
|
1604
|
+
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr, monotonicNow)
|
|
1580
1605
|
: undefined;
|
|
1581
1606
|
if (!coordinator) {
|
|
1582
1607
|
return {
|
|
@@ -1605,10 +1630,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1605
1630
|
const outcome = await coordinator.submit(work);
|
|
1606
1631
|
const executorSucceeded = execution.result !== undefined &&
|
|
1607
1632
|
coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1633
|
+
const allRefused = outcome.paths.length > 0 && outcome.paths.every((path) => path.status === "refused");
|
|
1634
|
+
const accepted = executorSucceeded
|
|
1635
|
+
? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
|
|
1636
|
+
: allRefused;
|
|
1612
1637
|
if (telemetry) {
|
|
1613
1638
|
const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
|
|
1614
1639
|
? "durable"
|
|
@@ -1618,7 +1643,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1618
1643
|
telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
|
|
1619
1644
|
}
|
|
1620
1645
|
return {
|
|
1621
|
-
result: execution.result ?? 1,
|
|
1646
|
+
result: execution.result ?? (allRefused ? 0 : 1),
|
|
1622
1647
|
accepted,
|
|
1623
1648
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
1624
1649
|
.filter((path) => path.status === "refused")
|
|
@@ -3218,11 +3243,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3218
3243
|
authToken: getAccessToken,
|
|
3219
3244
|
region: runtime.region,
|
|
3220
3245
|
});
|
|
3246
|
+
const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
|
|
3221
3247
|
const handles = await startES({
|
|
3222
3248
|
hqRoot,
|
|
3223
3249
|
apiUrl: runtime.apiUrl,
|
|
3224
3250
|
authToken: getAccessToken,
|
|
3225
|
-
deviceId:
|
|
3251
|
+
deviceId: deviceIdentity.machineId,
|
|
3252
|
+
deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
|
|
3253
|
+
deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
|
|
3226
3254
|
resolveTenantId: async () => {
|
|
3227
3255
|
const persons = await eventSyncClient.entity.listByType("person");
|
|
3228
3256
|
const pick = pickCanonicalPersonEntity(persons);
|
|
@@ -3238,18 +3266,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3238
3266
|
publishedContentHashStateDir: getStateDir(),
|
|
3239
3267
|
publishedContentHashLookup,
|
|
3240
3268
|
onReceiveActivity,
|
|
3241
|
-
onReceiptOutcome: (receipt) => {
|
|
3242
|
-
// This is a bounded diagnostic trace, deliberately separate from
|
|
3243
|
-
// low-cardinality metrics. It includes only the irreversible hash
|
|
3244
|
-
// of the event path and the closed receipt outcome.
|
|
3245
|
-
process.stderr.write(`${JSON.stringify({
|
|
3246
|
-
event: "sync.realtime.receipt",
|
|
3247
|
-
schemaVersion: 1,
|
|
3248
|
-
source: "receiver",
|
|
3249
|
-
direction: "pull",
|
|
3250
|
-
...receipt,
|
|
3251
|
-
})}\n`);
|
|
3252
|
-
},
|
|
3253
3269
|
});
|
|
3254
3270
|
if (!handles)
|
|
3255
3271
|
return;
|
|
@@ -3525,6 +3541,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3525
3541
|
// The initial reconcile is due immediately. On-time passes advance from this
|
|
3526
3542
|
// deadline; overruns restart their cooldown from completion.
|
|
3527
3543
|
let nextDueAt = monotonicNow();
|
|
3544
|
+
const passExitRetryBudget = new WatchPassExitRetryBudget();
|
|
3545
|
+
let passExitRetryNotBefore = null;
|
|
3546
|
+
const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
|
|
3547
|
+
const delayMs = passExitRetryBudget.record(exitCode, now);
|
|
3548
|
+
passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
|
|
3549
|
+
if (delayMs > 0) {
|
|
3550
|
+
process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
|
|
3551
|
+
}
|
|
3552
|
+
};
|
|
3528
3553
|
let pollTick = 0;
|
|
3529
3554
|
let lastCompletedFullReconcileAt = null;
|
|
3530
3555
|
// Anchor for the elapsed-time cadence: the most recent full reconcile that
|
|
@@ -3554,6 +3579,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3554
3579
|
return RUNNER_HEAP_RECYCLE_EXIT;
|
|
3555
3580
|
if (coordinatorAuthRequired)
|
|
3556
3581
|
return runtime.authRequiredPassExit;
|
|
3582
|
+
if (passExitRetryNotBefore !== null) {
|
|
3583
|
+
const retryNow = monotonicNow();
|
|
3584
|
+
if (retryNow < passExitRetryNotBefore) {
|
|
3585
|
+
// A watcher/receiver interrupt must not turn a known failing pass
|
|
3586
|
+
// back into a tight loop. Pending work remains queued and runs when
|
|
3587
|
+
// this non-interruptible cooldown has elapsed.
|
|
3588
|
+
await waitForNextPass(passExitRetryNotBefore - retryNow, false);
|
|
3589
|
+
continue;
|
|
3590
|
+
}
|
|
3591
|
+
passExitRetryNotBefore = null;
|
|
3592
|
+
}
|
|
3557
3593
|
pollTick++;
|
|
3558
3594
|
areaAutoMigration.onTick();
|
|
3559
3595
|
const pending = takePendingWatcherChange();
|
|
@@ -3585,11 +3621,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3585
3621
|
const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
|
|
3586
3622
|
const coverageStale = rootCoverageStale;
|
|
3587
3623
|
rootCoverageStale = false;
|
|
3588
|
-
const scheduledInterval =
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
cadenceNow !== null &&
|
|
3592
|
-
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3624
|
+
const scheduledInterval = lastFullReconcileAt !== null &&
|
|
3625
|
+
cadenceNow !== null &&
|
|
3626
|
+
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3593
3627
|
const triggers = {
|
|
3594
3628
|
"event-push-surfaces-inactive": eventPushSurfacesInactive,
|
|
3595
3629
|
"first-tick": firstTick,
|
|
@@ -3649,6 +3683,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3649
3683
|
};
|
|
3650
3684
|
let result;
|
|
3651
3685
|
let ranFullReconcile = false;
|
|
3686
|
+
let ranWatchPass = false;
|
|
3652
3687
|
let enqueuedFullChore = false;
|
|
3653
3688
|
if (shouldRunFull) {
|
|
3654
3689
|
emitDecision("full", reasons);
|
|
@@ -3668,6 +3703,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3668
3703
|
interruptWait();
|
|
3669
3704
|
return;
|
|
3670
3705
|
}
|
|
3706
|
+
recordWatchPassExit(exit);
|
|
3707
|
+
if (exit !== 0)
|
|
3708
|
+
interruptWait();
|
|
3671
3709
|
// Preserve the numeric success seam accepted by the prior
|
|
3672
3710
|
// coordinator while also accepting a completed RunnerPassOutcome
|
|
3673
3711
|
// (including a partial per-company pass) that proves journal
|
|
@@ -3712,6 +3750,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3712
3750
|
}
|
|
3713
3751
|
else {
|
|
3714
3752
|
ranFullReconcile = true;
|
|
3753
|
+
ranWatchPass = true;
|
|
3715
3754
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3716
3755
|
}
|
|
3717
3756
|
}
|
|
@@ -3741,6 +3780,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3741
3780
|
result = 0;
|
|
3742
3781
|
}
|
|
3743
3782
|
else {
|
|
3783
|
+
ranWatchPass = true;
|
|
3744
3784
|
result = overflowWithKnownRoutes
|
|
3745
3785
|
? await runOverflowRouteDrain(workingBatch)
|
|
3746
3786
|
: await runScopedDrain(workingBatch, pending.generation);
|
|
@@ -3758,6 +3798,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3758
3798
|
process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
|
|
3759
3799
|
`falling back to a full reconcile\n`);
|
|
3760
3800
|
ranFullReconcile = true;
|
|
3801
|
+
ranWatchPass = true;
|
|
3761
3802
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3762
3803
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3763
3804
|
}
|
|
@@ -3903,6 +3944,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3903
3944
|
realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
|
|
3904
3945
|
}
|
|
3905
3946
|
const now = monotonicNow();
|
|
3947
|
+
if (ranWatchPass)
|
|
3948
|
+
recordWatchPassExit(exitCode, now);
|
|
3906
3949
|
if (ranFullReconcile) {
|
|
3907
3950
|
// The awaited full pass finished on this tick; reuse the tick's clock
|
|
3908
3951
|
// sample so the cadence anchor never costs a second read.
|
|
@@ -3922,10 +3965,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3922
3965
|
? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
|
|
3923
3966
|
: advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
|
|
3924
3967
|
nextDueAt = schedule.nextDueAt;
|
|
3968
|
+
const retryDelayMs = passExitRetryNotBefore === null
|
|
3969
|
+
? 0
|
|
3970
|
+
: Math.max(0, passExitRetryNotBefore - now);
|
|
3971
|
+
const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
|
|
3925
3972
|
const untilHighWaterMs = realtimeState.nextHighWaterAt === null
|
|
3926
|
-
?
|
|
3973
|
+
? nextPassDelayMs
|
|
3927
3974
|
: Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
|
|
3928
|
-
await waitForNextPass(Math.min(
|
|
3975
|
+
await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
|
|
3929
3976
|
}
|
|
3930
3977
|
return 0;
|
|
3931
3978
|
}
|