@indigoai-us/hq-cloud 6.16.74 → 6.16.75-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +24 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +88 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +185 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex-knowledge.d.ts.map +1 -1
- package/dist/cli/reindex-knowledge.js +13 -8
- package/dist/cli/reindex-knowledge.js.map +1 -1
- package/package.json +5 -3
|
@@ -260,6 +260,16 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
|
|
|
260
260
|
export declare const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
261
261
|
export declare const DEFAULT_FULL_RECONCILE_MS: number | undefined;
|
|
262
262
|
export declare function resolveFullReconcileMs(env?: Record<string, string | undefined>): number | undefined;
|
|
263
|
+
/**
|
|
264
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
265
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
266
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
267
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
268
|
+
*/
|
|
269
|
+
export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
|
|
270
|
+
export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
271
|
+
export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
|
|
272
|
+
export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
|
|
263
273
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
264
274
|
export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
|
|
265
275
|
/** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
|
|
@@ -290,6 +300,20 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
|
|
|
290
300
|
nextDueAt: number;
|
|
291
301
|
delayMs: number;
|
|
292
302
|
};
|
|
303
|
+
/**
|
|
304
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
305
|
+
*
|
|
306
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
307
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
308
|
+
* short poll from continuously re-running a pass that is already known to
|
|
309
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
310
|
+
* not need wall-clock waits.
|
|
311
|
+
*/
|
|
312
|
+
export declare class WatchPassExitRetryBudget {
|
|
313
|
+
private lastFailureAt;
|
|
314
|
+
private failures;
|
|
315
|
+
record(exitCode: number, now: number): number;
|
|
316
|
+
}
|
|
293
317
|
export interface ParsedLoopArgs {
|
|
294
318
|
hqRoot: string;
|
|
295
319
|
lockTimeoutSec?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAG1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAuB,CAAC;AAEzE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAeD;;;;;GAKG;AACH,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAC3D,eAAO,MAAM,uCAAuC,IAAI,CAAC;AACzD,eAAO,MAAM,gCAAgC,QAAS,CAAC;AACvD,eAAO,MAAM,4BAA4B,QAAc,CAAC;AAGxD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAK;IAErB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;CAgC9C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA63GjB"}
|
|
@@ -900,6 +900,16 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
900
900
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
901
901
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
902
902
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
903
|
+
/**
|
|
904
|
+
* A failed watch pass is still retried in-process, but even an explicitly
|
|
905
|
+
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
906
|
+
* Three failures in thirty minutes escalate from this floor to exponential
|
|
907
|
+
* backoff; success or a quiet window starts a fresh budget.
|
|
908
|
+
*/
|
|
909
|
+
export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
|
|
910
|
+
export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
911
|
+
export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
|
|
912
|
+
export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
|
|
903
913
|
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
904
914
|
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
905
915
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
@@ -949,6 +959,41 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
949
959
|
nextDueAt = now + intervalMs;
|
|
950
960
|
return { nextDueAt, delayMs: intervalMs };
|
|
951
961
|
}
|
|
962
|
+
/**
|
|
963
|
+
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
964
|
+
*
|
|
965
|
+
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
966
|
+
* controls normal remote work, whereas this budget prevents a configured
|
|
967
|
+
* short poll from continuously re-running a pass that is already known to
|
|
968
|
+
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
969
|
+
* not need wall-clock waits.
|
|
970
|
+
*/
|
|
971
|
+
export class WatchPassExitRetryBudget {
|
|
972
|
+
lastFailureAt = null;
|
|
973
|
+
failures = 0;
|
|
974
|
+
record(exitCode, now) {
|
|
975
|
+
if (exitCode === 0) {
|
|
976
|
+
this.lastFailureAt = null;
|
|
977
|
+
this.failures = 0;
|
|
978
|
+
return 0;
|
|
979
|
+
}
|
|
980
|
+
if (this.lastFailureAt === null ||
|
|
981
|
+
now < this.lastFailureAt ||
|
|
982
|
+
now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
|
|
983
|
+
this.lastFailureAt = now;
|
|
984
|
+
this.failures = 0;
|
|
985
|
+
}
|
|
986
|
+
this.failures += 1;
|
|
987
|
+
this.lastFailureAt = now;
|
|
988
|
+
if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
|
|
989
|
+
// Preserve the established normal-cadence retry for isolated failures.
|
|
990
|
+
// The bounded cooldown starts only after repeated exits in the window.
|
|
991
|
+
return 0;
|
|
992
|
+
}
|
|
993
|
+
const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
|
|
994
|
+
return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
952
997
|
/**
|
|
953
998
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
954
999
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1461,8 +1506,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1461
1506
|
resolveInterruptWait = null;
|
|
1462
1507
|
resolve?.();
|
|
1463
1508
|
};
|
|
1464
|
-
const waitForNextPass = async (delayMs) => {
|
|
1465
|
-
if (interruptGeneration !== observedInterruptGeneration) {
|
|
1509
|
+
const waitForNextPass = async (delayMs, interruptible = true) => {
|
|
1510
|
+
if (interruptible && interruptGeneration !== observedInterruptGeneration) {
|
|
1466
1511
|
observedInterruptGeneration = interruptGeneration;
|
|
1467
1512
|
return;
|
|
1468
1513
|
}
|
|
@@ -1477,9 +1522,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1477
1522
|
}, idleHeartbeatIntervalMs);
|
|
1478
1523
|
heartbeatTimer.unref?.();
|
|
1479
1524
|
try {
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
new Promise((resolve) => {
|
|
1525
|
+
const waits = [sleep(delayMs), stoppedSignal];
|
|
1526
|
+
if (interruptible) {
|
|
1527
|
+
waits.push(new Promise((resolve) => {
|
|
1483
1528
|
resolveInterruptWait = resolve;
|
|
1484
1529
|
// Do not lose an event that arrived between the first generation
|
|
1485
1530
|
// check and registration of this wait.
|
|
@@ -1487,9 +1532,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1487
1532
|
resolveInterruptWait = null;
|
|
1488
1533
|
resolve();
|
|
1489
1534
|
}
|
|
1490
|
-
})
|
|
1491
|
-
|
|
1492
|
-
|
|
1535
|
+
}));
|
|
1536
|
+
}
|
|
1537
|
+
await Promise.race(waits);
|
|
1493
1538
|
}
|
|
1494
1539
|
finally {
|
|
1495
1540
|
clearInterval(heartbeatTimer);
|
|
@@ -3513,6 +3558,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3513
3558
|
// The initial reconcile is due immediately. On-time passes advance from this
|
|
3514
3559
|
// deadline; overruns restart their cooldown from completion.
|
|
3515
3560
|
let nextDueAt = monotonicNow();
|
|
3561
|
+
const passExitRetryBudget = new WatchPassExitRetryBudget();
|
|
3562
|
+
let passExitRetryNotBefore = null;
|
|
3563
|
+
const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
|
|
3564
|
+
const delayMs = passExitRetryBudget.record(exitCode, now);
|
|
3565
|
+
passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
|
|
3566
|
+
if (delayMs > 0) {
|
|
3567
|
+
process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
|
|
3568
|
+
}
|
|
3569
|
+
};
|
|
3516
3570
|
let pollTick = 0;
|
|
3517
3571
|
let lastCompletedFullReconcileAt = null;
|
|
3518
3572
|
// Anchor for the elapsed-time cadence: the most recent full reconcile that
|
|
@@ -3542,6 +3596,17 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3542
3596
|
return RUNNER_HEAP_RECYCLE_EXIT;
|
|
3543
3597
|
if (coordinatorAuthRequired)
|
|
3544
3598
|
return runtime.authRequiredPassExit;
|
|
3599
|
+
if (passExitRetryNotBefore !== null) {
|
|
3600
|
+
const retryNow = monotonicNow();
|
|
3601
|
+
if (retryNow < passExitRetryNotBefore) {
|
|
3602
|
+
// A watcher/receiver interrupt must not turn a known failing pass
|
|
3603
|
+
// back into a tight loop. Pending work remains queued and runs when
|
|
3604
|
+
// this non-interruptible cooldown has elapsed.
|
|
3605
|
+
await waitForNextPass(passExitRetryNotBefore - retryNow, false);
|
|
3606
|
+
continue;
|
|
3607
|
+
}
|
|
3608
|
+
passExitRetryNotBefore = null;
|
|
3609
|
+
}
|
|
3545
3610
|
pollTick++;
|
|
3546
3611
|
areaAutoMigration.onTick();
|
|
3547
3612
|
const pending = takePendingWatcherChange();
|
|
@@ -3637,6 +3702,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3637
3702
|
};
|
|
3638
3703
|
let result;
|
|
3639
3704
|
let ranFullReconcile = false;
|
|
3705
|
+
let ranWatchPass = false;
|
|
3640
3706
|
let enqueuedFullChore = false;
|
|
3641
3707
|
if (shouldRunFull) {
|
|
3642
3708
|
emitDecision("full", reasons);
|
|
@@ -3656,6 +3722,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3656
3722
|
interruptWait();
|
|
3657
3723
|
return;
|
|
3658
3724
|
}
|
|
3725
|
+
recordWatchPassExit(exit);
|
|
3726
|
+
if (exit !== 0)
|
|
3727
|
+
interruptWait();
|
|
3659
3728
|
// Preserve the numeric success seam accepted by the prior
|
|
3660
3729
|
// coordinator while also accepting a completed RunnerPassOutcome
|
|
3661
3730
|
// (including a partial per-company pass) that proves journal
|
|
@@ -3700,6 +3769,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3700
3769
|
}
|
|
3701
3770
|
else {
|
|
3702
3771
|
ranFullReconcile = true;
|
|
3772
|
+
ranWatchPass = true;
|
|
3703
3773
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3704
3774
|
}
|
|
3705
3775
|
}
|
|
@@ -3729,6 +3799,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3729
3799
|
result = 0;
|
|
3730
3800
|
}
|
|
3731
3801
|
else {
|
|
3802
|
+
ranWatchPass = true;
|
|
3732
3803
|
result = overflowWithKnownRoutes
|
|
3733
3804
|
? await runOverflowRouteDrain(workingBatch)
|
|
3734
3805
|
: await runScopedDrain(workingBatch, pending.generation);
|
|
@@ -3746,6 +3817,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3746
3817
|
process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
|
|
3747
3818
|
`falling back to a full reconcile\n`);
|
|
3748
3819
|
ranFullReconcile = true;
|
|
3820
|
+
ranWatchPass = true;
|
|
3749
3821
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3750
3822
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3751
3823
|
}
|
|
@@ -3891,6 +3963,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3891
3963
|
realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
|
|
3892
3964
|
}
|
|
3893
3965
|
const now = monotonicNow();
|
|
3966
|
+
if (ranWatchPass)
|
|
3967
|
+
recordWatchPassExit(exitCode, now);
|
|
3894
3968
|
if (ranFullReconcile) {
|
|
3895
3969
|
// The awaited full pass finished on this tick; reuse the tick's clock
|
|
3896
3970
|
// sample so the cadence anchor never costs a second read.
|
|
@@ -3910,10 +3984,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3910
3984
|
? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
|
|
3911
3985
|
: advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
|
|
3912
3986
|
nextDueAt = schedule.nextDueAt;
|
|
3987
|
+
const retryDelayMs = passExitRetryNotBefore === null
|
|
3988
|
+
? 0
|
|
3989
|
+
: Math.max(0, passExitRetryNotBefore - now);
|
|
3990
|
+
const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
|
|
3913
3991
|
const untilHighWaterMs = realtimeState.nextHighWaterAt === null
|
|
3914
|
-
?
|
|
3992
|
+
? nextPassDelayMs
|
|
3915
3993
|
: Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
|
|
3916
|
-
await waitForNextPass(Math.min(
|
|
3994
|
+
await waitForNextPass(Math.min(nextPassDelayMs, untilHighWaterMs));
|
|
3917
3995
|
}
|
|
3918
3996
|
return 0;
|
|
3919
3997
|
}
|