@indigoai-us/hq-cloud 6.16.75-alpha.1 → 6.16.76-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +0 -45
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +29 -129
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +134 -431
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/sync/full-pass-interval.d.ts +25 -0
- package/dist/sync/full-pass-interval.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.js +53 -0
- package/dist/sync/full-pass-interval.js.map +1 -0
- package/dist/sync/full-pass-interval.test.d.ts +2 -0
- package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.test.js +21 -0
- package/dist/sync/full-pass-interval.test.js.map +1 -0
- package/package.json +1 -1
|
@@ -239,37 +239,6 @@ export declare function takeEnumeratedBatchChunk(batch: TreeChangeBatch, limit:
|
|
|
239
239
|
chunk: TreeChangeBatch;
|
|
240
240
|
remainder: TreeChangeBatch | null;
|
|
241
241
|
};
|
|
242
|
-
/**
|
|
243
|
-
* Elapsed-time cadence for scheduled full reconciles: one hour by default,
|
|
244
|
-
* measured from the last full pass (its dispatch or its completion, whichever
|
|
245
|
-
* is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
|
|
246
|
-
* 10_800_000 for three hours); an unset or invalid value falls back to the
|
|
247
|
-
* hour.
|
|
248
|
-
*
|
|
249
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
250
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
251
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
252
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
253
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
254
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
255
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
256
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
257
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
258
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
259
|
-
*/
|
|
260
|
-
export declare const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
261
|
-
export declare const DEFAULT_FULL_RECONCILE_MS: number | undefined;
|
|
262
|
-
export declare function resolveFullReconcileMs(env?: Record<string, string | undefined>): number | undefined;
|
|
263
|
-
/**
|
|
264
|
-
* A failed watch pass is still retried in-process, but even an explicitly
|
|
265
|
-
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
266
|
-
* Three failures in thirty minutes escalate from this floor to exponential
|
|
267
|
-
* backoff; success or a quiet window starts a fresh budget.
|
|
268
|
-
*/
|
|
269
|
-
export declare const WATCH_PASS_EXIT_RETRY_WINDOW_MS: number;
|
|
270
|
-
export declare const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
271
|
-
export declare const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60000;
|
|
272
|
-
export declare const WATCH_PASS_EXIT_RETRY_MAX_MS: number;
|
|
273
242
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
274
243
|
export declare const WATCHER_REDISCOVERY_RETRY_MS: number;
|
|
275
244
|
/** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
|
|
@@ -300,20 +269,6 @@ export declare function advanceMonotonicDeadline(previousDueAt: number, now: num
|
|
|
300
269
|
nextDueAt: number;
|
|
301
270
|
delayMs: number;
|
|
302
271
|
};
|
|
303
|
-
/**
|
|
304
|
-
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
305
|
-
*
|
|
306
|
-
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
307
|
-
* controls normal remote work, whereas this budget prevents a configured
|
|
308
|
-
* short poll from continuously re-running a pass that is already known to
|
|
309
|
-
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
310
|
-
* not need wall-clock waits.
|
|
311
|
-
*/
|
|
312
|
-
export declare class WatchPassExitRetryBudget {
|
|
313
|
-
private lastFailureAt;
|
|
314
|
-
private failures;
|
|
315
|
-
record(exitCode: number, now: number): number;
|
|
316
|
-
}
|
|
317
272
|
export interface ParsedLoopArgs {
|
|
318
273
|
hqRoot: string;
|
|
319
274
|
lockTimeoutSec?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAwBA,OAAO,EAKL,KAAK,UAAU,EAEf,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAUL,WAAW,EASZ,MAAM,eAAe,CAAC;AAavB,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AASlC,OAAO,EACL,UAAU,EA0CX,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAEL,qBAAqB,EACtB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,EAGL,KAAK,uBAAuB,EAC7B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EAIrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAYhD,OAAO,KAAK,EACV,yBAAyB,EACzB,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAE1B,yEAAyE;AACzE,eAAO,MAAM,uBAAuB,QAAiB,CAAC;AACtD;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,QACI,CAAC;AAChD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAiC,CAAC;AACtE;;;;;;GAMG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAK/C,wBAAgB,wCAAwC,CACtD,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAED,wBAAgB,iCAAiC,CAC/C,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,IAAI,CAEN;AAUD,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,SAAS,QAAQ;gBACd,UAAU,EAAE,MAAM;CAI/B;AAED,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,SAAS,QAAQ;gBACd,OAAO,EAAE,MAAM;CAI5B;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CASA;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,eAAe,EAAE,GAChC,uBAAuB,GAAG,SAAS,CAkBrC;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,EAAE,yBAAyB,CAAC;IACtC,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC;CACrB;AAqBD;;;;;;;;GAQG;AACH,qBAAa,wBAAwB,CAAC,CAAC;IAkBnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO;IAnB1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,MAAM,CAAqC;IACnD,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmC;IACrE,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,eAAe,CAAyB;IAIhD,OAAO,CAAC,kBAAkB,CAAqC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;gBAGZ,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAA0B,EACzC,OAAO,GAAE;QACxB,IAAI,CAAC,EAAE,qBAAqB,CAAC;KACzB;IAGR,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,qBAAqB,IAAI,OAAO,CAEnC;IAED,+FAA+F;IAC/F,IAAI,yBAAyB,IAAI,MAAM,CAMtC;IAED,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAI/C,MAAM,CACJ,IAAI,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,EAAE,gBAAgB,EAC3B,IAAI,GAAE,MAAM,GAAG,MAAe,EAC9B,yBAAyB,UAAQ,EACjC,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,UAAwB,EACnC,sBAAsB,UAAQ,GAC7B,OAAO,CAAC,CAAC,CAAC;IAsFb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,YAAY;IAMpB,6EAA6E;IAC7E,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,KAAK;IAyBb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAO5B,sFAAsF;IACtF,OAAO,CAAC,qBAAqB;IA8B7B,+EAA+E;IAC/E,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;YAab,aAAa;YAiDb,UAAU;CAWzB;AAGD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,OAAO,GAAG,aAAa,EACpC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAMT;AAmDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,4BAA4B,qCAAqC,CAAC;AAC/E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAC3D;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAa,CAAC;AAE3D;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAUD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAmC1C,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,cAAc,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACzD,qEAAqE;IACrE,YAAY,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,yBAAyB,GAAG,UAAU,CAY9E;AAED,wBAAgB,qBAAqB,IAAI;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB,CASA;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,IAAI,EAAE,CAAC,EAAE,CAAC;CACX;AAiDD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG;IAC3D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;CAC9B,CA4BA;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;CAAE,CAiC/D;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,qBAAqB,EAAE,OAAO,EAC9B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,EAClB,UAAU,CAAC,EAAE,yBAAyB,EACtC,eAAe,CAAC,EAAE,MAAM,IAAI,EAC5B,gBAAgB,CAAC,EAAE,CACjB,OAAO,EAAE,aAAa,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,KAClE,IAAI,KACN,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,iDAAiD,CAC/D,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,SAAS,EACzC,gBAAgB,EAAE,MAAM,GACvB,MAAM,GAAG,SAAS,CAOpB;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAgDD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAw1GjB"}
|
|
@@ -27,12 +27,12 @@ import { HeapCensus, startRunnerHeapSampling, heapSampleCloudProperties, heapPre
|
|
|
27
27
|
import { setRunnerEmitObserver } from "./sync-runner-events.js";
|
|
28
28
|
import { DefaultParallelSyncCoordinator } from "../sync/parallel-sync-coordinator.js";
|
|
29
29
|
import { createHqFlagsCapSource, FullPassCapController, } from "../sync/full-pass-caps.js";
|
|
30
|
+
import { createHqFlagsIntervalSource, } from "../sync/full-pass-interval.js";
|
|
30
31
|
import { DEFAULT_REALTIME_AUTHORIZATION_TIMEOUT_MS } from "../sync/realtime-rollout.js";
|
|
31
32
|
import { RealtimeTelemetrySpan, writeRealtimeTelemetryToStderr, } from "../sync/realtime-telemetry.js";
|
|
32
33
|
import { InlineSyncWorkerAdapter, } from "../sync/sync-work.js";
|
|
33
34
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
34
35
|
import { resolveSkipCompanies } from "./sync-runner-planning.js";
|
|
35
|
-
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
36
36
|
/** A deferred full chore must get the next admission after this long. */
|
|
37
37
|
export const CHORE_STARVATION_CAP_MS = 15 * 60 * 1000;
|
|
38
38
|
/**
|
|
@@ -857,36 +857,6 @@ export function takeEnumeratedBatchChunk(batch, limit) {
|
|
|
857
857
|
remainder: asBatch(remainderPaths, splitChanges(remainderPaths)),
|
|
858
858
|
};
|
|
859
859
|
}
|
|
860
|
-
/**
|
|
861
|
-
* Elapsed-time cadence for scheduled full reconciles: one hour by default,
|
|
862
|
-
* measured from the last full pass (its dispatch or its completion, whichever
|
|
863
|
-
* is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
|
|
864
|
-
* 10_800_000 for three hours); an unset or invalid value falls back to the
|
|
865
|
-
* hour.
|
|
866
|
-
*
|
|
867
|
-
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
868
|
-
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
869
|
-
* remote poll, so on an active vault a new full pass started 4-8 minutes —
|
|
870
|
-
* once observed 21 seconds — after the previous one finished, and each pass
|
|
871
|
-
* on a real install takes ~10 minutes of listing and HEAD traffic. The runner
|
|
872
|
-
* was effectively reconciling continuously. Full reconcile is the correctness
|
|
873
|
-
* backstop behind event-push (a missed watcher event, a new directory the
|
|
874
|
-
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
875
|
-
* and bare-wake triggers still force it immediately when live events cannot
|
|
876
|
-
* be trusted, so an hour is the cadence of the backstop, not of realtime.
|
|
877
|
-
*/
|
|
878
|
-
export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
879
|
-
export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
|
|
880
|
-
export function resolveFullReconcileMs(env = process.env) {
|
|
881
|
-
const raw = env[FULL_RECONCILE_MS_ENV];
|
|
882
|
-
if (raw === undefined || raw.trim() === "")
|
|
883
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
884
|
-
const parsed = Number(raw);
|
|
885
|
-
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
886
|
-
return DEFAULT_FULL_RECONCILE_MS;
|
|
887
|
-
const intervalMs = Math.trunc(parsed);
|
|
888
|
-
return intervalMs > 0 ? intervalMs : DEFAULT_FULL_RECONCILE_MS;
|
|
889
|
-
}
|
|
890
860
|
/**
|
|
891
861
|
* Adaptive remote-poll backoff bounds (whole-box CPU aware). When
|
|
892
862
|
* `--poll-remote-ms` is NOT passed, the watch loop scales its poll cadence to
|
|
@@ -900,16 +870,6 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
900
870
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
901
871
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
902
872
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
903
|
-
/**
|
|
904
|
-
* A failed watch pass is still retried in-process, but even an explicitly
|
|
905
|
-
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
906
|
-
* Three failures in thirty minutes escalate from this floor to exponential
|
|
907
|
-
* backoff; success or a quiet window starts a fresh budget.
|
|
908
|
-
*/
|
|
909
|
-
export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
|
|
910
|
-
export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
911
|
-
export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
|
|
912
|
-
export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
|
|
913
873
|
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
914
874
|
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
915
875
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
@@ -959,41 +919,6 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
959
919
|
nextDueAt = now + intervalMs;
|
|
960
920
|
return { nextDueAt, delayMs: intervalMs };
|
|
961
921
|
}
|
|
962
|
-
/**
|
|
963
|
-
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
964
|
-
*
|
|
965
|
-
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
966
|
-
* controls normal remote work, whereas this budget prevents a configured
|
|
967
|
-
* short poll from continuously re-running a pass that is already known to
|
|
968
|
-
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
969
|
-
* not need wall-clock waits.
|
|
970
|
-
*/
|
|
971
|
-
export class WatchPassExitRetryBudget {
|
|
972
|
-
lastFailureAt = null;
|
|
973
|
-
failures = 0;
|
|
974
|
-
record(exitCode, now) {
|
|
975
|
-
if (exitCode === 0) {
|
|
976
|
-
this.lastFailureAt = null;
|
|
977
|
-
this.failures = 0;
|
|
978
|
-
return 0;
|
|
979
|
-
}
|
|
980
|
-
if (this.lastFailureAt === null ||
|
|
981
|
-
now < this.lastFailureAt ||
|
|
982
|
-
now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
|
|
983
|
-
this.lastFailureAt = now;
|
|
984
|
-
this.failures = 0;
|
|
985
|
-
}
|
|
986
|
-
this.failures += 1;
|
|
987
|
-
this.lastFailureAt = now;
|
|
988
|
-
if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
|
|
989
|
-
// Preserve the established normal-cadence retry for isolated failures.
|
|
990
|
-
// The bounded cooldown starts only after repeated exits in the window.
|
|
991
|
-
return 0;
|
|
992
|
-
}
|
|
993
|
-
const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
|
|
994
|
-
return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
|
|
995
|
-
}
|
|
996
|
-
}
|
|
997
922
|
/**
|
|
998
923
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
999
924
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1115,7 +1040,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1115
1040
|
const sampleLoadAvg = deps.sampleLoadAvg ?? (() => os.loadavg()[0]);
|
|
1116
1041
|
const cpuCount = os.cpus().length;
|
|
1117
1042
|
const eventPush = argv.includes("--event-push");
|
|
1118
|
-
const
|
|
1043
|
+
const fullPassIntervalSource = deps.fullPassIntervalSource ??
|
|
1044
|
+
createHqFlagsIntervalSource({
|
|
1045
|
+
endpoint: process.env.HQ_FLAGS_API_URL,
|
|
1046
|
+
getToken: runtime.defaultGetAccessToken,
|
|
1047
|
+
});
|
|
1048
|
+
let fullReconcileIntervalMs = fullPassIntervalSource.read();
|
|
1119
1049
|
const companiesMode = argv.includes("--companies");
|
|
1120
1050
|
const companyIdx = argv.indexOf("--company");
|
|
1121
1051
|
const companyMode = companyIdx >= 0;
|
|
@@ -1285,11 +1215,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1285
1215
|
flags: flagSource,
|
|
1286
1216
|
platform: process.platform,
|
|
1287
1217
|
});
|
|
1288
|
-
if (
|
|
1218
|
+
if (process.env.HQ_FLAGS_API_URL && process.env.VITEST !== "true") {
|
|
1289
1219
|
const refresh = setTimeout(() => {
|
|
1290
|
-
void
|
|
1291
|
-
|
|
1292
|
-
|
|
1220
|
+
void Promise.all([
|
|
1221
|
+
deps.fullPassCaps ? Promise.resolve(null) : flagSource.refresh?.(),
|
|
1222
|
+
deps.fullPassIntervalSource ? Promise.resolve(null) : fullPassIntervalSource.refresh?.(),
|
|
1223
|
+
]).then(([capFlags, intervalMs]) => {
|
|
1224
|
+
if (capFlags)
|
|
1225
|
+
fullPassCaps.applyFlags(capFlags);
|
|
1226
|
+
if (intervalMs)
|
|
1227
|
+
fullReconcileIntervalMs = intervalMs;
|
|
1293
1228
|
});
|
|
1294
1229
|
}, 0);
|
|
1295
1230
|
refresh.unref?.();
|
|
@@ -1506,8 +1441,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1506
1441
|
resolveInterruptWait = null;
|
|
1507
1442
|
resolve?.();
|
|
1508
1443
|
};
|
|
1509
|
-
const waitForNextPass = async (delayMs
|
|
1510
|
-
if (
|
|
1444
|
+
const waitForNextPass = async (delayMs) => {
|
|
1445
|
+
if (interruptGeneration !== observedInterruptGeneration) {
|
|
1511
1446
|
observedInterruptGeneration = interruptGeneration;
|
|
1512
1447
|
return;
|
|
1513
1448
|
}
|
|
@@ -1522,9 +1457,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1522
1457
|
}, idleHeartbeatIntervalMs);
|
|
1523
1458
|
heartbeatTimer.unref?.();
|
|
1524
1459
|
try {
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1460
|
+
await Promise.race([
|
|
1461
|
+
sleep(delayMs),
|
|
1462
|
+
new Promise((resolve) => {
|
|
1528
1463
|
resolveInterruptWait = resolve;
|
|
1529
1464
|
// Do not lose an event that arrived between the first generation
|
|
1530
1465
|
// check and registration of this wait.
|
|
@@ -1532,9 +1467,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1532
1467
|
resolveInterruptWait = null;
|
|
1533
1468
|
resolve();
|
|
1534
1469
|
}
|
|
1535
|
-
})
|
|
1536
|
-
|
|
1537
|
-
|
|
1470
|
+
}),
|
|
1471
|
+
stoppedSignal,
|
|
1472
|
+
]);
|
|
1538
1473
|
}
|
|
1539
1474
|
finally {
|
|
1540
1475
|
clearInterval(heartbeatTimer);
|
|
@@ -3558,15 +3493,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3558
3493
|
// The initial reconcile is due immediately. On-time passes advance from this
|
|
3559
3494
|
// deadline; overruns restart their cooldown from completion.
|
|
3560
3495
|
let nextDueAt = monotonicNow();
|
|
3561
|
-
const passExitRetryBudget = new WatchPassExitRetryBudget();
|
|
3562
|
-
let passExitRetryNotBefore = null;
|
|
3563
|
-
const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
|
|
3564
|
-
const delayMs = passExitRetryBudget.record(exitCode, now);
|
|
3565
|
-
passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
|
|
3566
|
-
if (delayMs > 0) {
|
|
3567
|
-
process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
|
|
3568
|
-
}
|
|
3569
|
-
};
|
|
3570
3496
|
let pollTick = 0;
|
|
3571
3497
|
let lastCompletedFullReconcileAt = null;
|
|
3572
3498
|
// Anchor for the elapsed-time cadence: the most recent full reconcile that
|
|
@@ -3596,17 +3522,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3596
3522
|
return RUNNER_HEAP_RECYCLE_EXIT;
|
|
3597
3523
|
if (coordinatorAuthRequired)
|
|
3598
3524
|
return runtime.authRequiredPassExit;
|
|
3599
|
-
if (passExitRetryNotBefore !== null) {
|
|
3600
|
-
const retryNow = monotonicNow();
|
|
3601
|
-
if (retryNow < passExitRetryNotBefore) {
|
|
3602
|
-
// A watcher/receiver interrupt must not turn a known failing pass
|
|
3603
|
-
// back into a tight loop. Pending work remains queued and runs when
|
|
3604
|
-
// this non-interruptible cooldown has elapsed.
|
|
3605
|
-
await waitForNextPass(passExitRetryNotBefore - retryNow, false);
|
|
3606
|
-
continue;
|
|
3607
|
-
}
|
|
3608
|
-
passExitRetryNotBefore = null;
|
|
3609
|
-
}
|
|
3610
3525
|
pollTick++;
|
|
3611
3526
|
areaAutoMigration.onTick();
|
|
3612
3527
|
const pending = takePendingWatcherChange();
|
|
@@ -3638,11 +3553,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3638
3553
|
const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
|
|
3639
3554
|
const coverageStale = rootCoverageStale;
|
|
3640
3555
|
rootCoverageStale = false;
|
|
3641
|
-
const scheduledInterval =
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
cadenceNow !== null &&
|
|
3645
|
-
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3556
|
+
const scheduledInterval = lastFullReconcileAt !== null &&
|
|
3557
|
+
cadenceNow !== null &&
|
|
3558
|
+
cadenceNow - lastFullReconcileAt >= fullReconcileIntervalMs;
|
|
3646
3559
|
const triggers = {
|
|
3647
3560
|
"event-push-surfaces-inactive": eventPushSurfacesInactive,
|
|
3648
3561
|
"first-tick": firstTick,
|
|
@@ -3702,7 +3615,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3702
3615
|
};
|
|
3703
3616
|
let result;
|
|
3704
3617
|
let ranFullReconcile = false;
|
|
3705
|
-
let ranWatchPass = false;
|
|
3706
3618
|
let enqueuedFullChore = false;
|
|
3707
3619
|
if (shouldRunFull) {
|
|
3708
3620
|
emitDecision("full", reasons);
|
|
@@ -3722,9 +3634,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3722
3634
|
interruptWait();
|
|
3723
3635
|
return;
|
|
3724
3636
|
}
|
|
3725
|
-
recordWatchPassExit(exit);
|
|
3726
|
-
if (exit !== 0)
|
|
3727
|
-
interruptWait();
|
|
3728
3637
|
// Preserve the numeric success seam accepted by the prior
|
|
3729
3638
|
// coordinator while also accepting a completed RunnerPassOutcome
|
|
3730
3639
|
// (including a partial per-company pass) that proves journal
|
|
@@ -3769,7 +3678,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3769
3678
|
}
|
|
3770
3679
|
else {
|
|
3771
3680
|
ranFullReconcile = true;
|
|
3772
|
-
ranWatchPass = true;
|
|
3773
3681
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3774
3682
|
}
|
|
3775
3683
|
}
|
|
@@ -3799,7 +3707,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3799
3707
|
result = 0;
|
|
3800
3708
|
}
|
|
3801
3709
|
else {
|
|
3802
|
-
ranWatchPass = true;
|
|
3803
3710
|
result = overflowWithKnownRoutes
|
|
3804
3711
|
? await runOverflowRouteDrain(workingBatch)
|
|
3805
3712
|
: await runScopedDrain(workingBatch, pending.generation);
|
|
@@ -3817,7 +3724,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3817
3724
|
process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
|
|
3818
3725
|
`falling back to a full reconcile\n`);
|
|
3819
3726
|
ranFullReconcile = true;
|
|
3820
|
-
ranWatchPass = true;
|
|
3821
3727
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3822
3728
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3823
3729
|
}
|
|
@@ -3963,8 +3869,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3963
3869
|
realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
|
|
3964
3870
|
}
|
|
3965
3871
|
const now = monotonicNow();
|
|
3966
|
-
if (ranWatchPass)
|
|
3967
|
-
recordWatchPassExit(exitCode, now);
|
|
3968
3872
|
if (ranFullReconcile) {
|
|
3969
3873
|
// The awaited full pass finished on this tick; reuse the tick's clock
|
|
3970
3874
|
// sample so the cadence anchor never costs a second read.
|
|
@@ -3984,14 +3888,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3984
3888
|
? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
|
|
3985
3889
|
: advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
|
|
3986
3890
|
nextDueAt = schedule.nextDueAt;
|
|
3987
|
-
const retryDelayMs = passExitRetryNotBefore === null
|
|
3988
|
-
? 0
|
|
3989
|
-
: Math.max(0, passExitRetryNotBefore - now);
|
|
3990
|
-
const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
|
|
3991
3891
|
const untilHighWaterMs = realtimeState.nextHighWaterAt === null
|
|
3992
|
-
?
|
|
3892
|
+
? schedule.delayMs
|
|
3993
3893
|
: Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
|
|
3994
|
-
await waitForNextPass(Math.min(
|
|
3894
|
+
await waitForNextPass(Math.min(schedule.delayMs, untilHighWaterMs));
|
|
3995
3895
|
}
|
|
3996
3896
|
return 0;
|
|
3997
3897
|
}
|