@indigoai-us/hq-cloud 6.15.64 → 6.15.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +19 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +329 -55
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +8 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +582 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/watcher.d.ts +17 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +74 -1
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +57 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type TreeChange } from "../watcher.js";
|
|
1
2
|
import { readJournal } from "../journal.js";
|
|
2
3
|
import type { TelemetryClaims } from "../telemetry-events.js";
|
|
3
4
|
import type { Direction, RunnerLoopDeps, RunnerPassOutcome } from "./sync-runner.js";
|
|
@@ -12,6 +13,24 @@ import type { Direction, RunnerLoopDeps, RunnerPassOutcome } from "./sync-runner
|
|
|
12
13
|
*/
|
|
13
14
|
export declare const DEFAULT_EVENT_BATCH_LIMIT = 10000;
|
|
14
15
|
export declare const EVENT_BATCH_LIMIT_ENV = "HQ_SYNC_EVENT_BATCH_LIMIT";
|
|
16
|
+
/**
|
|
17
|
+
* Fast-lane admission is deliberately conservative: a normal editor save is
|
|
18
|
+
* a handful of files and well below 4 MiB, whereas session-log catch-up is
|
|
19
|
+
* neither. Operators may tune both limits without changing scheduling
|
|
20
|
+
* semantics. Invalid values retain the safe defaults.
|
|
21
|
+
*/
|
|
22
|
+
export declare const FAST_LANE_MAX_FILES_ENV = "HQ_SYNC_FAST_LANE_MAX_FILES";
|
|
23
|
+
export declare const FAST_LANE_MAX_BYTES_ENV = "HQ_SYNC_FAST_LANE_MAX_BYTES";
|
|
24
|
+
export declare const DEFAULT_FAST_LANE_MAX_FILES = 32;
|
|
25
|
+
export declare const DEFAULT_FAST_LANE_MAX_BYTES: number;
|
|
26
|
+
export declare function resolveFastLaneLimits(): {
|
|
27
|
+
maxFiles: number;
|
|
28
|
+
maxBytes: number;
|
|
29
|
+
};
|
|
30
|
+
/** A missing/stat-failed path is never allowed to bypass the slow lane. */
|
|
31
|
+
export declare function isFastLocalBatch(paths: Iterable<string>, changes?: ReadonlyMap<string, Pick<TreeChange, "kind">>): boolean;
|
|
32
|
+
/** Receiver events carry no content length, so only bounded event batches qualify. */
|
|
33
|
+
export declare function isFastReceiverBatch(eventCount: number): boolean;
|
|
15
34
|
export declare function resolveEventBatchLimit(env?: Record<string, string | undefined>): number;
|
|
16
35
|
/**
|
|
17
36
|
* Optional elapsed-time cadence for full reconciles. Leaving this unset keeps
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sync-runner-watch-loop.d.ts","sourceRoot":"","sources":["../../src/bin/sync-runner-watch-loop.ts"],"names":[],"mappings":"AAkBA,OAAO,EAKL,KAAK,UAAU,EAEhB,MAAM,eAAe,CAAC;AACvB,OAAO,EAML,WAAW,EAEZ,MAAM,eAAe,CAAC;AAWvB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAY9D,OAAO,KAAK,EACV,SAAS,EAET,cAAc,EACd,iBAAiB,EAElB,MAAM,kBAAkB,CAAC;AAsD1B;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAChD,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAC9C,eAAO,MAAM,2BAA2B,QAAkB,CAAC;AAO3D,wBAAgB,qBAAqB,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAK9E;AAED,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvB,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GACtD,OAAO,CAyBT;AAED,sFAAsF;AACtF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,CAMR;AAED;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,8BAA8B,CAAC;AACjE,eAAO,MAAM,yBAAyB,EAAE,MAAM,GAAG,SAAqB,CAAC;AAEvE,wBAAgB,sBAAsB,CACpC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,MAAM,GAAG,SAAS,CAOpB;AAiBD,iFAAiF;AACjF,eAAO,MAAM,4BAA4B,QAAa,CAAC;AACvD,kFAAkF;AAClF,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAE5D;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAA+B,EACxC,MAAM,GAAE,MAA8B,GACrC,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GACjB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC,EAAE,CACnC,QAAQ,EAAE,MAAM,EAAE,KACf,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,CAAC,GAAG,IAAI,CAAC;IAC7E,qBAAqB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC;AAE5D;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,WAAW,GAC3B,KAAK,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CASvC;AAWD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oCAAoC,CAClD,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,MAAM,GACrB,OAAO,CAIT;AAuBD,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EAAE,EACd,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CAiwDjB"}
|
|
@@ -12,7 +12,7 @@ import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
|
12
12
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
13
13
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
14
14
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
15
|
-
import { clearLocalDeleteIntent, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, readJournal, writeJournal, } from "../journal.js";
|
|
15
|
+
import { clearLocalDeleteIntent, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, listJournals, readJournal, writeJournal, } from "../journal.js";
|
|
16
16
|
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
17
17
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
18
18
|
import { buildRoutePullArgv, buildScopedPushArgv, buildScopedDrainPullArgv, buildTargetedPullArgv, isWatchRouteSelected, routeChangeToTarget, routeKey, } from "./sync-runner-watch-routes.js";
|
|
@@ -44,6 +44,63 @@ function emitFullReconcileDecision({ mode, reasons, triggers, pollTick, configur
|
|
|
44
44
|
*/
|
|
45
45
|
export const DEFAULT_EVENT_BATCH_LIMIT = 10_000;
|
|
46
46
|
export const EVENT_BATCH_LIMIT_ENV = "HQ_SYNC_EVENT_BATCH_LIMIT";
|
|
47
|
+
/**
|
|
48
|
+
* Fast-lane admission is deliberately conservative: a normal editor save is
|
|
49
|
+
* a handful of files and well below 4 MiB, whereas session-log catch-up is
|
|
50
|
+
* neither. Operators may tune both limits without changing scheduling
|
|
51
|
+
* semantics. Invalid values retain the safe defaults.
|
|
52
|
+
*/
|
|
53
|
+
export const FAST_LANE_MAX_FILES_ENV = "HQ_SYNC_FAST_LANE_MAX_FILES";
|
|
54
|
+
export const FAST_LANE_MAX_BYTES_ENV = "HQ_SYNC_FAST_LANE_MAX_BYTES";
|
|
55
|
+
export const DEFAULT_FAST_LANE_MAX_FILES = 32;
|
|
56
|
+
export const DEFAULT_FAST_LANE_MAX_BYTES = 4 * 1024 * 1024;
|
|
57
|
+
function positiveEnvInt(name, fallback) {
|
|
58
|
+
const value = Number.parseInt(process.env[name] ?? "", 10);
|
|
59
|
+
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
60
|
+
}
|
|
61
|
+
export function resolveFastLaneLimits() {
|
|
62
|
+
return {
|
|
63
|
+
maxFiles: positiveEnvInt(FAST_LANE_MAX_FILES_ENV, DEFAULT_FAST_LANE_MAX_FILES),
|
|
64
|
+
maxBytes: positiveEnvInt(FAST_LANE_MAX_BYTES_ENV, DEFAULT_FAST_LANE_MAX_BYTES),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** A missing/stat-failed path is never allowed to bypass the slow lane. */
|
|
68
|
+
export function isFastLocalBatch(paths, changes) {
|
|
69
|
+
const { maxFiles, maxBytes } = resolveFastLaneLimits();
|
|
70
|
+
let files = 0;
|
|
71
|
+
let bytes = 0;
|
|
72
|
+
for (const absolutePath of paths) {
|
|
73
|
+
files += 1;
|
|
74
|
+
if (files > maxFiles)
|
|
75
|
+
return false;
|
|
76
|
+
// `unlinkDir` arrives after its path is gone. Its scoped push expands the
|
|
77
|
+
// journal subtree through deleteScopeRoots, so its payload cannot be
|
|
78
|
+
// bounded by the absent directory's lstat result.
|
|
79
|
+
if (changes?.get(absolutePath)?.kind === "unlinkDir")
|
|
80
|
+
return false;
|
|
81
|
+
try {
|
|
82
|
+
const stat = fs.lstatSync(absolutePath);
|
|
83
|
+
// A directory-scoped operation may expand to an arbitrary payload. Do
|
|
84
|
+
// not guess: it belongs in the slow lane.
|
|
85
|
+
if (stat.isDirectory())
|
|
86
|
+
return false;
|
|
87
|
+
bytes += stat.isFile() ? stat.size : 0;
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
// Deletes are zero-byte operations; every other stat failure is unknown
|
|
91
|
+
// payload and must retain normal serialization.
|
|
92
|
+
if (err.code !== "ENOENT")
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
if (bytes > maxBytes)
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
return files > 0;
|
|
99
|
+
}
|
|
100
|
+
/** Receiver events carry no content length, so only bounded event batches qualify. */
|
|
101
|
+
export function isFastReceiverBatch(eventCount) {
|
|
102
|
+
return eventCount > 0 && eventCount <= resolveFastLaneLimits().maxFiles;
|
|
103
|
+
}
|
|
47
104
|
export function resolveEventBatchLimit(env = process.env) {
|
|
48
105
|
const raw = env[EVENT_BATCH_LIMIT_ENV];
|
|
49
106
|
if (raw === undefined || raw.trim() === "")
|
|
@@ -84,6 +141,8 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
84
141
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
85
142
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
86
143
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
144
|
+
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
145
|
+
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
87
146
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
88
147
|
export const WATCHER_REDISCOVERY_RETRY_MS = 5 * 60_000;
|
|
89
148
|
/** Ceiling for exponential retry delays after consecutive watcher stand-downs. */
|
|
@@ -222,6 +281,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
222
281
|
skipPersonalEnv === "yes",
|
|
223
282
|
};
|
|
224
283
|
const hqRoot = parsed.hqRoot;
|
|
284
|
+
const serviceStartedAt = Date.now();
|
|
225
285
|
// Delayed and daily: this must never be attached to the 15s poll cadence.
|
|
226
286
|
// Stage 0 remains dry-run only; the standalone hq-backup-prune command is
|
|
227
287
|
// the explicitly-gated apply surface.
|
|
@@ -285,7 +345,47 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
285
345
|
throw err;
|
|
286
346
|
}
|
|
287
347
|
}
|
|
288
|
-
|
|
348
|
+
// `activeSlowPass` is assigned before its task has necessarily acquired the
|
|
349
|
+
// cross-process lock (it may be waiting behind a rescue or a second runner).
|
|
350
|
+
// Fast work may bypass the lock only while this process *actually owns* it;
|
|
351
|
+
// otherwise it must participate in normal lock acquisition.
|
|
352
|
+
let processOwnsOperationLock = false;
|
|
353
|
+
let operationLockUsers = 0;
|
|
354
|
+
let resolveNoOperationLockUsers = null;
|
|
355
|
+
const acquireOperationLockUser = () => {
|
|
356
|
+
operationLockUsers += 1;
|
|
357
|
+
};
|
|
358
|
+
const releaseOperationLockUser = () => {
|
|
359
|
+
operationLockUsers -= 1;
|
|
360
|
+
if (operationLockUsers !== 0)
|
|
361
|
+
return;
|
|
362
|
+
const resolve = resolveNoOperationLockUsers;
|
|
363
|
+
resolveNoOperationLockUsers = null;
|
|
364
|
+
resolve?.();
|
|
365
|
+
};
|
|
366
|
+
const waitForOperationLockUsers = () => {
|
|
367
|
+
if (operationLockUsers === 0)
|
|
368
|
+
return null;
|
|
369
|
+
return new Promise((resolve) => {
|
|
370
|
+
resolveNoOperationLockUsers = resolve;
|
|
371
|
+
});
|
|
372
|
+
};
|
|
373
|
+
const runPassWithLock = async (passArgvForRun, prepare, lane = "slow") => {
|
|
374
|
+
// The operation lock remains the cross-process sync/rescue exclusion
|
|
375
|
+
// boundary. A fast scoped pass is the one deliberate in-process exception:
|
|
376
|
+
// its keyed journal delta and per-object mutation are independent from the
|
|
377
|
+
// slow pass's bulk paths, and the slow pass already owns the root lock.
|
|
378
|
+
// Never bypass the lock when no slow pass is active.
|
|
379
|
+
if (lane === "fast" && processOwnsOperationLock) {
|
|
380
|
+
acquireOperationLockUser();
|
|
381
|
+
try {
|
|
382
|
+
prepare?.();
|
|
383
|
+
return await runPass(passArgvForRun);
|
|
384
|
+
}
|
|
385
|
+
finally {
|
|
386
|
+
releaseOperationLockUser();
|
|
387
|
+
}
|
|
388
|
+
}
|
|
289
389
|
try {
|
|
290
390
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
291
391
|
timeoutSec: 0,
|
|
@@ -293,10 +393,24 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
293
393
|
deferToWaiters: true,
|
|
294
394
|
});
|
|
295
395
|
try {
|
|
396
|
+
if (lane === "slow") {
|
|
397
|
+
processOwnsOperationLock = true;
|
|
398
|
+
acquireOperationLockUser();
|
|
399
|
+
}
|
|
296
400
|
prepare?.();
|
|
297
401
|
return await runPass(passArgvForRun);
|
|
298
402
|
}
|
|
299
403
|
finally {
|
|
404
|
+
if (lane === "slow") {
|
|
405
|
+
releaseOperationLockUser();
|
|
406
|
+
// The root lock protects a fast pass that was admitted while this
|
|
407
|
+
// slow pass owned it. Do not make it acquirable by rescue/another
|
|
408
|
+
// runner until every such in-process user has completed.
|
|
409
|
+
const remainingUsers = waitForOperationLockUsers();
|
|
410
|
+
if (remainingUsers)
|
|
411
|
+
await remainingUsers;
|
|
412
|
+
processOwnsOperationLock = false;
|
|
413
|
+
}
|
|
300
414
|
handle.release();
|
|
301
415
|
}
|
|
302
416
|
}
|
|
@@ -309,9 +423,24 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
309
423
|
}
|
|
310
424
|
}
|
|
311
425
|
try {
|
|
312
|
-
return await withOperationLock(hqRoot, "sync", () => {
|
|
313
|
-
|
|
314
|
-
|
|
426
|
+
return await withOperationLock(hqRoot, "sync", async () => {
|
|
427
|
+
if (lane === "slow") {
|
|
428
|
+
processOwnsOperationLock = true;
|
|
429
|
+
acquireOperationLockUser();
|
|
430
|
+
}
|
|
431
|
+
try {
|
|
432
|
+
prepare?.();
|
|
433
|
+
return await runPass(passArgvForRun);
|
|
434
|
+
}
|
|
435
|
+
finally {
|
|
436
|
+
if (lane === "slow") {
|
|
437
|
+
releaseOperationLockUser();
|
|
438
|
+
const remainingUsers = waitForOperationLockUsers();
|
|
439
|
+
if (remainingUsers)
|
|
440
|
+
await remainingUsers;
|
|
441
|
+
processOwnsOperationLock = false;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
315
444
|
}, { timeoutSec: parsed.lockTimeoutSec, deferToWaiters: true });
|
|
316
445
|
}
|
|
317
446
|
catch (err) {
|
|
@@ -413,51 +542,89 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
413
542
|
observedInterruptGeneration = interruptGeneration;
|
|
414
543
|
}
|
|
415
544
|
};
|
|
416
|
-
let
|
|
417
|
-
|
|
545
|
+
let activeSlowPass = null;
|
|
546
|
+
let activeFastPass = null;
|
|
547
|
+
const pendingSlowPasses = [];
|
|
548
|
+
const pendingFastPasses = [];
|
|
418
549
|
const resolveStoppedQueue = () => {
|
|
419
|
-
|
|
420
|
-
|
|
550
|
+
for (const queue of [pendingFastPasses, pendingSlowPasses]) {
|
|
551
|
+
while (queue.length > 0)
|
|
552
|
+
queue.shift()?.resolve(0);
|
|
421
553
|
}
|
|
422
554
|
};
|
|
423
555
|
const drainQueuedPasses = () => {
|
|
424
|
-
if (activePass !== null)
|
|
425
|
-
return;
|
|
426
556
|
if (stopped) {
|
|
427
557
|
resolveStoppedQueue();
|
|
428
558
|
return;
|
|
429
559
|
}
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
560
|
+
if (activeFastPass === null) {
|
|
561
|
+
const fast = pendingFastPasses.shift();
|
|
562
|
+
if (fast) {
|
|
563
|
+
const current = startGuardedTask(fast.task, "fast");
|
|
564
|
+
void current.then(fast.resolve, fast.reject);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
// A fast pass that acquired the operation lock normally (rather than
|
|
568
|
+
// bypassing an in-process slow holder) still owns the cross-process
|
|
569
|
+
// boundary. Slow work must queue behind it; only fast work is allowed to
|
|
570
|
+
// overlap an already-lock-owning slow pass.
|
|
571
|
+
if (activeSlowPass === null && activeFastPass === null) {
|
|
572
|
+
const slow = pendingSlowPasses.shift();
|
|
573
|
+
if (slow) {
|
|
574
|
+
const current = startGuardedTask(slow.task, "slow");
|
|
575
|
+
void current.then(slow.resolve, slow.reject);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
435
578
|
};
|
|
436
|
-
const startGuardedTask = (task) => {
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
579
|
+
const startGuardedTask = (task, lane) => {
|
|
580
|
+
// Claim the lane before invoking `task`: a task can synchronously activate
|
|
581
|
+
// a watcher/scheduler callback before returning its promise. Without this
|
|
582
|
+
// placeholder that re-entrant callback sees an idle lane and starts a
|
|
583
|
+
// second lock contender.
|
|
584
|
+
const placeholder = Promise.resolve(0);
|
|
585
|
+
if (lane === "fast")
|
|
586
|
+
activeFastPass = placeholder;
|
|
587
|
+
else
|
|
588
|
+
activeSlowPass = placeholder;
|
|
589
|
+
let current;
|
|
590
|
+
try {
|
|
591
|
+
current = stopped ? placeholder : task();
|
|
592
|
+
}
|
|
593
|
+
catch (err) {
|
|
594
|
+
current = Promise.reject(err);
|
|
595
|
+
}
|
|
596
|
+
if (lane === "fast")
|
|
597
|
+
activeFastPass = current;
|
|
598
|
+
else
|
|
599
|
+
activeSlowPass = current;
|
|
441
600
|
void current
|
|
442
601
|
.finally(() => {
|
|
443
|
-
if (
|
|
444
|
-
|
|
602
|
+
if (lane === "fast" && activeFastPass === current) {
|
|
603
|
+
activeFastPass = null;
|
|
604
|
+
drainQueuedPasses();
|
|
605
|
+
}
|
|
606
|
+
else if (lane === "slow" && activeSlowPass === current) {
|
|
607
|
+
activeSlowPass = null;
|
|
445
608
|
drainQueuedPasses();
|
|
446
609
|
}
|
|
447
610
|
})
|
|
448
611
|
.catch(() => undefined);
|
|
449
612
|
return current;
|
|
450
613
|
};
|
|
451
|
-
const runGuardedTask = (task) => {
|
|
452
|
-
|
|
453
|
-
|
|
614
|
+
const runGuardedTask = (task, lane = "slow") => {
|
|
615
|
+
const active = lane === "fast"
|
|
616
|
+
? activeFastPass
|
|
617
|
+
: activeSlowPass ?? activeFastPass;
|
|
618
|
+
const queued = lane === "fast" ? pendingFastPasses : pendingSlowPasses;
|
|
619
|
+
if (active === null && queued.length === 0) {
|
|
620
|
+
return startGuardedTask(task, lane);
|
|
454
621
|
}
|
|
455
622
|
return new Promise((resolve, reject) => {
|
|
456
|
-
|
|
623
|
+
queued.push({ task, lane, resolve, reject });
|
|
457
624
|
drainQueuedPasses();
|
|
458
625
|
});
|
|
459
626
|
};
|
|
460
|
-
const runGuarded = (passArgvForRun, prepare) => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare));
|
|
627
|
+
const runGuarded = (passArgvForRun, prepare, lane = "slow") => runGuardedTask(() => runPassWithLock(passArgvForRun, prepare, lane), lane);
|
|
461
628
|
let watcher = null;
|
|
462
629
|
let eventPushSurfacesActivated = false;
|
|
463
630
|
let watcherUnavailableWarned = false;
|
|
@@ -542,6 +709,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
542
709
|
let pendingWatcherBatchGeneration = null;
|
|
543
710
|
const lastPublishedWatcherBatchGenerationByRoute = new Map();
|
|
544
711
|
let pendingWatcherBareChange = false;
|
|
712
|
+
let pendingWatcherWarmupCatchup = false;
|
|
713
|
+
let pendingWatcherWarmupCatchupFailed = false;
|
|
545
714
|
let pendingWatcherOverflowed = false;
|
|
546
715
|
let pendingWatcherDroppedPaths = 0;
|
|
547
716
|
let pendingWatcherDroppedBytes = 0;
|
|
@@ -550,26 +719,28 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
550
719
|
// pending state carried them; one hint-less overflow poisons the union.
|
|
551
720
|
const pendingWatcherDroppedRouteHints = new Set();
|
|
552
721
|
let pendingWatcherDroppedRoutesUnknown = false;
|
|
722
|
+
const selectWatchBatch = (batch) => {
|
|
723
|
+
const paths = new Map();
|
|
724
|
+
const changes = new Map();
|
|
725
|
+
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
726
|
+
const route = routeChangeToTarget(relativePath);
|
|
727
|
+
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
728
|
+
continue;
|
|
729
|
+
paths.set(absolutePath, relativePath);
|
|
730
|
+
const change = batch.changes?.get(absolutePath);
|
|
731
|
+
if (change)
|
|
732
|
+
changes.set(absolutePath, change);
|
|
733
|
+
}
|
|
734
|
+
if (paths.size === 0)
|
|
735
|
+
return null;
|
|
736
|
+
return { ...batch, paths, changes };
|
|
737
|
+
};
|
|
553
738
|
const addPendingWatcherChange = (changedRelPath, batch) => {
|
|
554
739
|
if (batch) {
|
|
555
|
-
const
|
|
556
|
-
|
|
557
|
-
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
558
|
-
const route = routeChangeToTarget(relativePath);
|
|
559
|
-
if (!route || !isWatchRouteSelected(route, watchRouteSelection))
|
|
560
|
-
continue;
|
|
561
|
-
paths.set(absolutePath, relativePath);
|
|
562
|
-
const change = batch.changes?.get(absolutePath);
|
|
563
|
-
if (change)
|
|
564
|
-
changes.set(absolutePath, change);
|
|
565
|
-
}
|
|
566
|
-
if (paths.size === 0)
|
|
740
|
+
const selectedBatch = selectWatchBatch(batch);
|
|
741
|
+
if (!selectedBatch)
|
|
567
742
|
return false;
|
|
568
|
-
batch =
|
|
569
|
-
...batch,
|
|
570
|
-
paths,
|
|
571
|
-
changes,
|
|
572
|
-
};
|
|
743
|
+
batch = selectedBatch;
|
|
573
744
|
}
|
|
574
745
|
else if (changedRelPath) {
|
|
575
746
|
const route = routeChangeToTarget(changedRelPath);
|
|
@@ -651,18 +822,22 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
651
822
|
}
|
|
652
823
|
: null;
|
|
653
824
|
const bare = pendingWatcherBareChange;
|
|
825
|
+
const warmupCatchup = pendingWatcherWarmupCatchup;
|
|
826
|
+
const warmupCatchupFailed = pendingWatcherWarmupCatchupFailed;
|
|
654
827
|
const generation = pendingWatcherBatchGeneration;
|
|
655
828
|
pendingWatcherPaths.clear();
|
|
656
829
|
pendingWatcherChanges.clear();
|
|
657
830
|
pendingWatcherOriginalBatch = null;
|
|
658
831
|
pendingWatcherBareChange = false;
|
|
832
|
+
pendingWatcherWarmupCatchup = false;
|
|
833
|
+
pendingWatcherWarmupCatchupFailed = false;
|
|
659
834
|
pendingWatcherOverflowed = false;
|
|
660
835
|
pendingWatcherDroppedPaths = 0;
|
|
661
836
|
pendingWatcherDroppedBytes = 0;
|
|
662
837
|
pendingWatcherDroppedRouteHints.clear();
|
|
663
838
|
pendingWatcherDroppedRoutesUnknown = false;
|
|
664
839
|
pendingWatcherBatchGeneration = null;
|
|
665
|
-
return { batch, bare, generation };
|
|
840
|
+
return { batch, bare, warmupCatchup, warmupCatchupFailed, generation };
|
|
666
841
|
};
|
|
667
842
|
const restorePendingWatcherPaths = (paths, generation) => {
|
|
668
843
|
pendingWatcherBatchGeneration ??=
|
|
@@ -841,6 +1016,20 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
841
1016
|
}
|
|
842
1017
|
return snapshots;
|
|
843
1018
|
};
|
|
1019
|
+
const warmupJournalPaths = () => {
|
|
1020
|
+
const paths = [];
|
|
1021
|
+
for (const { slug, journal } of listJournals()) {
|
|
1022
|
+
const route = slug === PERSONAL_VAULT_JOURNAL_SLUG || slug === "personal"
|
|
1023
|
+
? { kind: "personal" }
|
|
1024
|
+
: { kind: "company", slug };
|
|
1025
|
+
if (!isWatchRouteSelected(route, watchRouteSelection))
|
|
1026
|
+
continue;
|
|
1027
|
+
for (const key of Object.keys(journal.files)) {
|
|
1028
|
+
paths.push(route.kind === "personal" ? key : `companies/${route.slug}/${key}`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
return paths;
|
|
1032
|
+
};
|
|
844
1033
|
const prepareWatcherChanges = (changes) => {
|
|
845
1034
|
const journals = new Map();
|
|
846
1035
|
const dirty = new Set();
|
|
@@ -939,7 +1128,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
939
1128
|
}
|
|
940
1129
|
return false;
|
|
941
1130
|
};
|
|
942
|
-
const runScopedDrain = async (batch, watcherBatchGeneration) => {
|
|
1131
|
+
const runScopedDrain = async (batch, watcherBatchGeneration, wakeOnPushFailure = false) => {
|
|
943
1132
|
const grouped = batch && batch.paths.size > 0 ? groupBatchByRoute(batch) : null;
|
|
944
1133
|
const skipCompanies = grouped ? resolveSkipCompanies() : null;
|
|
945
1134
|
const isPausedRoute = (route) => route.kind === "company" &&
|
|
@@ -953,9 +1142,12 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
953
1142
|
}
|
|
954
1143
|
const scopedArgv = buildScopedPushArgv(group.route, group.relPaths, passArgv, group.deleteScopeRoots);
|
|
955
1144
|
try {
|
|
1145
|
+
const lane = isFastLocalBatch(group.paths.keys(), group.changes)
|
|
1146
|
+
? "fast"
|
|
1147
|
+
: "slow";
|
|
956
1148
|
const result = await runGuarded(scopedArgv, group.changes.size > 0
|
|
957
1149
|
? () => prepareWatcherChanges(group.changes)
|
|
958
|
-
: undefined);
|
|
1150
|
+
: undefined, lane);
|
|
959
1151
|
if (passExitCode(result) === 0) {
|
|
960
1152
|
const refusedPaths = new Set(typeof result === "number"
|
|
961
1153
|
? []
|
|
@@ -980,11 +1172,15 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
980
1172
|
}
|
|
981
1173
|
else {
|
|
982
1174
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
|
|
1175
|
+
if (wakeOnPushFailure)
|
|
1176
|
+
interruptWait();
|
|
983
1177
|
}
|
|
984
1178
|
}
|
|
985
1179
|
catch (err) {
|
|
986
1180
|
restorePendingWatcherPaths(group.paths, watcherBatchGeneration);
|
|
987
1181
|
process.stderr.write(`watch scoped push failed, will retry next tick: ${describeError(err)}\n`);
|
|
1182
|
+
if (wakeOnPushFailure)
|
|
1183
|
+
interruptWait();
|
|
988
1184
|
}
|
|
989
1185
|
}
|
|
990
1186
|
}
|
|
@@ -1012,7 +1208,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1012
1208
|
continue;
|
|
1013
1209
|
ranPersonalDrain = true;
|
|
1014
1210
|
}
|
|
1015
|
-
const exit = passExitCode(await runGuarded(pullArgv));
|
|
1211
|
+
const exit = passExitCode(await runGuarded(pullArgv, undefined, isFastLocalBatch(group.paths.keys(), group.changes) ? "fast" : "slow"));
|
|
1016
1212
|
if (exit !== 0 && worstExit === 0)
|
|
1017
1213
|
worstExit = exit;
|
|
1018
1214
|
}
|
|
@@ -1146,6 +1342,31 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1146
1342
|
watcher.onChange((changedRelPath, batch) => {
|
|
1147
1343
|
if (stopped)
|
|
1148
1344
|
return;
|
|
1345
|
+
// A complete, small watcher batch is dispatched immediately while a
|
|
1346
|
+
// bulk pass is in flight. Leaving it in the next poll iteration would
|
|
1347
|
+
// still make it wait for that pass even though the fast lane exists.
|
|
1348
|
+
const selectedBatch = batch ? selectWatchBatch(batch) : null;
|
|
1349
|
+
if (selectedBatch &&
|
|
1350
|
+
!selectedBatch.overflowed &&
|
|
1351
|
+
activeSlowPass !== null &&
|
|
1352
|
+
isFastLocalBatch(selectedBatch.paths.keys(), selectedBatch.changes)) {
|
|
1353
|
+
const generation = ++nextWatcherBatchGeneration;
|
|
1354
|
+
// Fast dispatch bypasses the pending-batch ingress, so preserve both
|
|
1355
|
+
// of its contracts here: only selected routes may run, and V2 sees
|
|
1356
|
+
// the same dirty signal as the compatibility drain.
|
|
1357
|
+
realtimeState.scheduler?.signal("watcher");
|
|
1358
|
+
void runScopedDrain(selectedBatch, generation, true).then((exit) => {
|
|
1359
|
+
if (exit === 0)
|
|
1360
|
+
return;
|
|
1361
|
+
restorePendingWatcherPaths(selectedBatch.paths, generation);
|
|
1362
|
+
interruptWait();
|
|
1363
|
+
}).catch((err) => {
|
|
1364
|
+
restorePendingWatcherPaths(selectedBatch.paths, generation);
|
|
1365
|
+
process.stderr.write(`watch fast-lane scoped push failed, will retry next tick: ${describeError(err)}\n`);
|
|
1366
|
+
interruptWait();
|
|
1367
|
+
});
|
|
1368
|
+
return;
|
|
1369
|
+
}
|
|
1149
1370
|
if (addPendingWatcherChange(changedRelPath, batch)) {
|
|
1150
1371
|
// The V1 watcher batch keeps its existing compatibility behavior. A
|
|
1151
1372
|
// V2-enrolled scope observes the same dirty hint through its own
|
|
@@ -1178,6 +1399,30 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1178
1399
|
// initial walk will finish.
|
|
1179
1400
|
watcherRediscoveryRetries = 0;
|
|
1180
1401
|
process.stderr.write(`hq-sync-runner: event-push watcher active (watched-paths=${watchedPaths ?? "unknown"})\n`);
|
|
1402
|
+
const startedAt = Date.now();
|
|
1403
|
+
try {
|
|
1404
|
+
const catchup = watcher.collectWarmupCatchup?.(serviceStartedAt - WATCHER_WARMUP_SAFETY_MARGIN_MS, warmupJournalPaths());
|
|
1405
|
+
if (!catchup)
|
|
1406
|
+
return;
|
|
1407
|
+
const queued = addPendingWatcherChange(undefined, catchup.batch);
|
|
1408
|
+
pendingWatcherWarmupCatchup ||= queued;
|
|
1409
|
+
process.stderr.write(JSON.stringify({
|
|
1410
|
+
type: "watcher-warmup-catchup",
|
|
1411
|
+
reason: "watch-set-ready",
|
|
1412
|
+
pathsFound: catchup.batch.paths.size,
|
|
1413
|
+
scannedPaths: catchup.scannedPaths,
|
|
1414
|
+
elapsedMs: Date.now() - startedAt,
|
|
1415
|
+
queued,
|
|
1416
|
+
}) + "\n");
|
|
1417
|
+
if (queued)
|
|
1418
|
+
interruptWait();
|
|
1419
|
+
}
|
|
1420
|
+
catch (err) {
|
|
1421
|
+
process.stderr.write(`hq-sync-runner: watcher warm-up catch-up failed: ${describeError(err)}\n`);
|
|
1422
|
+
// A scan failure must not silently extend the warm-up blind spot.
|
|
1423
|
+
pendingWatcherWarmupCatchupFailed = true;
|
|
1424
|
+
interruptWait();
|
|
1425
|
+
}
|
|
1181
1426
|
};
|
|
1182
1427
|
if (watcher.onReady)
|
|
1183
1428
|
watcher.onReady(reportWatcherReady);
|
|
@@ -1215,20 +1460,27 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1215
1460
|
// lock.
|
|
1216
1461
|
interruptWait();
|
|
1217
1462
|
const pathsByRoute = new Map();
|
|
1463
|
+
let eligibleEventCount = 0;
|
|
1218
1464
|
for (const event of events) {
|
|
1219
1465
|
const route = routeChangeToTarget(event.relativePath);
|
|
1220
1466
|
if (!route)
|
|
1221
1467
|
continue;
|
|
1468
|
+
eligibleEventCount += 1;
|
|
1222
1469
|
const key = routeKey(route);
|
|
1223
1470
|
const group = pathsByRoute.get(key) ?? { route, paths: [] };
|
|
1224
1471
|
group.paths.push(event.relativePath);
|
|
1225
1472
|
pathsByRoute.set(key, group);
|
|
1226
1473
|
}
|
|
1474
|
+
// One receiver delivery can fan out to many routes. Apply the advertised
|
|
1475
|
+
// cap to its whole eligible payload, not independently to each route.
|
|
1476
|
+
const lane = isFastReceiverBatch(eligibleEventCount)
|
|
1477
|
+
? "fast"
|
|
1478
|
+
: "slow";
|
|
1227
1479
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
1228
1480
|
// One guarded targeted pass per affected route, with every coalesced
|
|
1229
1481
|
// path for that route expressed through the watcher's existing argv seam.
|
|
1230
1482
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
1231
|
-
const result = await runGuarded(targetedArgv);
|
|
1483
|
+
const result = await runGuarded(targetedArgv, undefined, lane);
|
|
1232
1484
|
if (passExitCode(result) !== 0) {
|
|
1233
1485
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
1234
1486
|
}
|
|
@@ -1246,10 +1498,23 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1246
1498
|
void deps.startRealtimeScheduler({
|
|
1247
1499
|
hqRoot,
|
|
1248
1500
|
runGuarded: async (drain) => {
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1501
|
+
let outcome;
|
|
1502
|
+
try {
|
|
1503
|
+
outcome = await runGuardedTask(() => runTaskWithLock(async () => {
|
|
1504
|
+
await drain();
|
|
1505
|
+
return 0;
|
|
1506
|
+
}));
|
|
1507
|
+
}
|
|
1508
|
+
catch (err) {
|
|
1509
|
+
// Scheduler clients may have already been disposed when a queued
|
|
1510
|
+
// drain finally reaches the lock. Surface the late failure, but do
|
|
1511
|
+
// not leak an unhandled rejection after shutdown has completed.
|
|
1512
|
+
if (stopped) {
|
|
1513
|
+
process.stderr.write("realtime scheduler: cancelled drain failed after shutdown\n");
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
throw err;
|
|
1517
|
+
}
|
|
1253
1518
|
if (passExitCode(outcome) !== 0) {
|
|
1254
1519
|
throw new Error(`realtime scheduler drain exited ${passExitCode(outcome)}`);
|
|
1255
1520
|
}
|
|
@@ -1391,6 +1656,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1391
1656
|
const eventPushSurfacesInactive = eventPush && !eventPushSurfacesActivated;
|
|
1392
1657
|
const firstTick = pollTick === 1;
|
|
1393
1658
|
const bareWake = pending.bare;
|
|
1659
|
+
const watcherWarmupCatchup = pending.warmupCatchup;
|
|
1660
|
+
const watcherWarmupCatchupFailed = pending.warmupCatchupFailed;
|
|
1394
1661
|
const overflowRoutesUnknown = pending.batch?.overflowed === true && !overflowWithKnownRoutes;
|
|
1395
1662
|
const scheduledInterval = fullReconcileIntervalMs === undefined
|
|
1396
1663
|
? pollTick % FULL_RECONCILE_EVERY_TICKS === 0
|
|
@@ -1402,6 +1669,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1402
1669
|
"bare-wake": bareWake,
|
|
1403
1670
|
"batch-over-limit": batchOverLimit,
|
|
1404
1671
|
"overflow-routes-unknown": overflowRoutesUnknown,
|
|
1672
|
+
"watcher-warmup-catch-up": watcherWarmupCatchup,
|
|
1673
|
+
"watcher-warmup-catch-up-failed": watcherWarmupCatchupFailed,
|
|
1405
1674
|
"scheduled-interval": scheduledInterval,
|
|
1406
1675
|
};
|
|
1407
1676
|
const reasons = Object.entries(triggers)
|
|
@@ -1412,6 +1681,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1412
1681
|
bareWake ||
|
|
1413
1682
|
batchOverLimit ||
|
|
1414
1683
|
overflowRoutesUnknown ||
|
|
1684
|
+
watcherWarmupCatchupFailed ||
|
|
1415
1685
|
scheduledInterval;
|
|
1416
1686
|
const emitDecision = (mode, decisionReasons) => {
|
|
1417
1687
|
emitFullReconcileDecision({
|
|
@@ -1447,7 +1717,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1447
1717
|
// were dropped); ordinary batches remain path-scoped. Label those
|
|
1448
1718
|
// distinct scopes so operators can distinguish them from a full
|
|
1449
1719
|
// all-vault reconcile.
|
|
1450
|
-
emitDecision(
|
|
1720
|
+
emitDecision(watcherWarmupCatchup
|
|
1721
|
+
? "catch-up"
|
|
1722
|
+
: overflowWithKnownRoutes
|
|
1723
|
+
? "route"
|
|
1724
|
+
: "scoped", reasons);
|
|
1451
1725
|
}
|
|
1452
1726
|
catch (err) {
|
|
1453
1727
|
// Never silently drop a change: any failure inside the targeted
|