@indigoai-us/hq-cloud 6.16.82-alpha.0 → 6.16.84-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +118 -71
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +507 -248
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +50 -9
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +224 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +78 -27
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +107 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +15 -7
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +148 -15
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +321 -2
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/full-pass-interval.d.ts +25 -0
- package/dist/sync/full-pass-interval.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.js +53 -0
- package/dist/sync/full-pass-interval.js.map +1 -0
- package/dist/sync/full-pass-interval.test.d.ts +2 -0
- package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.test.js +21 -0
- package/dist/sync/full-pass-interval.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +0 -14
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +12 -49
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +7 -171
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -16,11 +16,12 @@ import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
|
16
16
|
import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
|
|
17
17
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
18
18
|
import { AUTH_WAIT_INITIAL_RECHECK_MS, AUTH_WAIT_RESUME_MESSAGE, AUTH_WAIT_START_MESSAGE, AUTH_WAIT_STEADY_RECHECK_MS, } from "./sync-runner-auth-wait.js";
|
|
19
|
-
import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
|
|
19
|
+
import { ObjectLockChecksumRequiredError, PresignObjectIO, S3SdkObjectIO, } from "../object-io.js";
|
|
20
20
|
import { setBandwidthGovernorForTesting } from "../bandwidth.js";
|
|
21
21
|
import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
22
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
23
|
-
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES,
|
|
23
|
+
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
24
|
+
import { DEFAULT_FULL_PASS_INTERVAL_MS } from "../sync/full-pass-interval.js";
|
|
24
25
|
import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
|
|
25
26
|
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
26
27
|
import { AreaLedger } from "../sync/area-ledger.js";
|
|
@@ -565,6 +566,79 @@ describe("mid-pass identity refusal", () => {
|
|
|
565
566
|
expect(deps.stderr.events().some((e) => e.type === "auth-error")).toBe(false);
|
|
566
567
|
});
|
|
567
568
|
});
|
|
569
|
+
describe("remote failure contract", () => {
|
|
570
|
+
const person = {
|
|
571
|
+
uid: "ent_person_remote_failure_contract",
|
|
572
|
+
type: "person",
|
|
573
|
+
bucketName: "hq-vault-personal-contract",
|
|
574
|
+
status: "active",
|
|
575
|
+
};
|
|
576
|
+
it.each([
|
|
577
|
+
[
|
|
578
|
+
"terminal 403 refusal",
|
|
579
|
+
() => new CognitoRefreshError("session reauth required (403)", true, 403),
|
|
580
|
+
0,
|
|
581
|
+
"auth-error",
|
|
582
|
+
"quick refresh",
|
|
583
|
+
],
|
|
584
|
+
[
|
|
585
|
+
"404 route",
|
|
586
|
+
() => new VaultNotFoundError("remote route no longer exists (404)"),
|
|
587
|
+
PARTIAL_SYNC_EXIT,
|
|
588
|
+
"error",
|
|
589
|
+
"404",
|
|
590
|
+
],
|
|
591
|
+
[
|
|
592
|
+
"429 throttle",
|
|
593
|
+
() => Object.assign(new Error("remote throttled (429)"), { statusCode: 429 }),
|
|
594
|
+
TRANSIENT_NETWORK_EXIT,
|
|
595
|
+
"transient-network",
|
|
596
|
+
"429",
|
|
597
|
+
],
|
|
598
|
+
[
|
|
599
|
+
"503 throttle",
|
|
600
|
+
() => Object.assign(new Error("remote unavailable (503)"), { statusCode: 503 }),
|
|
601
|
+
TRANSIENT_NETWORK_EXIT,
|
|
602
|
+
"transient-network",
|
|
603
|
+
"503",
|
|
604
|
+
],
|
|
605
|
+
[
|
|
606
|
+
"Object Lock checksum refusal",
|
|
607
|
+
() => new ObjectLockChecksumRequiredError("remote.md", "checksum required"),
|
|
608
|
+
PARTIAL_SYNC_EXIT,
|
|
609
|
+
"error",
|
|
610
|
+
"ObjectLockChecksumRequired",
|
|
611
|
+
],
|
|
612
|
+
[
|
|
613
|
+
"timeout",
|
|
614
|
+
() => Object.assign(new Error("remote timed out"), { name: "TimeoutError" }),
|
|
615
|
+
TRANSIENT_NETWORK_EXIT,
|
|
616
|
+
"transient-network",
|
|
617
|
+
"timed out",
|
|
618
|
+
],
|
|
619
|
+
[
|
|
620
|
+
"abort",
|
|
621
|
+
() => Object.assign(new Error("remote aborted"), { name: "AbortError" }),
|
|
622
|
+
TRANSIENT_NETWORK_EXIT,
|
|
623
|
+
"transient-network",
|
|
624
|
+
"aborted",
|
|
625
|
+
],
|
|
626
|
+
])("keeps the journal stable and records the runner outcome for %s", async (_description, makeFailure, expectedExit, telemetryType, telemetryReason) => {
|
|
627
|
+
const before = readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
628
|
+
const deps = makeDeps({
|
|
629
|
+
sync: vi.fn().mockRejectedValue(makeFailure()),
|
|
630
|
+
createVaultClient: () => makeVaultStub({ listPersons: () => Promise.resolve([person]) }),
|
|
631
|
+
});
|
|
632
|
+
expect(await runRunner(["--personal"], deps)).toBe(expectedExit);
|
|
633
|
+
expect(readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files).toEqual(before.files);
|
|
634
|
+
const events = [...deps.stdout.events(), ...deps.stderr.events()];
|
|
635
|
+
expect(events.some((event) => event.type === telemetryType)).toBe(true);
|
|
636
|
+
expect(`${deps.stdout.raw()}${deps.stderr.raw()}`).toContain(telemetryReason);
|
|
637
|
+
if (telemetryType === "transient-network") {
|
|
638
|
+
expect(events.some((event) => event.type === "error")).toBe(false);
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
});
|
|
568
642
|
// ---------------------------------------------------------------------------
|
|
569
643
|
// memberships retry (3x with backoff, then abort)
|
|
570
644
|
// ---------------------------------------------------------------------------
|
|
@@ -4636,28 +4710,13 @@ describe("advanceMonotonicDeadline", () => {
|
|
|
4636
4710
|
});
|
|
4637
4711
|
});
|
|
4638
4712
|
});
|
|
4639
|
-
describe("watch-loop reconcile
|
|
4713
|
+
describe("watch-loop reconcile configuration", () => {
|
|
4640
4714
|
it("uses the shipped 10,000 event-batch default while allowing smaller and larger overrides", () => {
|
|
4641
4715
|
expect(DEFAULT_EVENT_BATCH_LIMIT).toBe(10_000);
|
|
4642
4716
|
expect(resolveEventBatchLimit({})).toBe(10_000);
|
|
4643
4717
|
expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "5" })).toBe(5);
|
|
4644
4718
|
expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "50000" })).toBe(50_000);
|
|
4645
4719
|
});
|
|
4646
|
-
it("defaults the full-reconcile cadence to one elapsed hour for an unset or invalid interval", () => {
|
|
4647
|
-
// The legacy default was "every tenth tick" with no elapsed-time floor.
|
|
4648
|
-
// Ticks fire on every watcher batch, not just polls, so on an active
|
|
4649
|
-
// vault that meant a full pass every 4-8 minutes — and each pass on a
|
|
4650
|
-
// real install lasts ~10 minutes. Full reconcile is the correctness
|
|
4651
|
-
// backstop behind event-push, so elapsed time is the right clock.
|
|
4652
|
-
expect(DEFAULT_FULL_RECONCILE_MS).toBe(3_600_000);
|
|
4653
|
-
expect(resolveFullReconcileMs({})).toBe(DEFAULT_FULL_RECONCILE_MS);
|
|
4654
|
-
for (const raw of ["", "0", "0.5", "-1", "NaN", "not-a-number", "Infinity"]) {
|
|
4655
|
-
expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: raw })).toBe(DEFAULT_FULL_RECONCILE_MS);
|
|
4656
|
-
}
|
|
4657
|
-
});
|
|
4658
|
-
it("accepts a positive millisecond full-reconcile interval", () => {
|
|
4659
|
-
expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: "10800000" })).toBe(10_800_000);
|
|
4660
|
-
});
|
|
4661
4720
|
});
|
|
4662
4721
|
describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", () => {
|
|
4663
4722
|
// Capture the ms handed to sleep on the FIRST poll, then hang so the loop
|
|
@@ -4873,10 +4932,8 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
4873
4932
|
it("parallel coordinator activates every event surface after a partial completed full reconcile", async () => {
|
|
4874
4933
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
4875
4934
|
const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
|
|
4876
|
-
const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4877
4935
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
4878
4936
|
process.env.HQ_SYNC_EVENT_SYNC = "1";
|
|
4879
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "60000";
|
|
4880
4937
|
const watcher = makeWatcherStub();
|
|
4881
4938
|
const receiver = {
|
|
4882
4939
|
connected: false,
|
|
@@ -4936,19 +4993,13 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
4936
4993
|
delete process.env.HQ_SYNC_EVENT_SYNC;
|
|
4937
4994
|
else
|
|
4938
4995
|
process.env.HQ_SYNC_EVENT_SYNC = previousEventSync;
|
|
4939
|
-
if (previousFullReconcileMs === undefined)
|
|
4940
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4941
|
-
else
|
|
4942
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
|
|
4943
4996
|
}
|
|
4944
4997
|
});
|
|
4945
4998
|
it("parallel coordinator activates event surfaces and advances cadence after a numeric-zero full reconcile", async () => {
|
|
4946
4999
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
4947
5000
|
const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
|
|
4948
|
-
const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4949
5001
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
4950
5002
|
process.env.HQ_SYNC_EVENT_SYNC = "1";
|
|
4951
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "60000";
|
|
4952
5003
|
const watcher = makeWatcherStub();
|
|
4953
5004
|
const receiver = {
|
|
4954
5005
|
connected: false,
|
|
@@ -5008,10 +5059,55 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5008
5059
|
delete process.env.HQ_SYNC_EVENT_SYNC;
|
|
5009
5060
|
else
|
|
5010
5061
|
process.env.HQ_SYNC_EVENT_SYNC = previousEventSync;
|
|
5011
|
-
|
|
5012
|
-
|
|
5062
|
+
}
|
|
5063
|
+
});
|
|
5064
|
+
it("backs off failed parallel full-reconcile completions before another chore is admitted", async () => {
|
|
5065
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5066
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
5067
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
5068
|
+
const delays = [];
|
|
5069
|
+
const waiters = [];
|
|
5070
|
+
let now = 0;
|
|
5071
|
+
let shutdown = () => { };
|
|
5072
|
+
const sleep = (ms) => {
|
|
5073
|
+
delays.push(ms);
|
|
5074
|
+
return new Promise((resolve) => {
|
|
5075
|
+
waiters.push(resolve);
|
|
5076
|
+
});
|
|
5077
|
+
};
|
|
5078
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
5079
|
+
runPass,
|
|
5080
|
+
clock: new FakeClock(),
|
|
5081
|
+
monotonicNow: () => now,
|
|
5082
|
+
sleep,
|
|
5083
|
+
fullPassIntervalSource: { read: () => 1 },
|
|
5084
|
+
onShutdownSignal: (handler) => {
|
|
5085
|
+
shutdown = handler;
|
|
5086
|
+
return () => { };
|
|
5087
|
+
},
|
|
5088
|
+
});
|
|
5089
|
+
try {
|
|
5090
|
+
for (let attempt = 0; attempt < 50 && runPass.mock.calls.length < 3; attempt += 1) {
|
|
5091
|
+
const resolve = waiters.shift();
|
|
5092
|
+
if (resolve) {
|
|
5093
|
+
now += 1;
|
|
5094
|
+
resolve();
|
|
5095
|
+
await flushLoopMicrotasks(30);
|
|
5096
|
+
}
|
|
5097
|
+
else {
|
|
5098
|
+
await new Promise((resolveNext) => setImmediate(resolveNext));
|
|
5099
|
+
}
|
|
5100
|
+
}
|
|
5101
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
5102
|
+
expect(delays).toContain(60_000);
|
|
5103
|
+
}
|
|
5104
|
+
finally {
|
|
5105
|
+
shutdown();
|
|
5106
|
+
await expect(loop).resolves.toBe(0);
|
|
5107
|
+
if (previousCoordinator === undefined)
|
|
5108
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5013
5109
|
else
|
|
5014
|
-
process.env.
|
|
5110
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
5015
5111
|
}
|
|
5016
5112
|
});
|
|
5017
5113
|
it("parallel coordinator keeps event surfaces off for an auth-required full reconcile", async () => {
|
|
@@ -5144,7 +5240,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5144
5240
|
sleep.tick();
|
|
5145
5241
|
await flushLoopMicrotasks();
|
|
5146
5242
|
}
|
|
5147
|
-
clockState.now =
|
|
5243
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
5148
5244
|
sleep.tick();
|
|
5149
5245
|
await flushLoopMicrotasks();
|
|
5150
5246
|
// Tick 1 and the first tick after an hour has elapsed are full
|
|
@@ -5212,7 +5308,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5212
5308
|
}
|
|
5213
5309
|
expect(runPass).toHaveBeenCalledTimes(1);
|
|
5214
5310
|
// The hourly cadence, not a tick count, schedules the next full pass.
|
|
5215
|
-
clockState.now =
|
|
5311
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
5216
5312
|
sleep.tick();
|
|
5217
5313
|
await flushLoopMicrotasks(30);
|
|
5218
5314
|
expect(runPass).toHaveBeenCalledTimes(2);
|
|
@@ -6135,7 +6231,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6135
6231
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6136
6232
|
}
|
|
6137
6233
|
});
|
|
6138
|
-
it("
|
|
6234
|
+
it("acknowledges superseded receiver work and records refused telemetry", async () => {
|
|
6139
6235
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6140
6236
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6141
6237
|
const watcher = makeBatchWatcherStub();
|
|
@@ -6189,7 +6285,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6189
6285
|
events: [event("companies/indigo/superseded.md")],
|
|
6190
6286
|
signal: new AbortController().signal,
|
|
6191
6287
|
});
|
|
6192
|
-
await expect(superseded).
|
|
6288
|
+
await expect(superseded).resolves.toBeUndefined();
|
|
6193
6289
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
|
|
6194
6290
|
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":3,"source":"receiver","direction":"pull","outcome":"refused"');
|
|
6195
6291
|
releaseHeld();
|
|
@@ -6207,6 +6303,124 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6207
6303
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6208
6304
|
}
|
|
6209
6305
|
});
|
|
6306
|
+
it("measures a queued realtime pass at the operation gate and marks append lock only after admission", async () => {
|
|
6307
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6308
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6309
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-telemetry-operation-gate-"));
|
|
6310
|
+
const watcher = makeBatchWatcherStub();
|
|
6311
|
+
let receiverSync;
|
|
6312
|
+
let shutdown = () => { };
|
|
6313
|
+
let releaseGuard = () => { };
|
|
6314
|
+
const guard = new Promise((resolve) => { releaseGuard = resolve; });
|
|
6315
|
+
let markGuardStarted = () => { };
|
|
6316
|
+
const guardStarted = new Promise((resolve) => { markGuardStarted = resolve; });
|
|
6317
|
+
let markTargetStarted = () => { };
|
|
6318
|
+
const targetStarted = new Promise((resolve) => { markTargetStarted = resolve; });
|
|
6319
|
+
let monotonicMs = 0;
|
|
6320
|
+
let guardReleased = false;
|
|
6321
|
+
let targetStartedAt;
|
|
6322
|
+
const timings = [];
|
|
6323
|
+
const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
6324
|
+
if (typeof chunk === "string" && chunk.startsWith("{")) {
|
|
6325
|
+
try {
|
|
6326
|
+
const record = JSON.parse(chunk);
|
|
6327
|
+
if (typeof record === "object" &&
|
|
6328
|
+
record !== null &&
|
|
6329
|
+
record.event === "sync.realtime.timing")
|
|
6330
|
+
timings.push(record);
|
|
6331
|
+
}
|
|
6332
|
+
catch {
|
|
6333
|
+
// Non-telemetry diagnostics are outside this assertion.
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
return true;
|
|
6337
|
+
});
|
|
6338
|
+
const includesScope = (argv, basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
|
|
6339
|
+
const runPass = vi.fn(async (argv) => {
|
|
6340
|
+
if (includesScope(argv, "guard.md")) {
|
|
6341
|
+
markGuardStarted();
|
|
6342
|
+
await guard;
|
|
6343
|
+
return completedPassOutcome();
|
|
6344
|
+
}
|
|
6345
|
+
if (includesScope(argv, "target.md")) {
|
|
6346
|
+
expect(guardReleased).toBe(true);
|
|
6347
|
+
targetStartedAt = monotonicMs;
|
|
6348
|
+
markTargetStarted();
|
|
6349
|
+
monotonicMs += 1;
|
|
6350
|
+
}
|
|
6351
|
+
return completedPassOutcome();
|
|
6352
|
+
});
|
|
6353
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6354
|
+
runPass,
|
|
6355
|
+
createWatcher: () => watcher,
|
|
6356
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
6357
|
+
receiverSync = syncBatchFn;
|
|
6358
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6359
|
+
},
|
|
6360
|
+
monotonicNow: () => monotonicMs,
|
|
6361
|
+
sleep: () => new Promise(() => { }),
|
|
6362
|
+
onShutdownSignal: (handler) => {
|
|
6363
|
+
shutdown = handler;
|
|
6364
|
+
return () => { };
|
|
6365
|
+
},
|
|
6366
|
+
});
|
|
6367
|
+
const event = (relativePath) => ({
|
|
6368
|
+
kind: "upsert",
|
|
6369
|
+
relativePath,
|
|
6370
|
+
contentHash: "sha256:remote",
|
|
6371
|
+
mtime: "2026-09-23T00:00:00.000Z",
|
|
6372
|
+
originDeviceId: "peer-device",
|
|
6373
|
+
originTenantId: "tenant-indigo",
|
|
6374
|
+
sequenceNumber: 1,
|
|
6375
|
+
eventTimestamp: "2026-09-23T00:00:00.000Z",
|
|
6376
|
+
});
|
|
6377
|
+
try {
|
|
6378
|
+
await flushLoopMicrotasks(30);
|
|
6379
|
+
runPass.mockClear();
|
|
6380
|
+
const guardedPass = receiverSync({
|
|
6381
|
+
events: [event("companies/alpha/guard.md")],
|
|
6382
|
+
signal: new AbortController().signal,
|
|
6383
|
+
});
|
|
6384
|
+
await guardStarted;
|
|
6385
|
+
const targetAbsolutePath = path.join(hqRoot, "companies", "beta", "target.md");
|
|
6386
|
+
fs.mkdirSync(path.dirname(targetAbsolutePath), { recursive: true });
|
|
6387
|
+
fs.writeFileSync(targetAbsolutePath, "x");
|
|
6388
|
+
watcher.emit("companies/beta/target.md", {
|
|
6389
|
+
paths: new Map([
|
|
6390
|
+
[targetAbsolutePath, "companies/beta/target.md"],
|
|
6391
|
+
]),
|
|
6392
|
+
changes: new Map(),
|
|
6393
|
+
});
|
|
6394
|
+
await flushLoopMicrotasks(30);
|
|
6395
|
+
expect(targetStartedAt).toBeUndefined();
|
|
6396
|
+
monotonicMs += 25;
|
|
6397
|
+
guardReleased = true;
|
|
6398
|
+
releaseGuard();
|
|
6399
|
+
await guardedPass;
|
|
6400
|
+
await targetStarted;
|
|
6401
|
+
await flushLoopMicrotasks(30);
|
|
6402
|
+
expect(targetStartedAt).toBe(25);
|
|
6403
|
+
expect(timings).toHaveLength(2);
|
|
6404
|
+
const targetTiming = timings.find((record) => record.source === "watcher" &&
|
|
6405
|
+
record.direction === "push" &&
|
|
6406
|
+
record.outcome === "durable");
|
|
6407
|
+
expect(targetTiming).toBeDefined();
|
|
6408
|
+
expect(targetTiming?.timingMs?.operationGate).toBe(25);
|
|
6409
|
+
expect(targetTiming?.timingMs?.appendLock).toBe(0);
|
|
6410
|
+
}
|
|
6411
|
+
finally {
|
|
6412
|
+
guardReleased = true;
|
|
6413
|
+
releaseGuard();
|
|
6414
|
+
shutdown();
|
|
6415
|
+
await loop;
|
|
6416
|
+
stderr.mockRestore();
|
|
6417
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6418
|
+
if (previousCoordinator === undefined)
|
|
6419
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6420
|
+
else
|
|
6421
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6422
|
+
}
|
|
6423
|
+
});
|
|
6210
6424
|
it("REGRESSION: a fast receiver pull overtakes a slow watcher push and records admitted occupancy", async () => {
|
|
6211
6425
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6212
6426
|
const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
|
|
@@ -8898,6 +9112,137 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
8898
9112
|
await expect(loop).resolves.toBe(0);
|
|
8899
9113
|
expect(watcher.disposed).toBe(false);
|
|
8900
9114
|
});
|
|
9115
|
+
it("bounds repeated watch-pass exits with a fake-clock retry budget while keeping the runner alive", async () => {
|
|
9116
|
+
const watcher = makeWatcherStub();
|
|
9117
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
9118
|
+
const delays = [];
|
|
9119
|
+
const waiters = [];
|
|
9120
|
+
let now = 0;
|
|
9121
|
+
let shutdown = () => { };
|
|
9122
|
+
let settled;
|
|
9123
|
+
const sleep = (ms) => {
|
|
9124
|
+
delays.push(ms);
|
|
9125
|
+
return new Promise((resolve) => {
|
|
9126
|
+
waiters.push(resolve);
|
|
9127
|
+
});
|
|
9128
|
+
};
|
|
9129
|
+
const tick = async (advanceMs) => {
|
|
9130
|
+
now += advanceMs;
|
|
9131
|
+
const resolve = waiters.shift();
|
|
9132
|
+
if (!resolve)
|
|
9133
|
+
throw new Error("retry sleep was not pending");
|
|
9134
|
+
resolve();
|
|
9135
|
+
await flushLoopMicrotasks();
|
|
9136
|
+
};
|
|
9137
|
+
const waitForRetrySleep = async () => {
|
|
9138
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
9139
|
+
if (waiters.length > 0)
|
|
9140
|
+
return;
|
|
9141
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
9142
|
+
}
|
|
9143
|
+
throw new Error("timed out waiting for retry sleep");
|
|
9144
|
+
};
|
|
9145
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
9146
|
+
runPass,
|
|
9147
|
+
clock: new FakeClock(),
|
|
9148
|
+
createWatcher: () => watcher,
|
|
9149
|
+
monotonicNow: () => now,
|
|
9150
|
+
sleep,
|
|
9151
|
+
onShutdownSignal: (handler) => {
|
|
9152
|
+
shutdown = handler;
|
|
9153
|
+
return () => { };
|
|
9154
|
+
},
|
|
9155
|
+
}).then((code) => {
|
|
9156
|
+
settled = code;
|
|
9157
|
+
return code;
|
|
9158
|
+
});
|
|
9159
|
+
try {
|
|
9160
|
+
await waitForRetrySleep();
|
|
9161
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9162
|
+
expect(delays).toEqual([1]);
|
|
9163
|
+
await tick(1_000);
|
|
9164
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9165
|
+
expect(delays).toEqual([1, 1]);
|
|
9166
|
+
await tick(1_000);
|
|
9167
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9168
|
+
expect(delays).toEqual([1, 1, 60_000]);
|
|
9169
|
+
await tick(60_000);
|
|
9170
|
+
await tick(120_000);
|
|
9171
|
+
await tick(240_000);
|
|
9172
|
+
await tick(480_000);
|
|
9173
|
+
expect(runPass).toHaveBeenCalledTimes(7);
|
|
9174
|
+
expect(delays).toEqual([
|
|
9175
|
+
1,
|
|
9176
|
+
1,
|
|
9177
|
+
60_000,
|
|
9178
|
+
120_000,
|
|
9179
|
+
240_000,
|
|
9180
|
+
480_000,
|
|
9181
|
+
600_000,
|
|
9182
|
+
]);
|
|
9183
|
+
// The retry window is quiet-time based, not anchored to the first
|
|
9184
|
+
// failure. Continuing failures after thirty minutes must remain capped.
|
|
9185
|
+
await tick(600_000);
|
|
9186
|
+
await tick(600_000);
|
|
9187
|
+
expect(runPass).toHaveBeenCalledTimes(9);
|
|
9188
|
+
expect(delays.slice(-2)).toEqual([600_000, 600_000]);
|
|
9189
|
+
expect(settled).toBeUndefined();
|
|
9190
|
+
expect(watcher.disposed).toBe(false);
|
|
9191
|
+
}
|
|
9192
|
+
finally {
|
|
9193
|
+
shutdown();
|
|
9194
|
+
await expect(loop).resolves.toBe(0);
|
|
9195
|
+
}
|
|
9196
|
+
});
|
|
9197
|
+
it("does not let a watcher interrupt bypass a failed-pass retry deadline", async () => {
|
|
9198
|
+
const watcher = makeWatcherStub();
|
|
9199
|
+
let passes = 0;
|
|
9200
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
9201
|
+
passes += 1;
|
|
9202
|
+
return passes === 1 ? 0 : 1;
|
|
9203
|
+
});
|
|
9204
|
+
const waiters = [];
|
|
9205
|
+
const now = 0;
|
|
9206
|
+
let shutdown = () => { };
|
|
9207
|
+
const sleep = vi.fn(() => new Promise((resolve) => {
|
|
9208
|
+
waiters.push(resolve);
|
|
9209
|
+
}));
|
|
9210
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
9211
|
+
runPass,
|
|
9212
|
+
clock: new FakeClock(),
|
|
9213
|
+
createWatcher: () => watcher,
|
|
9214
|
+
monotonicNow: () => now,
|
|
9215
|
+
sleep,
|
|
9216
|
+
onShutdownSignal: (handler) => {
|
|
9217
|
+
shutdown = handler;
|
|
9218
|
+
return () => { };
|
|
9219
|
+
},
|
|
9220
|
+
});
|
|
9221
|
+
try {
|
|
9222
|
+
await waitForPollSleep({
|
|
9223
|
+
pendingCount: () => waiters.length,
|
|
9224
|
+
});
|
|
9225
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9226
|
+
watcher.emit();
|
|
9227
|
+
await flushLoopMicrotasks(30);
|
|
9228
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9229
|
+
watcher.emit();
|
|
9230
|
+
await flushLoopMicrotasks(30);
|
|
9231
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9232
|
+
watcher.emit();
|
|
9233
|
+
await flushLoopMicrotasks(30);
|
|
9234
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9235
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9236
|
+
watcher.emit();
|
|
9237
|
+
await flushLoopMicrotasks(30);
|
|
9238
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9239
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9240
|
+
}
|
|
9241
|
+
finally {
|
|
9242
|
+
shutdown();
|
|
9243
|
+
await expect(loop).resolves.toBe(0);
|
|
9244
|
+
}
|
|
9245
|
+
});
|
|
8901
9246
|
it("alerts when a partial pass failed before a company's download leg", async () => {
|
|
8902
9247
|
const watcher = makeWatcherStub();
|
|
8903
9248
|
const stderrWrite = vi
|
|
@@ -8932,7 +9277,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
8932
9277
|
runPass,
|
|
8933
9278
|
clock: new FakeClock(),
|
|
8934
9279
|
createWatcher: () => watcher,
|
|
8935
|
-
monotonicNow: () => (monotonic +=
|
|
9280
|
+
monotonicNow: () => (monotonic += DEFAULT_FULL_PASS_INTERVAL_MS),
|
|
8936
9281
|
sleep: sleep.sleep,
|
|
8937
9282
|
onShutdownSignal: (handler) => {
|
|
8938
9283
|
triggerShutdown = handler;
|
|
@@ -9914,9 +10259,7 @@ describe("CooperativePassScheduler", () => {
|
|
|
9914
10259
|
describe("first-tick realtime yield", () => {
|
|
9915
10260
|
it("commits a watcher push at the first chore checkpoint without a wall-clock wait", async () => {
|
|
9916
10261
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
9917
|
-
const previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
9918
10262
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
9919
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
9920
10263
|
const timings = [];
|
|
9921
10264
|
const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
9922
10265
|
const text = String(chunk);
|
|
@@ -10015,10 +10358,6 @@ describe("first-tick realtime yield", () => {
|
|
|
10015
10358
|
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10016
10359
|
else
|
|
10017
10360
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
10018
|
-
if (previousInterval === undefined)
|
|
10019
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10020
|
-
else
|
|
10021
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
|
|
10022
10361
|
}
|
|
10023
10362
|
});
|
|
10024
10363
|
});
|
|
@@ -10313,8 +10652,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10313
10652
|
capture.restore();
|
|
10314
10653
|
});
|
|
10315
10654
|
it("logs event-push-surfaces-inactive without silently attributing the first tick", async () => {
|
|
10316
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10317
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10318
10655
|
const capture = captureFullReconcileDecisions();
|
|
10319
10656
|
try {
|
|
10320
10657
|
const sleep = makeSteppableSleep();
|
|
@@ -10343,10 +10680,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10343
10680
|
}
|
|
10344
10681
|
finally {
|
|
10345
10682
|
capture.restore();
|
|
10346
|
-
if (saved === undefined)
|
|
10347
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10348
|
-
else
|
|
10349
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10350
10683
|
}
|
|
10351
10684
|
});
|
|
10352
10685
|
it("reports every matching trigger instead of short-circuiting at the first", async () => {
|
|
@@ -10401,9 +10734,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10401
10734
|
});
|
|
10402
10735
|
it("skips a 404 company route in a mixed drain and records a completed full reconcile", async () => {
|
|
10403
10736
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10404
|
-
const previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10405
10737
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
10406
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10407
10738
|
const failedLines = [];
|
|
10408
10739
|
const decisions = [];
|
|
10409
10740
|
const timingRecords = [];
|
|
@@ -10503,17 +10834,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10503
10834
|
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10504
10835
|
else
|
|
10505
10836
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
10506
|
-
if (previousInterval === undefined)
|
|
10507
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10508
|
-
else
|
|
10509
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
|
|
10510
10837
|
}
|
|
10511
10838
|
});
|
|
10512
10839
|
it("does not escalate a fully-enumerated over-limit batch to a full reconcile", async () => {
|
|
10513
10840
|
const savedLimit = process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10514
|
-
const savedInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10515
10841
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = "2";
|
|
10516
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10517
10842
|
const journal = captureWatchLoopJournal();
|
|
10518
10843
|
try {
|
|
10519
10844
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -10553,17 +10878,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10553
10878
|
delete process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10554
10879
|
else
|
|
10555
10880
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = savedLimit;
|
|
10556
|
-
if (savedInterval === undefined)
|
|
10557
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10558
|
-
else
|
|
10559
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = savedInterval;
|
|
10560
10881
|
}
|
|
10561
10882
|
});
|
|
10562
10883
|
it("still escalates an overflowed batch with unknown routes to a full reconcile", async () => {
|
|
10563
10884
|
const savedLimit = process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10564
|
-
const savedInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10565
10885
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = "2";
|
|
10566
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10567
10886
|
const journal = captureWatchLoopJournal();
|
|
10568
10887
|
try {
|
|
10569
10888
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -10595,15 +10914,9 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10595
10914
|
delete process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10596
10915
|
else
|
|
10597
10916
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = savedLimit;
|
|
10598
|
-
if (savedInterval === undefined)
|
|
10599
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10600
|
-
else
|
|
10601
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = savedInterval;
|
|
10602
10917
|
}
|
|
10603
10918
|
});
|
|
10604
|
-
it("logs scheduled-interval as the sole full-reconcile trigger on the default
|
|
10605
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10606
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10919
|
+
it("logs scheduled-interval as the sole full-reconcile trigger on the six-hour default", async () => {
|
|
10607
10920
|
const capture = captureFullReconcileDecisions();
|
|
10608
10921
|
try {
|
|
10609
10922
|
const clockState = { now: 0 };
|
|
@@ -10629,11 +10942,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10629
10942
|
sleep.tick();
|
|
10630
10943
|
await flushLoopMicrotasks();
|
|
10631
10944
|
}
|
|
10632
|
-
// A dozen ticks inside the
|
|
10945
|
+
// A dozen ticks inside the interval: never a scheduled full pass.
|
|
10633
10946
|
expect(runPass).not.toHaveBeenCalled();
|
|
10634
10947
|
capture.clear();
|
|
10635
10948
|
runPass.mockClear();
|
|
10636
|
-
clockState.now =
|
|
10949
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
10637
10950
|
sleep.tick();
|
|
10638
10951
|
await flushLoopMicrotasks();
|
|
10639
10952
|
expectFullReconcileDecision(capture.records[0], "full", [
|
|
@@ -10645,10 +10958,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10645
10958
|
}
|
|
10646
10959
|
finally {
|
|
10647
10960
|
capture.restore();
|
|
10648
|
-
if (saved === undefined)
|
|
10649
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10650
|
-
else
|
|
10651
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10652
10961
|
}
|
|
10653
10962
|
});
|
|
10654
10963
|
it("logs scoped passes with their batch path count and preserves the scoped route", async () => {
|
|
@@ -10669,8 +10978,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10669
10978
|
capture.restore();
|
|
10670
10979
|
});
|
|
10671
10980
|
it("reports interval diagnostics while the configured interval has not elapsed", async () => {
|
|
10672
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10673
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
|
|
10674
10981
|
const capture = captureFullReconcileDecisions();
|
|
10675
10982
|
try {
|
|
10676
10983
|
const clockState = { now: 0 };
|
|
@@ -10679,6 +10986,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10679
10986
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
10680
10987
|
const loop = runRunnerWithLoop(watchArgv, {
|
|
10681
10988
|
runPass,
|
|
10989
|
+
fullPassIntervalSource: { read: () => 1_000 },
|
|
10682
10990
|
monotonicNow: () => clockState.now,
|
|
10683
10991
|
createWatcher: () => makeBatchWatcherStub(),
|
|
10684
10992
|
sleep: sleep.sleep,
|
|
@@ -10714,10 +11022,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10714
11022
|
}
|
|
10715
11023
|
finally {
|
|
10716
11024
|
capture.restore();
|
|
10717
|
-
if (saved === undefined)
|
|
10718
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10719
|
-
else
|
|
10720
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10721
11025
|
}
|
|
10722
11026
|
});
|
|
10723
11027
|
it("startup runs one full reconcile, then watcher edits interrupt the pending wait", async () => {
|
|
@@ -11023,8 +11327,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11023
11327
|
await loop;
|
|
11024
11328
|
});
|
|
11025
11329
|
it("drops an unroutable batch and leaves the next unexpired poll idle", async () => {
|
|
11026
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11027
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
|
|
11028
11330
|
const capture = captureFullReconcileDecisions();
|
|
11029
11331
|
try {
|
|
11030
11332
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -11050,10 +11352,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11050
11352
|
}
|
|
11051
11353
|
finally {
|
|
11052
11354
|
capture.restore();
|
|
11053
|
-
if (saved === undefined)
|
|
11054
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11055
|
-
else
|
|
11056
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11057
11355
|
}
|
|
11058
11356
|
});
|
|
11059
11357
|
it("personal scoped ticks preserve personal mode for the pull leg", async () => {
|
|
@@ -11212,7 +11510,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11212
11510
|
}
|
|
11213
11511
|
expect(runPass).not.toHaveBeenCalled();
|
|
11214
11512
|
runPass.mockClear();
|
|
11215
|
-
clockState.now =
|
|
11513
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
11216
11514
|
sleep.tick();
|
|
11217
11515
|
await flushLoopMicrotasks();
|
|
11218
11516
|
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
@@ -11220,134 +11518,105 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11220
11518
|
sleep.tick();
|
|
11221
11519
|
await loop;
|
|
11222
11520
|
});
|
|
11223
|
-
it("schedules the full reconcile by elapsed time, not tick count,
|
|
11521
|
+
it("schedules the full reconcile by elapsed time, not tick count, at the flag fallback", async () => {
|
|
11224
11522
|
// Pre-fix the unset default was every tenth tick. Ticks fire on every
|
|
11225
11523
|
// watcher batch, so an active vault ran a ~10-minute full pass every
|
|
11226
11524
|
// 4-8 minutes — nearly continuous, and the runner's dominant CPU cost
|
|
11227
11525
|
// once per-pass hashing was fixed.
|
|
11228
|
-
const
|
|
11229
|
-
|
|
11230
|
-
|
|
11231
|
-
|
|
11232
|
-
|
|
11233
|
-
|
|
11234
|
-
|
|
11235
|
-
|
|
11236
|
-
|
|
11237
|
-
|
|
11238
|
-
|
|
11239
|
-
|
|
11240
|
-
|
|
11241
|
-
|
|
11242
|
-
|
|
11243
|
-
|
|
11244
|
-
|
|
11245
|
-
|
|
11246
|
-
|
|
11247
|
-
|
|
11248
|
-
|
|
11249
|
-
|
|
11250
|
-
|
|
11251
|
-
// two full passes here.
|
|
11252
|
-
for (let tick = 0; tick < 20; tick++) {
|
|
11253
|
-
clockState.now += 177_000;
|
|
11254
|
-
sleep.tick();
|
|
11255
|
-
await flushLoopMicrotasks();
|
|
11256
|
-
}
|
|
11257
|
-
expect(runPass).not.toHaveBeenCalled();
|
|
11258
|
-
runPass.mockClear();
|
|
11259
|
-
clockState.now = DEFAULT_FULL_RECONCILE_MS;
|
|
11526
|
+
const clockState = { now: 0 };
|
|
11527
|
+
const watcher = makeBatchWatcherStub();
|
|
11528
|
+
const sleep = makeSteppableSleep();
|
|
11529
|
+
let triggerShutdown = () => { };
|
|
11530
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11531
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11532
|
+
runPass,
|
|
11533
|
+
clock: new FakeClock(),
|
|
11534
|
+
monotonicNow: () => clockState.now,
|
|
11535
|
+
createWatcher: () => watcher,
|
|
11536
|
+
sleep: sleep.sleep,
|
|
11537
|
+
onShutdownSignal: (handler) => {
|
|
11538
|
+
triggerShutdown = handler;
|
|
11539
|
+
return () => { };
|
|
11540
|
+
},
|
|
11541
|
+
});
|
|
11542
|
+
await flushLoopMicrotasks();
|
|
11543
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11544
|
+
runPass.mockClear();
|
|
11545
|
+
// Twenty ticks in fifty-nine minutes: the old cadence would have run
|
|
11546
|
+
// two full passes here.
|
|
11547
|
+
for (let tick = 0; tick < 20; tick++) {
|
|
11548
|
+
clockState.now += 177_000;
|
|
11260
11549
|
sleep.tick();
|
|
11261
11550
|
await flushLoopMicrotasks();
|
|
11262
|
-
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11263
|
-
triggerShutdown();
|
|
11264
|
-
await loop;
|
|
11265
|
-
}
|
|
11266
|
-
finally {
|
|
11267
|
-
if (saved === undefined)
|
|
11268
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11269
|
-
else
|
|
11270
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11271
11551
|
}
|
|
11552
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
11553
|
+
runPass.mockClear();
|
|
11554
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
11555
|
+
sleep.tick();
|
|
11556
|
+
await flushLoopMicrotasks();
|
|
11557
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11558
|
+
triggerShutdown();
|
|
11559
|
+
await loop;
|
|
11272
11560
|
});
|
|
11273
|
-
it("uses the
|
|
11274
|
-
const
|
|
11275
|
-
|
|
11276
|
-
|
|
11277
|
-
|
|
11278
|
-
|
|
11279
|
-
|
|
11280
|
-
|
|
11281
|
-
|
|
11282
|
-
|
|
11283
|
-
|
|
11284
|
-
|
|
11285
|
-
|
|
11286
|
-
|
|
11287
|
-
|
|
11288
|
-
|
|
11289
|
-
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
|
|
11300
|
-
|
|
11301
|
-
|
|
11302
|
-
|
|
11303
|
-
|
|
11304
|
-
|
|
11305
|
-
|
|
11306
|
-
triggerShutdown();
|
|
11307
|
-
await loop;
|
|
11308
|
-
}
|
|
11309
|
-
finally {
|
|
11310
|
-
if (saved === undefined)
|
|
11311
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11312
|
-
else
|
|
11313
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11314
|
-
}
|
|
11561
|
+
it("uses the hq-flag interval source instead of the tick count", async () => {
|
|
11562
|
+
const clockState = { now: 0 };
|
|
11563
|
+
const watcher = makeBatchWatcherStub();
|
|
11564
|
+
const sleep = makeSteppableSleep();
|
|
11565
|
+
let triggerShutdown = () => { };
|
|
11566
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11567
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11568
|
+
runPass,
|
|
11569
|
+
fullPassIntervalSource: { read: () => 1_000 },
|
|
11570
|
+
clock: new FakeClock(),
|
|
11571
|
+
monotonicNow: () => clockState.now,
|
|
11572
|
+
createWatcher: () => watcher,
|
|
11573
|
+
sleep: sleep.sleep,
|
|
11574
|
+
onShutdownSignal: (handler) => {
|
|
11575
|
+
triggerShutdown = handler;
|
|
11576
|
+
return () => { };
|
|
11577
|
+
},
|
|
11578
|
+
});
|
|
11579
|
+
await flushLoopMicrotasks();
|
|
11580
|
+
// Startup always forces the first full reconcile, and starts the interval.
|
|
11581
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11582
|
+
runPass.mockClear();
|
|
11583
|
+
clockState.now = 999;
|
|
11584
|
+
sleep.tick();
|
|
11585
|
+
await flushLoopMicrotasks();
|
|
11586
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
11587
|
+
runPass.mockClear();
|
|
11588
|
+
clockState.now = 1_000;
|
|
11589
|
+
sleep.tick();
|
|
11590
|
+
await flushLoopMicrotasks();
|
|
11591
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11592
|
+
triggerShutdown();
|
|
11593
|
+
await loop;
|
|
11315
11594
|
});
|
|
11316
11595
|
it("does not let a full-reconcile interval suppress event-push activation", async () => {
|
|
11317
|
-
const
|
|
11318
|
-
|
|
11319
|
-
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11326
|
-
|
|
11327
|
-
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11339
|
-
|
|
11340
|
-
|
|
11341
|
-
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11342
|
-
triggerShutdown();
|
|
11343
|
-
await loop;
|
|
11344
|
-
}
|
|
11345
|
-
finally {
|
|
11346
|
-
if (saved === undefined)
|
|
11347
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11348
|
-
else
|
|
11349
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11350
|
-
}
|
|
11596
|
+
const watcher = makeBatchWatcherStub();
|
|
11597
|
+
const sleep = makeSteppableSleep();
|
|
11598
|
+
let triggerShutdown = () => { };
|
|
11599
|
+
// A numeric partial result does not activate event-push surfaces, so the
|
|
11600
|
+
// next tick must force another full pass even though three hours have not elapsed.
|
|
11601
|
+
const runPass = vi.fn().mockResolvedValue(PARTIAL_SYNC_EXIT);
|
|
11602
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11603
|
+
runPass,
|
|
11604
|
+
clock: new FakeClock(),
|
|
11605
|
+
monotonicNow: () => 0,
|
|
11606
|
+
createWatcher: () => watcher,
|
|
11607
|
+
sleep: sleep.sleep,
|
|
11608
|
+
onShutdownSignal: (handler) => {
|
|
11609
|
+
triggerShutdown = handler;
|
|
11610
|
+
return () => { };
|
|
11611
|
+
},
|
|
11612
|
+
});
|
|
11613
|
+
await flushLoopMicrotasks();
|
|
11614
|
+
runPass.mockClear();
|
|
11615
|
+
sleep.tick();
|
|
11616
|
+
await flushLoopMicrotasks();
|
|
11617
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11618
|
+
triggerShutdown();
|
|
11619
|
+
await loop;
|
|
11351
11620
|
});
|
|
11352
11621
|
it("overflow with known dropped-route hints fans out per touched route instead of a full reconcile", async () => {
|
|
11353
11622
|
const capture = captureFullReconcileDecisions();
|
|
@@ -11482,50 +11751,40 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11482
11751
|
await loop;
|
|
11483
11752
|
});
|
|
11484
11753
|
it("a batch above the shipped 10,000-path default stays on chunked targeted passes before the interval", async () => {
|
|
11485
|
-
const
|
|
11486
|
-
|
|
11487
|
-
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11491
|
-
|
|
11492
|
-
|
|
11493
|
-
|
|
11494
|
-
|
|
11495
|
-
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
|
|
11499
|
-
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
|
-
|
|
11505
|
-
|
|
11506
|
-
|
|
11507
|
-
|
|
11508
|
-
|
|
11509
|
-
|
|
11510
|
-
|
|
11511
|
-
|
|
11512
|
-
|
|
11513
|
-
|
|
11514
|
-
|
|
11515
|
-
|
|
11516
|
-
|
|
11517
|
-
|
|
11518
|
-
|
|
11519
|
-
expect(pushScopeCounts.reduce((sum, count) => sum + count, 0)).toBe(10_001);
|
|
11520
|
-
triggerShutdown();
|
|
11521
|
-
await loop;
|
|
11522
|
-
}
|
|
11523
|
-
finally {
|
|
11524
|
-
if (saved === undefined)
|
|
11525
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11526
|
-
else
|
|
11527
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11528
|
-
}
|
|
11754
|
+
const watcher = makeBatchWatcherStub();
|
|
11755
|
+
const sleep = makeSteppableSleep();
|
|
11756
|
+
let triggerShutdown = () => { };
|
|
11757
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11758
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11759
|
+
runPass,
|
|
11760
|
+
clock: new FakeClock(),
|
|
11761
|
+
monotonicNow: () => 0,
|
|
11762
|
+
createWatcher: () => watcher,
|
|
11763
|
+
sleep: sleep.sleep,
|
|
11764
|
+
onShutdownSignal: (handler) => {
|
|
11765
|
+
triggerShutdown = handler;
|
|
11766
|
+
return () => { };
|
|
11767
|
+
},
|
|
11768
|
+
});
|
|
11769
|
+
await flushLoopMicrotasks();
|
|
11770
|
+
runPass.mockClear();
|
|
11771
|
+
watcher.emit("companies/indigo/a.md", {
|
|
11772
|
+
paths: new Map(Array.from({ length: 10_001 }, (_, index) => [
|
|
11773
|
+
`/tmp/hq/companies/indigo/${index}.md`,
|
|
11774
|
+
`companies/indigo/${index}.md`,
|
|
11775
|
+
])),
|
|
11776
|
+
});
|
|
11777
|
+
sleep.tick();
|
|
11778
|
+
await flushLoopMicrotasks(40);
|
|
11779
|
+
const calls = runPass.mock.calls.map((call) => call[0]);
|
|
11780
|
+
expect(calls).not.toContainEqual(fullArgv);
|
|
11781
|
+
const pushScopeCounts = calls
|
|
11782
|
+
.filter((argv) => argv.includes("push"))
|
|
11783
|
+
.map((argv) => scopePathsFromArgv(argv).length);
|
|
11784
|
+
expect(Math.max(0, ...pushScopeCounts)).toBeLessThanOrEqual(10_000);
|
|
11785
|
+
expect(pushScopeCounts.reduce((sum, count) => sum + count, 0)).toBe(10_001);
|
|
11786
|
+
triggerShutdown();
|
|
11787
|
+
await loop;
|
|
11529
11788
|
});
|
|
11530
11789
|
it("a throwing targeted pass falls back to one full reconcile instead of dropping the change", async () => {
|
|
11531
11790
|
const capture = captureFullReconcileDecisions();
|