@indigoai-us/hq-cloud 6.16.83-alpha.0 → 6.16.84-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +22 -19
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +118 -71
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +510 -253
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +50 -9
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +224 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +78 -27
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +107 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +15 -7
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +148 -15
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +321 -2
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/full-pass-interval.d.ts +25 -0
- package/dist/sync/full-pass-interval.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.js +53 -0
- package/dist/sync/full-pass-interval.js.map +1 -0
- package/dist/sync/full-pass-interval.test.d.ts +2 -0
- package/dist/sync/full-pass-interval.test.d.ts.map +1 -0
- package/dist/sync/full-pass-interval.test.js +21 -0
- package/dist/sync/full-pass-interval.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +0 -14
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +12 -49
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +7 -171
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -16,11 +16,12 @@ import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
|
16
16
|
import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
|
|
17
17
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
18
18
|
import { AUTH_WAIT_INITIAL_RECHECK_MS, AUTH_WAIT_RESUME_MESSAGE, AUTH_WAIT_START_MESSAGE, AUTH_WAIT_STEADY_RECHECK_MS, } from "./sync-runner-auth-wait.js";
|
|
19
|
-
import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
|
|
19
|
+
import { ObjectLockChecksumRequiredError, PresignObjectIO, S3SdkObjectIO, } from "../object-io.js";
|
|
20
20
|
import { setBandwidthGovernorForTesting } from "../bandwidth.js";
|
|
21
21
|
import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
22
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
23
|
-
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES,
|
|
23
|
+
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
24
|
+
import { DEFAULT_FULL_PASS_INTERVAL_MS } from "../sync/full-pass-interval.js";
|
|
24
25
|
import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
|
|
25
26
|
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
26
27
|
import { AreaLedger } from "../sync/area-ledger.js";
|
|
@@ -565,6 +566,79 @@ describe("mid-pass identity refusal", () => {
|
|
|
565
566
|
expect(deps.stderr.events().some((e) => e.type === "auth-error")).toBe(false);
|
|
566
567
|
});
|
|
567
568
|
});
|
|
569
|
+
describe("remote failure contract", () => {
|
|
570
|
+
const person = {
|
|
571
|
+
uid: "ent_person_remote_failure_contract",
|
|
572
|
+
type: "person",
|
|
573
|
+
bucketName: "hq-vault-personal-contract",
|
|
574
|
+
status: "active",
|
|
575
|
+
};
|
|
576
|
+
it.each([
|
|
577
|
+
[
|
|
578
|
+
"terminal 403 refusal",
|
|
579
|
+
() => new CognitoRefreshError("session reauth required (403)", true, 403),
|
|
580
|
+
0,
|
|
581
|
+
"auth-error",
|
|
582
|
+
"quick refresh",
|
|
583
|
+
],
|
|
584
|
+
[
|
|
585
|
+
"404 route",
|
|
586
|
+
() => new VaultNotFoundError("remote route no longer exists (404)"),
|
|
587
|
+
PARTIAL_SYNC_EXIT,
|
|
588
|
+
"error",
|
|
589
|
+
"404",
|
|
590
|
+
],
|
|
591
|
+
[
|
|
592
|
+
"429 throttle",
|
|
593
|
+
() => Object.assign(new Error("remote throttled (429)"), { statusCode: 429 }),
|
|
594
|
+
TRANSIENT_NETWORK_EXIT,
|
|
595
|
+
"transient-network",
|
|
596
|
+
"429",
|
|
597
|
+
],
|
|
598
|
+
[
|
|
599
|
+
"503 throttle",
|
|
600
|
+
() => Object.assign(new Error("remote unavailable (503)"), { statusCode: 503 }),
|
|
601
|
+
TRANSIENT_NETWORK_EXIT,
|
|
602
|
+
"transient-network",
|
|
603
|
+
"503",
|
|
604
|
+
],
|
|
605
|
+
[
|
|
606
|
+
"Object Lock checksum refusal",
|
|
607
|
+
() => new ObjectLockChecksumRequiredError("remote.md", "checksum required"),
|
|
608
|
+
PARTIAL_SYNC_EXIT,
|
|
609
|
+
"error",
|
|
610
|
+
"ObjectLockChecksumRequired",
|
|
611
|
+
],
|
|
612
|
+
[
|
|
613
|
+
"timeout",
|
|
614
|
+
() => Object.assign(new Error("remote timed out"), { name: "TimeoutError" }),
|
|
615
|
+
TRANSIENT_NETWORK_EXIT,
|
|
616
|
+
"transient-network",
|
|
617
|
+
"timed out",
|
|
618
|
+
],
|
|
619
|
+
[
|
|
620
|
+
"abort",
|
|
621
|
+
() => Object.assign(new Error("remote aborted"), { name: "AbortError" }),
|
|
622
|
+
TRANSIENT_NETWORK_EXIT,
|
|
623
|
+
"transient-network",
|
|
624
|
+
"aborted",
|
|
625
|
+
],
|
|
626
|
+
])("keeps the journal stable and records the runner outcome for %s", async (_description, makeFailure, expectedExit, telemetryType, telemetryReason) => {
|
|
627
|
+
const before = readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
628
|
+
const deps = makeDeps({
|
|
629
|
+
sync: vi.fn().mockRejectedValue(makeFailure()),
|
|
630
|
+
createVaultClient: () => makeVaultStub({ listPersons: () => Promise.resolve([person]) }),
|
|
631
|
+
});
|
|
632
|
+
expect(await runRunner(["--personal"], deps)).toBe(expectedExit);
|
|
633
|
+
expect(readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files).toEqual(before.files);
|
|
634
|
+
const events = [...deps.stdout.events(), ...deps.stderr.events()];
|
|
635
|
+
expect(events.some((event) => event.type === telemetryType)).toBe(true);
|
|
636
|
+
expect(`${deps.stdout.raw()}${deps.stderr.raw()}`).toContain(telemetryReason);
|
|
637
|
+
if (telemetryType === "transient-network") {
|
|
638
|
+
expect(events.some((event) => event.type === "error")).toBe(false);
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
});
|
|
568
642
|
// ---------------------------------------------------------------------------
|
|
569
643
|
// memberships retry (3x with backoff, then abort)
|
|
570
644
|
// ---------------------------------------------------------------------------
|
|
@@ -4636,28 +4710,13 @@ describe("advanceMonotonicDeadline", () => {
|
|
|
4636
4710
|
});
|
|
4637
4711
|
});
|
|
4638
4712
|
});
|
|
4639
|
-
describe("watch-loop reconcile
|
|
4713
|
+
describe("watch-loop reconcile configuration", () => {
|
|
4640
4714
|
it("uses the shipped 10,000 event-batch default while allowing smaller and larger overrides", () => {
|
|
4641
4715
|
expect(DEFAULT_EVENT_BATCH_LIMIT).toBe(10_000);
|
|
4642
4716
|
expect(resolveEventBatchLimit({})).toBe(10_000);
|
|
4643
4717
|
expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "5" })).toBe(5);
|
|
4644
4718
|
expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "50000" })).toBe(50_000);
|
|
4645
4719
|
});
|
|
4646
|
-
it("defaults the full-reconcile cadence to one elapsed hour for an unset or invalid interval", () => {
|
|
4647
|
-
// The legacy default was "every tenth tick" with no elapsed-time floor.
|
|
4648
|
-
// Ticks fire on every watcher batch, not just polls, so on an active
|
|
4649
|
-
// vault that meant a full pass every 4-8 minutes — and each pass on a
|
|
4650
|
-
// real install lasts ~10 minutes. Full reconcile is the correctness
|
|
4651
|
-
// backstop behind event-push, so elapsed time is the right clock.
|
|
4652
|
-
expect(DEFAULT_FULL_RECONCILE_MS).toBe(3_600_000);
|
|
4653
|
-
expect(resolveFullReconcileMs({})).toBe(DEFAULT_FULL_RECONCILE_MS);
|
|
4654
|
-
for (const raw of ["", "0", "0.5", "-1", "NaN", "not-a-number", "Infinity"]) {
|
|
4655
|
-
expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: raw })).toBe(DEFAULT_FULL_RECONCILE_MS);
|
|
4656
|
-
}
|
|
4657
|
-
});
|
|
4658
|
-
it("accepts a positive millisecond full-reconcile interval", () => {
|
|
4659
|
-
expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: "10800000" })).toBe(10_800_000);
|
|
4660
|
-
});
|
|
4661
4720
|
});
|
|
4662
4721
|
describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", () => {
|
|
4663
4722
|
// Capture the ms handed to sleep on the FIRST poll, then hang so the loop
|
|
@@ -4873,10 +4932,8 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
4873
4932
|
it("parallel coordinator activates every event surface after a partial completed full reconcile", async () => {
|
|
4874
4933
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
4875
4934
|
const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
|
|
4876
|
-
const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4877
4935
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
4878
4936
|
process.env.HQ_SYNC_EVENT_SYNC = "1";
|
|
4879
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "60000";
|
|
4880
4937
|
const watcher = makeWatcherStub();
|
|
4881
4938
|
const receiver = {
|
|
4882
4939
|
connected: false,
|
|
@@ -4936,19 +4993,13 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
4936
4993
|
delete process.env.HQ_SYNC_EVENT_SYNC;
|
|
4937
4994
|
else
|
|
4938
4995
|
process.env.HQ_SYNC_EVENT_SYNC = previousEventSync;
|
|
4939
|
-
if (previousFullReconcileMs === undefined)
|
|
4940
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4941
|
-
else
|
|
4942
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
|
|
4943
4996
|
}
|
|
4944
4997
|
});
|
|
4945
4998
|
it("parallel coordinator activates event surfaces and advances cadence after a numeric-zero full reconcile", async () => {
|
|
4946
4999
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
4947
5000
|
const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
|
|
4948
|
-
const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
4949
5001
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
4950
5002
|
process.env.HQ_SYNC_EVENT_SYNC = "1";
|
|
4951
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "60000";
|
|
4952
5003
|
const watcher = makeWatcherStub();
|
|
4953
5004
|
const receiver = {
|
|
4954
5005
|
connected: false,
|
|
@@ -5008,10 +5059,55 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5008
5059
|
delete process.env.HQ_SYNC_EVENT_SYNC;
|
|
5009
5060
|
else
|
|
5010
5061
|
process.env.HQ_SYNC_EVENT_SYNC = previousEventSync;
|
|
5011
|
-
|
|
5012
|
-
|
|
5062
|
+
}
|
|
5063
|
+
});
|
|
5064
|
+
it("backs off failed parallel full-reconcile completions before another chore is admitted", async () => {
|
|
5065
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5066
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
5067
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
5068
|
+
const delays = [];
|
|
5069
|
+
const waiters = [];
|
|
5070
|
+
let now = 0;
|
|
5071
|
+
let shutdown = () => { };
|
|
5072
|
+
const sleep = (ms) => {
|
|
5073
|
+
delays.push(ms);
|
|
5074
|
+
return new Promise((resolve) => {
|
|
5075
|
+
waiters.push(resolve);
|
|
5076
|
+
});
|
|
5077
|
+
};
|
|
5078
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
5079
|
+
runPass,
|
|
5080
|
+
clock: new FakeClock(),
|
|
5081
|
+
monotonicNow: () => now,
|
|
5082
|
+
sleep,
|
|
5083
|
+
fullPassIntervalSource: { read: () => 1 },
|
|
5084
|
+
onShutdownSignal: (handler) => {
|
|
5085
|
+
shutdown = handler;
|
|
5086
|
+
return () => { };
|
|
5087
|
+
},
|
|
5088
|
+
});
|
|
5089
|
+
try {
|
|
5090
|
+
for (let attempt = 0; attempt < 50 && runPass.mock.calls.length < 3; attempt += 1) {
|
|
5091
|
+
const resolve = waiters.shift();
|
|
5092
|
+
if (resolve) {
|
|
5093
|
+
now += 1;
|
|
5094
|
+
resolve();
|
|
5095
|
+
await flushLoopMicrotasks(30);
|
|
5096
|
+
}
|
|
5097
|
+
else {
|
|
5098
|
+
await new Promise((resolveNext) => setImmediate(resolveNext));
|
|
5099
|
+
}
|
|
5100
|
+
}
|
|
5101
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
5102
|
+
expect(delays).toContain(60_000);
|
|
5103
|
+
}
|
|
5104
|
+
finally {
|
|
5105
|
+
shutdown();
|
|
5106
|
+
await expect(loop).resolves.toBe(0);
|
|
5107
|
+
if (previousCoordinator === undefined)
|
|
5108
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5013
5109
|
else
|
|
5014
|
-
process.env.
|
|
5110
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
5015
5111
|
}
|
|
5016
5112
|
});
|
|
5017
5113
|
it("parallel coordinator keeps event surfaces off for an auth-required full reconcile", async () => {
|
|
@@ -5144,7 +5240,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5144
5240
|
sleep.tick();
|
|
5145
5241
|
await flushLoopMicrotasks();
|
|
5146
5242
|
}
|
|
5147
|
-
clockState.now =
|
|
5243
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
5148
5244
|
sleep.tick();
|
|
5149
5245
|
await flushLoopMicrotasks();
|
|
5150
5246
|
// Tick 1 and the first tick after an hour has elapsed are full
|
|
@@ -5212,7 +5308,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5212
5308
|
}
|
|
5213
5309
|
expect(runPass).toHaveBeenCalledTimes(1);
|
|
5214
5310
|
// The hourly cadence, not a tick count, schedules the next full pass.
|
|
5215
|
-
clockState.now =
|
|
5311
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
5216
5312
|
sleep.tick();
|
|
5217
5313
|
await flushLoopMicrotasks(30);
|
|
5218
5314
|
expect(runPass).toHaveBeenCalledTimes(2);
|
|
@@ -6098,7 +6194,6 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6098
6194
|
}],
|
|
6099
6195
|
signal: new AbortController().signal,
|
|
6100
6196
|
});
|
|
6101
|
-
const firstReplacementFailure = expect(firstReplacement).rejects.toThrow("targeted pull failed");
|
|
6102
6197
|
await flushLoopMicrotasks(30); // queues the warm-up pull behind held work
|
|
6103
6198
|
const secondReplacement = receiverSync({
|
|
6104
6199
|
events: [{
|
|
@@ -6113,12 +6208,11 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6113
6208
|
}],
|
|
6114
6209
|
signal: new AbortController().signal,
|
|
6115
6210
|
});
|
|
6116
|
-
await
|
|
6211
|
+
await firstReplacement;
|
|
6117
6212
|
await flushLoopMicrotasks(30);
|
|
6118
6213
|
// Both warm-up passes were superseded while queued behind held realtime
|
|
6119
|
-
// work, so the coordinator never dispatched their executor. They
|
|
6120
|
-
//
|
|
6121
|
-
// their receipts before a targeted pull becomes durable.
|
|
6214
|
+
// work, so the coordinator never dispatched their executor. They remain
|
|
6215
|
+
// terminal no-ops rather than fabricated exit-1 runner failures.
|
|
6122
6216
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("warmup-superseded.md"))).toHaveLength(0);
|
|
6123
6217
|
expect(settled).toBeUndefined();
|
|
6124
6218
|
await Promise.resolve();
|
|
@@ -6137,7 +6231,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6137
6231
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6138
6232
|
}
|
|
6139
6233
|
});
|
|
6140
|
-
it("
|
|
6234
|
+
it("acknowledges superseded receiver work and records refused telemetry", async () => {
|
|
6141
6235
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6142
6236
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6143
6237
|
const watcher = makeBatchWatcherStub();
|
|
@@ -6191,7 +6285,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6191
6285
|
events: [event("companies/indigo/superseded.md")],
|
|
6192
6286
|
signal: new AbortController().signal,
|
|
6193
6287
|
});
|
|
6194
|
-
await expect(superseded).
|
|
6288
|
+
await expect(superseded).resolves.toBeUndefined();
|
|
6195
6289
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
|
|
6196
6290
|
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":3,"source":"receiver","direction":"pull","outcome":"refused"');
|
|
6197
6291
|
releaseHeld();
|
|
@@ -6209,6 +6303,124 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6209
6303
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6210
6304
|
}
|
|
6211
6305
|
});
|
|
6306
|
+
it("measures a queued realtime pass at the operation gate and marks append lock only after admission", async () => {
|
|
6307
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6308
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6309
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-telemetry-operation-gate-"));
|
|
6310
|
+
const watcher = makeBatchWatcherStub();
|
|
6311
|
+
let receiverSync;
|
|
6312
|
+
let shutdown = () => { };
|
|
6313
|
+
let releaseGuard = () => { };
|
|
6314
|
+
const guard = new Promise((resolve) => { releaseGuard = resolve; });
|
|
6315
|
+
let markGuardStarted = () => { };
|
|
6316
|
+
const guardStarted = new Promise((resolve) => { markGuardStarted = resolve; });
|
|
6317
|
+
let markTargetStarted = () => { };
|
|
6318
|
+
const targetStarted = new Promise((resolve) => { markTargetStarted = resolve; });
|
|
6319
|
+
let monotonicMs = 0;
|
|
6320
|
+
let guardReleased = false;
|
|
6321
|
+
let targetStartedAt;
|
|
6322
|
+
const timings = [];
|
|
6323
|
+
const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
6324
|
+
if (typeof chunk === "string" && chunk.startsWith("{")) {
|
|
6325
|
+
try {
|
|
6326
|
+
const record = JSON.parse(chunk);
|
|
6327
|
+
if (typeof record === "object" &&
|
|
6328
|
+
record !== null &&
|
|
6329
|
+
record.event === "sync.realtime.timing")
|
|
6330
|
+
timings.push(record);
|
|
6331
|
+
}
|
|
6332
|
+
catch {
|
|
6333
|
+
// Non-telemetry diagnostics are outside this assertion.
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
return true;
|
|
6337
|
+
});
|
|
6338
|
+
const includesScope = (argv, basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
|
|
6339
|
+
const runPass = vi.fn(async (argv) => {
|
|
6340
|
+
if (includesScope(argv, "guard.md")) {
|
|
6341
|
+
markGuardStarted();
|
|
6342
|
+
await guard;
|
|
6343
|
+
return completedPassOutcome();
|
|
6344
|
+
}
|
|
6345
|
+
if (includesScope(argv, "target.md")) {
|
|
6346
|
+
expect(guardReleased).toBe(true);
|
|
6347
|
+
targetStartedAt = monotonicMs;
|
|
6348
|
+
markTargetStarted();
|
|
6349
|
+
monotonicMs += 1;
|
|
6350
|
+
}
|
|
6351
|
+
return completedPassOutcome();
|
|
6352
|
+
});
|
|
6353
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6354
|
+
runPass,
|
|
6355
|
+
createWatcher: () => watcher,
|
|
6356
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
6357
|
+
receiverSync = syncBatchFn;
|
|
6358
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6359
|
+
},
|
|
6360
|
+
monotonicNow: () => monotonicMs,
|
|
6361
|
+
sleep: () => new Promise(() => { }),
|
|
6362
|
+
onShutdownSignal: (handler) => {
|
|
6363
|
+
shutdown = handler;
|
|
6364
|
+
return () => { };
|
|
6365
|
+
},
|
|
6366
|
+
});
|
|
6367
|
+
const event = (relativePath) => ({
|
|
6368
|
+
kind: "upsert",
|
|
6369
|
+
relativePath,
|
|
6370
|
+
contentHash: "sha256:remote",
|
|
6371
|
+
mtime: "2026-09-23T00:00:00.000Z",
|
|
6372
|
+
originDeviceId: "peer-device",
|
|
6373
|
+
originTenantId: "tenant-indigo",
|
|
6374
|
+
sequenceNumber: 1,
|
|
6375
|
+
eventTimestamp: "2026-09-23T00:00:00.000Z",
|
|
6376
|
+
});
|
|
6377
|
+
try {
|
|
6378
|
+
await flushLoopMicrotasks(30);
|
|
6379
|
+
runPass.mockClear();
|
|
6380
|
+
const guardedPass = receiverSync({
|
|
6381
|
+
events: [event("companies/alpha/guard.md")],
|
|
6382
|
+
signal: new AbortController().signal,
|
|
6383
|
+
});
|
|
6384
|
+
await guardStarted;
|
|
6385
|
+
const targetAbsolutePath = path.join(hqRoot, "companies", "beta", "target.md");
|
|
6386
|
+
fs.mkdirSync(path.dirname(targetAbsolutePath), { recursive: true });
|
|
6387
|
+
fs.writeFileSync(targetAbsolutePath, "x");
|
|
6388
|
+
watcher.emit("companies/beta/target.md", {
|
|
6389
|
+
paths: new Map([
|
|
6390
|
+
[targetAbsolutePath, "companies/beta/target.md"],
|
|
6391
|
+
]),
|
|
6392
|
+
changes: new Map(),
|
|
6393
|
+
});
|
|
6394
|
+
await flushLoopMicrotasks(30);
|
|
6395
|
+
expect(targetStartedAt).toBeUndefined();
|
|
6396
|
+
monotonicMs += 25;
|
|
6397
|
+
guardReleased = true;
|
|
6398
|
+
releaseGuard();
|
|
6399
|
+
await guardedPass;
|
|
6400
|
+
await targetStarted;
|
|
6401
|
+
await flushLoopMicrotasks(30);
|
|
6402
|
+
expect(targetStartedAt).toBe(25);
|
|
6403
|
+
expect(timings).toHaveLength(2);
|
|
6404
|
+
const targetTiming = timings.find((record) => record.source === "watcher" &&
|
|
6405
|
+
record.direction === "push" &&
|
|
6406
|
+
record.outcome === "durable");
|
|
6407
|
+
expect(targetTiming).toBeDefined();
|
|
6408
|
+
expect(targetTiming?.timingMs?.operationGate).toBe(25);
|
|
6409
|
+
expect(targetTiming?.timingMs?.appendLock).toBe(0);
|
|
6410
|
+
}
|
|
6411
|
+
finally {
|
|
6412
|
+
guardReleased = true;
|
|
6413
|
+
releaseGuard();
|
|
6414
|
+
shutdown();
|
|
6415
|
+
await loop;
|
|
6416
|
+
stderr.mockRestore();
|
|
6417
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6418
|
+
if (previousCoordinator === undefined)
|
|
6419
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6420
|
+
else
|
|
6421
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6422
|
+
}
|
|
6423
|
+
});
|
|
6212
6424
|
it("REGRESSION: a fast receiver pull overtakes a slow watcher push and records admitted occupancy", async () => {
|
|
6213
6425
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6214
6426
|
const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
|
|
@@ -8900,6 +9112,137 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
8900
9112
|
await expect(loop).resolves.toBe(0);
|
|
8901
9113
|
expect(watcher.disposed).toBe(false);
|
|
8902
9114
|
});
|
|
9115
|
+
it("bounds repeated watch-pass exits with a fake-clock retry budget while keeping the runner alive", async () => {
|
|
9116
|
+
const watcher = makeWatcherStub();
|
|
9117
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
9118
|
+
const delays = [];
|
|
9119
|
+
const waiters = [];
|
|
9120
|
+
let now = 0;
|
|
9121
|
+
let shutdown = () => { };
|
|
9122
|
+
let settled;
|
|
9123
|
+
const sleep = (ms) => {
|
|
9124
|
+
delays.push(ms);
|
|
9125
|
+
return new Promise((resolve) => {
|
|
9126
|
+
waiters.push(resolve);
|
|
9127
|
+
});
|
|
9128
|
+
};
|
|
9129
|
+
const tick = async (advanceMs) => {
|
|
9130
|
+
now += advanceMs;
|
|
9131
|
+
const resolve = waiters.shift();
|
|
9132
|
+
if (!resolve)
|
|
9133
|
+
throw new Error("retry sleep was not pending");
|
|
9134
|
+
resolve();
|
|
9135
|
+
await flushLoopMicrotasks();
|
|
9136
|
+
};
|
|
9137
|
+
const waitForRetrySleep = async () => {
|
|
9138
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
9139
|
+
if (waiters.length > 0)
|
|
9140
|
+
return;
|
|
9141
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
9142
|
+
}
|
|
9143
|
+
throw new Error("timed out waiting for retry sleep");
|
|
9144
|
+
};
|
|
9145
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
9146
|
+
runPass,
|
|
9147
|
+
clock: new FakeClock(),
|
|
9148
|
+
createWatcher: () => watcher,
|
|
9149
|
+
monotonicNow: () => now,
|
|
9150
|
+
sleep,
|
|
9151
|
+
onShutdownSignal: (handler) => {
|
|
9152
|
+
shutdown = handler;
|
|
9153
|
+
return () => { };
|
|
9154
|
+
},
|
|
9155
|
+
}).then((code) => {
|
|
9156
|
+
settled = code;
|
|
9157
|
+
return code;
|
|
9158
|
+
});
|
|
9159
|
+
try {
|
|
9160
|
+
await waitForRetrySleep();
|
|
9161
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9162
|
+
expect(delays).toEqual([1]);
|
|
9163
|
+
await tick(1_000);
|
|
9164
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9165
|
+
expect(delays).toEqual([1, 1]);
|
|
9166
|
+
await tick(1_000);
|
|
9167
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9168
|
+
expect(delays).toEqual([1, 1, 60_000]);
|
|
9169
|
+
await tick(60_000);
|
|
9170
|
+
await tick(120_000);
|
|
9171
|
+
await tick(240_000);
|
|
9172
|
+
await tick(480_000);
|
|
9173
|
+
expect(runPass).toHaveBeenCalledTimes(7);
|
|
9174
|
+
expect(delays).toEqual([
|
|
9175
|
+
1,
|
|
9176
|
+
1,
|
|
9177
|
+
60_000,
|
|
9178
|
+
120_000,
|
|
9179
|
+
240_000,
|
|
9180
|
+
480_000,
|
|
9181
|
+
600_000,
|
|
9182
|
+
]);
|
|
9183
|
+
// The retry window is quiet-time based, not anchored to the first
|
|
9184
|
+
// failure. Continuing failures after thirty minutes must remain capped.
|
|
9185
|
+
await tick(600_000);
|
|
9186
|
+
await tick(600_000);
|
|
9187
|
+
expect(runPass).toHaveBeenCalledTimes(9);
|
|
9188
|
+
expect(delays.slice(-2)).toEqual([600_000, 600_000]);
|
|
9189
|
+
expect(settled).toBeUndefined();
|
|
9190
|
+
expect(watcher.disposed).toBe(false);
|
|
9191
|
+
}
|
|
9192
|
+
finally {
|
|
9193
|
+
shutdown();
|
|
9194
|
+
await expect(loop).resolves.toBe(0);
|
|
9195
|
+
}
|
|
9196
|
+
});
|
|
9197
|
+
it("does not let a watcher interrupt bypass a failed-pass retry deadline", async () => {
|
|
9198
|
+
const watcher = makeWatcherStub();
|
|
9199
|
+
let passes = 0;
|
|
9200
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
9201
|
+
passes += 1;
|
|
9202
|
+
return passes === 1 ? 0 : 1;
|
|
9203
|
+
});
|
|
9204
|
+
const waiters = [];
|
|
9205
|
+
const now = 0;
|
|
9206
|
+
let shutdown = () => { };
|
|
9207
|
+
const sleep = vi.fn(() => new Promise((resolve) => {
|
|
9208
|
+
waiters.push(resolve);
|
|
9209
|
+
}));
|
|
9210
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
9211
|
+
runPass,
|
|
9212
|
+
clock: new FakeClock(),
|
|
9213
|
+
createWatcher: () => watcher,
|
|
9214
|
+
monotonicNow: () => now,
|
|
9215
|
+
sleep,
|
|
9216
|
+
onShutdownSignal: (handler) => {
|
|
9217
|
+
shutdown = handler;
|
|
9218
|
+
return () => { };
|
|
9219
|
+
},
|
|
9220
|
+
});
|
|
9221
|
+
try {
|
|
9222
|
+
await waitForPollSleep({
|
|
9223
|
+
pendingCount: () => waiters.length,
|
|
9224
|
+
});
|
|
9225
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9226
|
+
watcher.emit();
|
|
9227
|
+
await flushLoopMicrotasks(30);
|
|
9228
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9229
|
+
watcher.emit();
|
|
9230
|
+
await flushLoopMicrotasks(30);
|
|
9231
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9232
|
+
watcher.emit();
|
|
9233
|
+
await flushLoopMicrotasks(30);
|
|
9234
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9235
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9236
|
+
watcher.emit();
|
|
9237
|
+
await flushLoopMicrotasks(30);
|
|
9238
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9239
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9240
|
+
}
|
|
9241
|
+
finally {
|
|
9242
|
+
shutdown();
|
|
9243
|
+
await expect(loop).resolves.toBe(0);
|
|
9244
|
+
}
|
|
9245
|
+
});
|
|
8903
9246
|
it("alerts when a partial pass failed before a company's download leg", async () => {
|
|
8904
9247
|
const watcher = makeWatcherStub();
|
|
8905
9248
|
const stderrWrite = vi
|
|
@@ -8934,7 +9277,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
8934
9277
|
runPass,
|
|
8935
9278
|
clock: new FakeClock(),
|
|
8936
9279
|
createWatcher: () => watcher,
|
|
8937
|
-
monotonicNow: () => (monotonic +=
|
|
9280
|
+
monotonicNow: () => (monotonic += DEFAULT_FULL_PASS_INTERVAL_MS),
|
|
8938
9281
|
sleep: sleep.sleep,
|
|
8939
9282
|
onShutdownSignal: (handler) => {
|
|
8940
9283
|
triggerShutdown = handler;
|
|
@@ -9916,9 +10259,7 @@ describe("CooperativePassScheduler", () => {
|
|
|
9916
10259
|
describe("first-tick realtime yield", () => {
|
|
9917
10260
|
it("commits a watcher push at the first chore checkpoint without a wall-clock wait", async () => {
|
|
9918
10261
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
9919
|
-
const previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
9920
10262
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
9921
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
9922
10263
|
const timings = [];
|
|
9923
10264
|
const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
9924
10265
|
const text = String(chunk);
|
|
@@ -10017,10 +10358,6 @@ describe("first-tick realtime yield", () => {
|
|
|
10017
10358
|
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10018
10359
|
else
|
|
10019
10360
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
10020
|
-
if (previousInterval === undefined)
|
|
10021
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10022
|
-
else
|
|
10023
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
|
|
10024
10361
|
}
|
|
10025
10362
|
});
|
|
10026
10363
|
});
|
|
@@ -10315,8 +10652,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10315
10652
|
capture.restore();
|
|
10316
10653
|
});
|
|
10317
10654
|
it("logs event-push-surfaces-inactive without silently attributing the first tick", async () => {
|
|
10318
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10319
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10320
10655
|
const capture = captureFullReconcileDecisions();
|
|
10321
10656
|
try {
|
|
10322
10657
|
const sleep = makeSteppableSleep();
|
|
@@ -10345,10 +10680,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10345
10680
|
}
|
|
10346
10681
|
finally {
|
|
10347
10682
|
capture.restore();
|
|
10348
|
-
if (saved === undefined)
|
|
10349
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10350
|
-
else
|
|
10351
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10352
10683
|
}
|
|
10353
10684
|
});
|
|
10354
10685
|
it("reports every matching trigger instead of short-circuiting at the first", async () => {
|
|
@@ -10403,9 +10734,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10403
10734
|
});
|
|
10404
10735
|
it("skips a 404 company route in a mixed drain and records a completed full reconcile", async () => {
|
|
10405
10736
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10406
|
-
const previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10407
10737
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
10408
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10409
10738
|
const failedLines = [];
|
|
10410
10739
|
const decisions = [];
|
|
10411
10740
|
const timingRecords = [];
|
|
@@ -10505,17 +10834,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10505
10834
|
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
10506
10835
|
else
|
|
10507
10836
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
10508
|
-
if (previousInterval === undefined)
|
|
10509
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10510
|
-
else
|
|
10511
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
|
|
10512
10837
|
}
|
|
10513
10838
|
});
|
|
10514
10839
|
it("does not escalate a fully-enumerated over-limit batch to a full reconcile", async () => {
|
|
10515
10840
|
const savedLimit = process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10516
|
-
const savedInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10517
10841
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = "2";
|
|
10518
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10519
10842
|
const journal = captureWatchLoopJournal();
|
|
10520
10843
|
try {
|
|
10521
10844
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -10555,17 +10878,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10555
10878
|
delete process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10556
10879
|
else
|
|
10557
10880
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = savedLimit;
|
|
10558
|
-
if (savedInterval === undefined)
|
|
10559
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10560
|
-
else
|
|
10561
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = savedInterval;
|
|
10562
10881
|
}
|
|
10563
10882
|
});
|
|
10564
10883
|
it("still escalates an overflowed batch with unknown routes to a full reconcile", async () => {
|
|
10565
10884
|
const savedLimit = process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10566
|
-
const savedInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10567
10885
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = "2";
|
|
10568
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
|
|
10569
10886
|
const journal = captureWatchLoopJournal();
|
|
10570
10887
|
try {
|
|
10571
10888
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -10597,15 +10914,9 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10597
10914
|
delete process.env.HQ_SYNC_EVENT_BATCH_LIMIT;
|
|
10598
10915
|
else
|
|
10599
10916
|
process.env.HQ_SYNC_EVENT_BATCH_LIMIT = savedLimit;
|
|
10600
|
-
if (savedInterval === undefined)
|
|
10601
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10602
|
-
else
|
|
10603
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = savedInterval;
|
|
10604
10917
|
}
|
|
10605
10918
|
});
|
|
10606
|
-
it("logs scheduled-interval as the sole full-reconcile trigger on the default
|
|
10607
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10608
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10919
|
+
it("logs scheduled-interval as the sole full-reconcile trigger on the six-hour default", async () => {
|
|
10609
10920
|
const capture = captureFullReconcileDecisions();
|
|
10610
10921
|
try {
|
|
10611
10922
|
const clockState = { now: 0 };
|
|
@@ -10631,11 +10942,11 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10631
10942
|
sleep.tick();
|
|
10632
10943
|
await flushLoopMicrotasks();
|
|
10633
10944
|
}
|
|
10634
|
-
// A dozen ticks inside the
|
|
10945
|
+
// A dozen ticks inside the interval: never a scheduled full pass.
|
|
10635
10946
|
expect(runPass).not.toHaveBeenCalled();
|
|
10636
10947
|
capture.clear();
|
|
10637
10948
|
runPass.mockClear();
|
|
10638
|
-
clockState.now =
|
|
10949
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
10639
10950
|
sleep.tick();
|
|
10640
10951
|
await flushLoopMicrotasks();
|
|
10641
10952
|
expectFullReconcileDecision(capture.records[0], "full", [
|
|
@@ -10647,10 +10958,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10647
10958
|
}
|
|
10648
10959
|
finally {
|
|
10649
10960
|
capture.restore();
|
|
10650
|
-
if (saved === undefined)
|
|
10651
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10652
|
-
else
|
|
10653
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10654
10961
|
}
|
|
10655
10962
|
});
|
|
10656
10963
|
it("logs scoped passes with their batch path count and preserves the scoped route", async () => {
|
|
@@ -10671,8 +10978,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10671
10978
|
capture.restore();
|
|
10672
10979
|
});
|
|
10673
10980
|
it("reports interval diagnostics while the configured interval has not elapsed", async () => {
|
|
10674
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10675
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
|
|
10676
10981
|
const capture = captureFullReconcileDecisions();
|
|
10677
10982
|
try {
|
|
10678
10983
|
const clockState = { now: 0 };
|
|
@@ -10681,6 +10986,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10681
10986
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
10682
10987
|
const loop = runRunnerWithLoop(watchArgv, {
|
|
10683
10988
|
runPass,
|
|
10989
|
+
fullPassIntervalSource: { read: () => 1_000 },
|
|
10684
10990
|
monotonicNow: () => clockState.now,
|
|
10685
10991
|
createWatcher: () => makeBatchWatcherStub(),
|
|
10686
10992
|
sleep: sleep.sleep,
|
|
@@ -10716,10 +11022,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
10716
11022
|
}
|
|
10717
11023
|
finally {
|
|
10718
11024
|
capture.restore();
|
|
10719
|
-
if (saved === undefined)
|
|
10720
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
10721
|
-
else
|
|
10722
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
10723
11025
|
}
|
|
10724
11026
|
});
|
|
10725
11027
|
it("startup runs one full reconcile, then watcher edits interrupt the pending wait", async () => {
|
|
@@ -11025,8 +11327,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11025
11327
|
await loop;
|
|
11026
11328
|
});
|
|
11027
11329
|
it("drops an unroutable batch and leaves the next unexpired poll idle", async () => {
|
|
11028
|
-
const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11029
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
|
|
11030
11330
|
const capture = captureFullReconcileDecisions();
|
|
11031
11331
|
try {
|
|
11032
11332
|
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
@@ -11052,10 +11352,6 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11052
11352
|
}
|
|
11053
11353
|
finally {
|
|
11054
11354
|
capture.restore();
|
|
11055
|
-
if (saved === undefined)
|
|
11056
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11057
|
-
else
|
|
11058
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11059
11355
|
}
|
|
11060
11356
|
});
|
|
11061
11357
|
it("personal scoped ticks preserve personal mode for the pull leg", async () => {
|
|
@@ -11214,7 +11510,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11214
11510
|
}
|
|
11215
11511
|
expect(runPass).not.toHaveBeenCalled();
|
|
11216
11512
|
runPass.mockClear();
|
|
11217
|
-
clockState.now =
|
|
11513
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
11218
11514
|
sleep.tick();
|
|
11219
11515
|
await flushLoopMicrotasks();
|
|
11220
11516
|
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
@@ -11222,134 +11518,105 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11222
11518
|
sleep.tick();
|
|
11223
11519
|
await loop;
|
|
11224
11520
|
});
|
|
11225
|
-
it("schedules the full reconcile by elapsed time, not tick count,
|
|
11521
|
+
it("schedules the full reconcile by elapsed time, not tick count, at the flag fallback", async () => {
|
|
11226
11522
|
// Pre-fix the unset default was every tenth tick. Ticks fire on every
|
|
11227
11523
|
// watcher batch, so an active vault ran a ~10-minute full pass every
|
|
11228
11524
|
// 4-8 minutes — nearly continuous, and the runner's dominant CPU cost
|
|
11229
11525
|
// once per-pass hashing was fixed.
|
|
11230
|
-
const
|
|
11231
|
-
|
|
11232
|
-
|
|
11233
|
-
|
|
11234
|
-
|
|
11235
|
-
|
|
11236
|
-
|
|
11237
|
-
|
|
11238
|
-
|
|
11239
|
-
|
|
11240
|
-
|
|
11241
|
-
|
|
11242
|
-
|
|
11243
|
-
|
|
11244
|
-
|
|
11245
|
-
|
|
11246
|
-
|
|
11247
|
-
|
|
11248
|
-
|
|
11249
|
-
|
|
11250
|
-
|
|
11251
|
-
|
|
11252
|
-
|
|
11253
|
-
// two full passes here.
|
|
11254
|
-
for (let tick = 0; tick < 20; tick++) {
|
|
11255
|
-
clockState.now += 177_000;
|
|
11256
|
-
sleep.tick();
|
|
11257
|
-
await flushLoopMicrotasks();
|
|
11258
|
-
}
|
|
11259
|
-
expect(runPass).not.toHaveBeenCalled();
|
|
11260
|
-
runPass.mockClear();
|
|
11261
|
-
clockState.now = DEFAULT_FULL_RECONCILE_MS;
|
|
11526
|
+
const clockState = { now: 0 };
|
|
11527
|
+
const watcher = makeBatchWatcherStub();
|
|
11528
|
+
const sleep = makeSteppableSleep();
|
|
11529
|
+
let triggerShutdown = () => { };
|
|
11530
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11531
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11532
|
+
runPass,
|
|
11533
|
+
clock: new FakeClock(),
|
|
11534
|
+
monotonicNow: () => clockState.now,
|
|
11535
|
+
createWatcher: () => watcher,
|
|
11536
|
+
sleep: sleep.sleep,
|
|
11537
|
+
onShutdownSignal: (handler) => {
|
|
11538
|
+
triggerShutdown = handler;
|
|
11539
|
+
return () => { };
|
|
11540
|
+
},
|
|
11541
|
+
});
|
|
11542
|
+
await flushLoopMicrotasks();
|
|
11543
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11544
|
+
runPass.mockClear();
|
|
11545
|
+
// Twenty ticks in fifty-nine minutes: the old cadence would have run
|
|
11546
|
+
// two full passes here.
|
|
11547
|
+
for (let tick = 0; tick < 20; tick++) {
|
|
11548
|
+
clockState.now += 177_000;
|
|
11262
11549
|
sleep.tick();
|
|
11263
11550
|
await flushLoopMicrotasks();
|
|
11264
|
-
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11265
|
-
triggerShutdown();
|
|
11266
|
-
await loop;
|
|
11267
|
-
}
|
|
11268
|
-
finally {
|
|
11269
|
-
if (saved === undefined)
|
|
11270
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11271
|
-
else
|
|
11272
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11273
11551
|
}
|
|
11552
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
11553
|
+
runPass.mockClear();
|
|
11554
|
+
clockState.now = DEFAULT_FULL_PASS_INTERVAL_MS;
|
|
11555
|
+
sleep.tick();
|
|
11556
|
+
await flushLoopMicrotasks();
|
|
11557
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11558
|
+
triggerShutdown();
|
|
11559
|
+
await loop;
|
|
11274
11560
|
});
|
|
11275
|
-
it("uses the
|
|
11276
|
-
const
|
|
11277
|
-
|
|
11278
|
-
|
|
11279
|
-
|
|
11280
|
-
|
|
11281
|
-
|
|
11282
|
-
|
|
11283
|
-
|
|
11284
|
-
|
|
11285
|
-
|
|
11286
|
-
|
|
11287
|
-
|
|
11288
|
-
|
|
11289
|
-
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
|
|
11300
|
-
|
|
11301
|
-
|
|
11302
|
-
|
|
11303
|
-
|
|
11304
|
-
|
|
11305
|
-
|
|
11306
|
-
|
|
11307
|
-
|
|
11308
|
-
triggerShutdown();
|
|
11309
|
-
await loop;
|
|
11310
|
-
}
|
|
11311
|
-
finally {
|
|
11312
|
-
if (saved === undefined)
|
|
11313
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11314
|
-
else
|
|
11315
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11316
|
-
}
|
|
11561
|
+
it("uses the hq-flag interval source instead of the tick count", async () => {
|
|
11562
|
+
const clockState = { now: 0 };
|
|
11563
|
+
const watcher = makeBatchWatcherStub();
|
|
11564
|
+
const sleep = makeSteppableSleep();
|
|
11565
|
+
let triggerShutdown = () => { };
|
|
11566
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11567
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11568
|
+
runPass,
|
|
11569
|
+
fullPassIntervalSource: { read: () => 1_000 },
|
|
11570
|
+
clock: new FakeClock(),
|
|
11571
|
+
monotonicNow: () => clockState.now,
|
|
11572
|
+
createWatcher: () => watcher,
|
|
11573
|
+
sleep: sleep.sleep,
|
|
11574
|
+
onShutdownSignal: (handler) => {
|
|
11575
|
+
triggerShutdown = handler;
|
|
11576
|
+
return () => { };
|
|
11577
|
+
},
|
|
11578
|
+
});
|
|
11579
|
+
await flushLoopMicrotasks();
|
|
11580
|
+
// Startup always forces the first full reconcile, and starts the interval.
|
|
11581
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11582
|
+
runPass.mockClear();
|
|
11583
|
+
clockState.now = 999;
|
|
11584
|
+
sleep.tick();
|
|
11585
|
+
await flushLoopMicrotasks();
|
|
11586
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
11587
|
+
runPass.mockClear();
|
|
11588
|
+
clockState.now = 1_000;
|
|
11589
|
+
sleep.tick();
|
|
11590
|
+
await flushLoopMicrotasks();
|
|
11591
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11592
|
+
triggerShutdown();
|
|
11593
|
+
await loop;
|
|
11317
11594
|
});
|
|
11318
11595
|
it("does not let a full-reconcile interval suppress event-push activation", async () => {
|
|
11319
|
-
const
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11326
|
-
|
|
11327
|
-
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11339
|
-
|
|
11340
|
-
|
|
11341
|
-
|
|
11342
|
-
|
|
11343
|
-
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11344
|
-
triggerShutdown();
|
|
11345
|
-
await loop;
|
|
11346
|
-
}
|
|
11347
|
-
finally {
|
|
11348
|
-
if (saved === undefined)
|
|
11349
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11350
|
-
else
|
|
11351
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11352
|
-
}
|
|
11596
|
+
const watcher = makeBatchWatcherStub();
|
|
11597
|
+
const sleep = makeSteppableSleep();
|
|
11598
|
+
let triggerShutdown = () => { };
|
|
11599
|
+
// A numeric partial result does not activate event-push surfaces, so the
|
|
11600
|
+
// next tick must force another full pass even though three hours have not elapsed.
|
|
11601
|
+
const runPass = vi.fn().mockResolvedValue(PARTIAL_SYNC_EXIT);
|
|
11602
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11603
|
+
runPass,
|
|
11604
|
+
clock: new FakeClock(),
|
|
11605
|
+
monotonicNow: () => 0,
|
|
11606
|
+
createWatcher: () => watcher,
|
|
11607
|
+
sleep: sleep.sleep,
|
|
11608
|
+
onShutdownSignal: (handler) => {
|
|
11609
|
+
triggerShutdown = handler;
|
|
11610
|
+
return () => { };
|
|
11611
|
+
},
|
|
11612
|
+
});
|
|
11613
|
+
await flushLoopMicrotasks();
|
|
11614
|
+
runPass.mockClear();
|
|
11615
|
+
sleep.tick();
|
|
11616
|
+
await flushLoopMicrotasks();
|
|
11617
|
+
expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
|
|
11618
|
+
triggerShutdown();
|
|
11619
|
+
await loop;
|
|
11353
11620
|
});
|
|
11354
11621
|
it("overflow with known dropped-route hints fans out per touched route instead of a full reconcile", async () => {
|
|
11355
11622
|
const capture = captureFullReconcileDecisions();
|
|
@@ -11484,50 +11751,40 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
11484
11751
|
await loop;
|
|
11485
11752
|
});
|
|
11486
11753
|
it("a batch above the shipped 10,000-path default stays on chunked targeted passes before the interval", async () => {
|
|
11487
|
-
const
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11491
|
-
|
|
11492
|
-
|
|
11493
|
-
|
|
11494
|
-
|
|
11495
|
-
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
|
|
11499
|
-
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
|
-
|
|
11505
|
-
|
|
11506
|
-
|
|
11507
|
-
|
|
11508
|
-
|
|
11509
|
-
|
|
11510
|
-
|
|
11511
|
-
|
|
11512
|
-
|
|
11513
|
-
|
|
11514
|
-
|
|
11515
|
-
|
|
11516
|
-
|
|
11517
|
-
|
|
11518
|
-
|
|
11519
|
-
|
|
11520
|
-
|
|
11521
|
-
expect(pushScopeCounts.reduce((sum, count) => sum + count, 0)).toBe(10_001);
|
|
11522
|
-
triggerShutdown();
|
|
11523
|
-
await loop;
|
|
11524
|
-
}
|
|
11525
|
-
finally {
|
|
11526
|
-
if (saved === undefined)
|
|
11527
|
-
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
11528
|
-
else
|
|
11529
|
-
process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
|
|
11530
|
-
}
|
|
11754
|
+
const watcher = makeBatchWatcherStub();
|
|
11755
|
+
const sleep = makeSteppableSleep();
|
|
11756
|
+
let triggerShutdown = () => { };
|
|
11757
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
11758
|
+
const loop = runRunnerWithLoop(watchArgv, {
|
|
11759
|
+
runPass,
|
|
11760
|
+
clock: new FakeClock(),
|
|
11761
|
+
monotonicNow: () => 0,
|
|
11762
|
+
createWatcher: () => watcher,
|
|
11763
|
+
sleep: sleep.sleep,
|
|
11764
|
+
onShutdownSignal: (handler) => {
|
|
11765
|
+
triggerShutdown = handler;
|
|
11766
|
+
return () => { };
|
|
11767
|
+
},
|
|
11768
|
+
});
|
|
11769
|
+
await flushLoopMicrotasks();
|
|
11770
|
+
runPass.mockClear();
|
|
11771
|
+
watcher.emit("companies/indigo/a.md", {
|
|
11772
|
+
paths: new Map(Array.from({ length: 10_001 }, (_, index) => [
|
|
11773
|
+
`/tmp/hq/companies/indigo/${index}.md`,
|
|
11774
|
+
`companies/indigo/${index}.md`,
|
|
11775
|
+
])),
|
|
11776
|
+
});
|
|
11777
|
+
sleep.tick();
|
|
11778
|
+
await flushLoopMicrotasks(40);
|
|
11779
|
+
const calls = runPass.mock.calls.map((call) => call[0]);
|
|
11780
|
+
expect(calls).not.toContainEqual(fullArgv);
|
|
11781
|
+
const pushScopeCounts = calls
|
|
11782
|
+
.filter((argv) => argv.includes("push"))
|
|
11783
|
+
.map((argv) => scopePathsFromArgv(argv).length);
|
|
11784
|
+
expect(Math.max(0, ...pushScopeCounts)).toBeLessThanOrEqual(10_000);
|
|
11785
|
+
expect(pushScopeCounts.reduce((sum, count) => sum + count, 0)).toBe(10_001);
|
|
11786
|
+
triggerShutdown();
|
|
11787
|
+
await loop;
|
|
11531
11788
|
});
|
|
11532
11789
|
it("a throwing targeted pass falls back to one full reconcile instead of dropping the change", async () => {
|
|
11533
11790
|
const capture = captureFullReconcileDecisions();
|