@indigoai-us/hq-cloud 6.16.75-alpha.0 → 6.16.75-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +24 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +89 -11
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +185 -118
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -5014,6 +5014,60 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5014
5014
|
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
|
|
5015
5015
|
}
|
|
5016
5016
|
});
|
|
5017
|
+
it("backs off failed parallel full-reconcile completions before another chore is admitted", async () => {
|
|
5018
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5019
|
+
const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
5020
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
5021
|
+
process.env.HQ_SYNC_FULL_RECONCILE_MS = "1";
|
|
5022
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
5023
|
+
const delays = [];
|
|
5024
|
+
const waiters = [];
|
|
5025
|
+
let now = 0;
|
|
5026
|
+
let shutdown = () => { };
|
|
5027
|
+
const sleep = (ms) => {
|
|
5028
|
+
delays.push(ms);
|
|
5029
|
+
return new Promise((resolve) => {
|
|
5030
|
+
waiters.push(resolve);
|
|
5031
|
+
});
|
|
5032
|
+
};
|
|
5033
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
5034
|
+
runPass,
|
|
5035
|
+
clock: new FakeClock(),
|
|
5036
|
+
monotonicNow: () => now,
|
|
5037
|
+
sleep,
|
|
5038
|
+
onShutdownSignal: (handler) => {
|
|
5039
|
+
shutdown = handler;
|
|
5040
|
+
return () => { };
|
|
5041
|
+
},
|
|
5042
|
+
});
|
|
5043
|
+
try {
|
|
5044
|
+
for (let attempt = 0; attempt < 50 && runPass.mock.calls.length < 3; attempt += 1) {
|
|
5045
|
+
const resolve = waiters.shift();
|
|
5046
|
+
if (resolve) {
|
|
5047
|
+
now += 1;
|
|
5048
|
+
resolve();
|
|
5049
|
+
await flushLoopMicrotasks(30);
|
|
5050
|
+
}
|
|
5051
|
+
else {
|
|
5052
|
+
await new Promise((resolveNext) => setImmediate(resolveNext));
|
|
5053
|
+
}
|
|
5054
|
+
}
|
|
5055
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
5056
|
+
expect(delays).toContain(60_000);
|
|
5057
|
+
}
|
|
5058
|
+
finally {
|
|
5059
|
+
shutdown();
|
|
5060
|
+
await expect(loop).resolves.toBe(0);
|
|
5061
|
+
if (previousCoordinator === undefined)
|
|
5062
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5063
|
+
else
|
|
5064
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
5065
|
+
if (previousFullReconcileMs === undefined)
|
|
5066
|
+
delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
|
|
5067
|
+
else
|
|
5068
|
+
process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
|
|
5069
|
+
}
|
|
5070
|
+
});
|
|
5017
5071
|
it("parallel coordinator keeps event surfaces off for an auth-required full reconcile", async () => {
|
|
5018
5072
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5019
5073
|
const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
|
|
@@ -6207,124 +6261,6 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6207
6261
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6208
6262
|
}
|
|
6209
6263
|
});
|
|
6210
|
-
it("measures a queued realtime pass at the operation gate and marks append lock only after admission", async () => {
|
|
6211
|
-
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6212
|
-
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6213
|
-
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-telemetry-operation-gate-"));
|
|
6214
|
-
const watcher = makeBatchWatcherStub();
|
|
6215
|
-
let receiverSync;
|
|
6216
|
-
let shutdown = () => { };
|
|
6217
|
-
let releaseGuard = () => { };
|
|
6218
|
-
const guard = new Promise((resolve) => { releaseGuard = resolve; });
|
|
6219
|
-
let markGuardStarted = () => { };
|
|
6220
|
-
const guardStarted = new Promise((resolve) => { markGuardStarted = resolve; });
|
|
6221
|
-
let markTargetStarted = () => { };
|
|
6222
|
-
const targetStarted = new Promise((resolve) => { markTargetStarted = resolve; });
|
|
6223
|
-
let monotonicMs = 0;
|
|
6224
|
-
let guardReleased = false;
|
|
6225
|
-
let targetStartedAt;
|
|
6226
|
-
const timings = [];
|
|
6227
|
-
const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
6228
|
-
if (typeof chunk === "string" && chunk.startsWith("{")) {
|
|
6229
|
-
try {
|
|
6230
|
-
const record = JSON.parse(chunk);
|
|
6231
|
-
if (typeof record === "object" &&
|
|
6232
|
-
record !== null &&
|
|
6233
|
-
record.event === "sync.realtime.timing")
|
|
6234
|
-
timings.push(record);
|
|
6235
|
-
}
|
|
6236
|
-
catch {
|
|
6237
|
-
// Non-telemetry diagnostics are outside this assertion.
|
|
6238
|
-
}
|
|
6239
|
-
}
|
|
6240
|
-
return true;
|
|
6241
|
-
});
|
|
6242
|
-
const includesScope = (argv, basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
|
|
6243
|
-
const runPass = vi.fn(async (argv) => {
|
|
6244
|
-
if (includesScope(argv, "guard.md")) {
|
|
6245
|
-
markGuardStarted();
|
|
6246
|
-
await guard;
|
|
6247
|
-
return completedPassOutcome();
|
|
6248
|
-
}
|
|
6249
|
-
if (includesScope(argv, "target.md")) {
|
|
6250
|
-
expect(guardReleased).toBe(true);
|
|
6251
|
-
targetStartedAt = monotonicMs;
|
|
6252
|
-
markTargetStarted();
|
|
6253
|
-
monotonicMs += 1;
|
|
6254
|
-
}
|
|
6255
|
-
return completedPassOutcome();
|
|
6256
|
-
});
|
|
6257
|
-
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6258
|
-
runPass,
|
|
6259
|
-
createWatcher: () => watcher,
|
|
6260
|
-
createReceiver: ({ syncBatchFn }) => {
|
|
6261
|
-
receiverSync = syncBatchFn;
|
|
6262
|
-
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6263
|
-
},
|
|
6264
|
-
monotonicNow: () => monotonicMs,
|
|
6265
|
-
sleep: () => new Promise(() => { }),
|
|
6266
|
-
onShutdownSignal: (handler) => {
|
|
6267
|
-
shutdown = handler;
|
|
6268
|
-
return () => { };
|
|
6269
|
-
},
|
|
6270
|
-
});
|
|
6271
|
-
const event = (relativePath) => ({
|
|
6272
|
-
kind: "upsert",
|
|
6273
|
-
relativePath,
|
|
6274
|
-
contentHash: "sha256:remote",
|
|
6275
|
-
mtime: "2026-09-23T00:00:00.000Z",
|
|
6276
|
-
originDeviceId: "peer-device",
|
|
6277
|
-
originTenantId: "tenant-indigo",
|
|
6278
|
-
sequenceNumber: 1,
|
|
6279
|
-
eventTimestamp: "2026-09-23T00:00:00.000Z",
|
|
6280
|
-
});
|
|
6281
|
-
try {
|
|
6282
|
-
await flushLoopMicrotasks(30);
|
|
6283
|
-
runPass.mockClear();
|
|
6284
|
-
const guardedPass = receiverSync({
|
|
6285
|
-
events: [event("companies/alpha/guard.md")],
|
|
6286
|
-
signal: new AbortController().signal,
|
|
6287
|
-
});
|
|
6288
|
-
await guardStarted;
|
|
6289
|
-
const targetAbsolutePath = path.join(hqRoot, "companies", "beta", "target.md");
|
|
6290
|
-
fs.mkdirSync(path.dirname(targetAbsolutePath), { recursive: true });
|
|
6291
|
-
fs.writeFileSync(targetAbsolutePath, "x");
|
|
6292
|
-
watcher.emit("companies/beta/target.md", {
|
|
6293
|
-
paths: new Map([
|
|
6294
|
-
[targetAbsolutePath, "companies/beta/target.md"],
|
|
6295
|
-
]),
|
|
6296
|
-
changes: new Map(),
|
|
6297
|
-
});
|
|
6298
|
-
await flushLoopMicrotasks(30);
|
|
6299
|
-
expect(targetStartedAt).toBeUndefined();
|
|
6300
|
-
monotonicMs += 25;
|
|
6301
|
-
guardReleased = true;
|
|
6302
|
-
releaseGuard();
|
|
6303
|
-
await guardedPass;
|
|
6304
|
-
await targetStarted;
|
|
6305
|
-
await flushLoopMicrotasks(30);
|
|
6306
|
-
expect(targetStartedAt).toBe(25);
|
|
6307
|
-
expect(timings).toHaveLength(2);
|
|
6308
|
-
const targetTiming = timings.find((record) => record.source === "watcher" &&
|
|
6309
|
-
record.direction === "push" &&
|
|
6310
|
-
record.outcome === "durable");
|
|
6311
|
-
expect(targetTiming).toBeDefined();
|
|
6312
|
-
expect(targetTiming?.timingMs?.operationGate).toBe(25);
|
|
6313
|
-
expect(targetTiming?.timingMs?.appendLock).toBe(0);
|
|
6314
|
-
}
|
|
6315
|
-
finally {
|
|
6316
|
-
guardReleased = true;
|
|
6317
|
-
releaseGuard();
|
|
6318
|
-
shutdown();
|
|
6319
|
-
await loop;
|
|
6320
|
-
stderr.mockRestore();
|
|
6321
|
-
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6322
|
-
if (previousCoordinator === undefined)
|
|
6323
|
-
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6324
|
-
else
|
|
6325
|
-
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6326
|
-
}
|
|
6327
|
-
});
|
|
6328
6264
|
it("REGRESSION: a fast receiver pull overtakes a slow watcher push and records admitted occupancy", async () => {
|
|
6329
6265
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6330
6266
|
const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
|
|
@@ -9016,6 +8952,137 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
9016
8952
|
await expect(loop).resolves.toBe(0);
|
|
9017
8953
|
expect(watcher.disposed).toBe(false);
|
|
9018
8954
|
});
|
|
8955
|
+
it("bounds repeated watch-pass exits with a fake-clock retry budget while keeping the runner alive", async () => {
|
|
8956
|
+
const watcher = makeWatcherStub();
|
|
8957
|
+
const runPass = vi.fn().mockResolvedValue(1);
|
|
8958
|
+
const delays = [];
|
|
8959
|
+
const waiters = [];
|
|
8960
|
+
let now = 0;
|
|
8961
|
+
let shutdown = () => { };
|
|
8962
|
+
let settled;
|
|
8963
|
+
const sleep = (ms) => {
|
|
8964
|
+
delays.push(ms);
|
|
8965
|
+
return new Promise((resolve) => {
|
|
8966
|
+
waiters.push(resolve);
|
|
8967
|
+
});
|
|
8968
|
+
};
|
|
8969
|
+
const tick = async (advanceMs) => {
|
|
8970
|
+
now += advanceMs;
|
|
8971
|
+
const resolve = waiters.shift();
|
|
8972
|
+
if (!resolve)
|
|
8973
|
+
throw new Error("retry sleep was not pending");
|
|
8974
|
+
resolve();
|
|
8975
|
+
await flushLoopMicrotasks();
|
|
8976
|
+
};
|
|
8977
|
+
const waitForRetrySleep = async () => {
|
|
8978
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
8979
|
+
if (waiters.length > 0)
|
|
8980
|
+
return;
|
|
8981
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
8982
|
+
}
|
|
8983
|
+
throw new Error("timed out waiting for retry sleep");
|
|
8984
|
+
};
|
|
8985
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
8986
|
+
runPass,
|
|
8987
|
+
clock: new FakeClock(),
|
|
8988
|
+
createWatcher: () => watcher,
|
|
8989
|
+
monotonicNow: () => now,
|
|
8990
|
+
sleep,
|
|
8991
|
+
onShutdownSignal: (handler) => {
|
|
8992
|
+
shutdown = handler;
|
|
8993
|
+
return () => { };
|
|
8994
|
+
},
|
|
8995
|
+
}).then((code) => {
|
|
8996
|
+
settled = code;
|
|
8997
|
+
return code;
|
|
8998
|
+
});
|
|
8999
|
+
try {
|
|
9000
|
+
await waitForRetrySleep();
|
|
9001
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9002
|
+
expect(delays).toEqual([1]);
|
|
9003
|
+
await tick(1_000);
|
|
9004
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9005
|
+
expect(delays).toEqual([1, 1]);
|
|
9006
|
+
await tick(1_000);
|
|
9007
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9008
|
+
expect(delays).toEqual([1, 1, 60_000]);
|
|
9009
|
+
await tick(60_000);
|
|
9010
|
+
await tick(120_000);
|
|
9011
|
+
await tick(240_000);
|
|
9012
|
+
await tick(480_000);
|
|
9013
|
+
expect(runPass).toHaveBeenCalledTimes(7);
|
|
9014
|
+
expect(delays).toEqual([
|
|
9015
|
+
1,
|
|
9016
|
+
1,
|
|
9017
|
+
60_000,
|
|
9018
|
+
120_000,
|
|
9019
|
+
240_000,
|
|
9020
|
+
480_000,
|
|
9021
|
+
600_000,
|
|
9022
|
+
]);
|
|
9023
|
+
// The retry window is quiet-time based, not anchored to the first
|
|
9024
|
+
// failure. Continuing failures after thirty minutes must remain capped.
|
|
9025
|
+
await tick(600_000);
|
|
9026
|
+
await tick(600_000);
|
|
9027
|
+
expect(runPass).toHaveBeenCalledTimes(9);
|
|
9028
|
+
expect(delays.slice(-2)).toEqual([600_000, 600_000]);
|
|
9029
|
+
expect(settled).toBeUndefined();
|
|
9030
|
+
expect(watcher.disposed).toBe(false);
|
|
9031
|
+
}
|
|
9032
|
+
finally {
|
|
9033
|
+
shutdown();
|
|
9034
|
+
await expect(loop).resolves.toBe(0);
|
|
9035
|
+
}
|
|
9036
|
+
});
|
|
9037
|
+
it("does not let a watcher interrupt bypass a failed-pass retry deadline", async () => {
|
|
9038
|
+
const watcher = makeWatcherStub();
|
|
9039
|
+
let passes = 0;
|
|
9040
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
9041
|
+
passes += 1;
|
|
9042
|
+
return passes === 1 ? 0 : 1;
|
|
9043
|
+
});
|
|
9044
|
+
const waiters = [];
|
|
9045
|
+
const now = 0;
|
|
9046
|
+
let shutdown = () => { };
|
|
9047
|
+
const sleep = vi.fn(() => new Promise((resolve) => {
|
|
9048
|
+
waiters.push(resolve);
|
|
9049
|
+
}));
|
|
9050
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
|
|
9051
|
+
runPass,
|
|
9052
|
+
clock: new FakeClock(),
|
|
9053
|
+
createWatcher: () => watcher,
|
|
9054
|
+
monotonicNow: () => now,
|
|
9055
|
+
sleep,
|
|
9056
|
+
onShutdownSignal: (handler) => {
|
|
9057
|
+
shutdown = handler;
|
|
9058
|
+
return () => { };
|
|
9059
|
+
},
|
|
9060
|
+
});
|
|
9061
|
+
try {
|
|
9062
|
+
await waitForPollSleep({
|
|
9063
|
+
pendingCount: () => waiters.length,
|
|
9064
|
+
});
|
|
9065
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
9066
|
+
watcher.emit();
|
|
9067
|
+
await flushLoopMicrotasks(30);
|
|
9068
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
9069
|
+
watcher.emit();
|
|
9070
|
+
await flushLoopMicrotasks(30);
|
|
9071
|
+
expect(runPass).toHaveBeenCalledTimes(3);
|
|
9072
|
+
watcher.emit();
|
|
9073
|
+
await flushLoopMicrotasks(30);
|
|
9074
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9075
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9076
|
+
watcher.emit();
|
|
9077
|
+
await flushLoopMicrotasks(30);
|
|
9078
|
+
expect(runPass).toHaveBeenCalledTimes(4);
|
|
9079
|
+
expect(sleep).toHaveBeenLastCalledWith(60_000);
|
|
9080
|
+
}
|
|
9081
|
+
finally {
|
|
9082
|
+
shutdown();
|
|
9083
|
+
await expect(loop).resolves.toBe(0);
|
|
9084
|
+
}
|
|
9085
|
+
});
|
|
9019
9086
|
it("alerts when a partial pass failed before a company's download leg", async () => {
|
|
9020
9087
|
const watcher = makeWatcherStub();
|
|
9021
9088
|
const stderrWrite = vi
|