@indigoai-us/hq-cloud 6.16.75-alpha.0 → 6.16.75-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5014,6 +5014,60 @@ describe("runRunnerWithLoop — event-push wiring", () => {
5014
5014
  process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
5015
5015
  }
5016
5016
  });
5017
+ it("backs off failed parallel full-reconcile completions before another chore is admitted", async () => {
5018
+ const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
5019
+ const previousFullReconcileMs = process.env.HQ_SYNC_FULL_RECONCILE_MS;
5020
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
5021
+ process.env.HQ_SYNC_FULL_RECONCILE_MS = "1";
5022
+ const runPass = vi.fn().mockResolvedValue(1);
5023
+ const delays = [];
5024
+ const waiters = [];
5025
+ let now = 0;
5026
+ let shutdown = () => { };
5027
+ const sleep = (ms) => {
5028
+ delays.push(ms);
5029
+ return new Promise((resolve) => {
5030
+ waiters.push(resolve);
5031
+ });
5032
+ };
5033
+ const loop = runRunnerWithLoop(["--companies", "--watch", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
5034
+ runPass,
5035
+ clock: new FakeClock(),
5036
+ monotonicNow: () => now,
5037
+ sleep,
5038
+ onShutdownSignal: (handler) => {
5039
+ shutdown = handler;
5040
+ return () => { };
5041
+ },
5042
+ });
5043
+ try {
5044
+ for (let attempt = 0; attempt < 50 && runPass.mock.calls.length < 3; attempt += 1) {
5045
+ const resolve = waiters.shift();
5046
+ if (resolve) {
5047
+ now += 1;
5048
+ resolve();
5049
+ await flushLoopMicrotasks(30);
5050
+ }
5051
+ else {
5052
+ await new Promise((resolveNext) => setImmediate(resolveNext));
5053
+ }
5054
+ }
5055
+ expect(runPass).toHaveBeenCalledTimes(3);
5056
+ expect(delays).toContain(60_000);
5057
+ }
5058
+ finally {
5059
+ shutdown();
5060
+ await expect(loop).resolves.toBe(0);
5061
+ if (previousCoordinator === undefined)
5062
+ delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
5063
+ else
5064
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
5065
+ if (previousFullReconcileMs === undefined)
5066
+ delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
5067
+ else
5068
+ process.env.HQ_SYNC_FULL_RECONCILE_MS = previousFullReconcileMs;
5069
+ }
5070
+ });
5017
5071
  it("parallel coordinator keeps event surfaces off for an auth-required full reconcile", async () => {
5018
5072
  const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
5019
5073
  const previousEventSync = process.env.HQ_SYNC_EVENT_SYNC;
@@ -6207,124 +6261,6 @@ describe("runRunnerWithLoop — event-push wiring", () => {
6207
6261
  process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
6208
6262
  }
6209
6263
  });
6210
- it("measures a queued realtime pass at the operation gate and marks append lock only after admission", async () => {
6211
- const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6212
- process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
6213
- const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-telemetry-operation-gate-"));
6214
- const watcher = makeBatchWatcherStub();
6215
- let receiverSync;
6216
- let shutdown = () => { };
6217
- let releaseGuard = () => { };
6218
- const guard = new Promise((resolve) => { releaseGuard = resolve; });
6219
- let markGuardStarted = () => { };
6220
- const guardStarted = new Promise((resolve) => { markGuardStarted = resolve; });
6221
- let markTargetStarted = () => { };
6222
- const targetStarted = new Promise((resolve) => { markTargetStarted = resolve; });
6223
- let monotonicMs = 0;
6224
- let guardReleased = false;
6225
- let targetStartedAt;
6226
- const timings = [];
6227
- const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
6228
- if (typeof chunk === "string" && chunk.startsWith("{")) {
6229
- try {
6230
- const record = JSON.parse(chunk);
6231
- if (typeof record === "object" &&
6232
- record !== null &&
6233
- record.event === "sync.realtime.timing")
6234
- timings.push(record);
6235
- }
6236
- catch {
6237
- // Non-telemetry diagnostics are outside this assertion.
6238
- }
6239
- }
6240
- return true;
6241
- });
6242
- const includesScope = (argv, basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
6243
- const runPass = vi.fn(async (argv) => {
6244
- if (includesScope(argv, "guard.md")) {
6245
- markGuardStarted();
6246
- await guard;
6247
- return completedPassOutcome();
6248
- }
6249
- if (includesScope(argv, "target.md")) {
6250
- expect(guardReleased).toBe(true);
6251
- targetStartedAt = monotonicMs;
6252
- markTargetStarted();
6253
- monotonicMs += 1;
6254
- }
6255
- return completedPassOutcome();
6256
- });
6257
- const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
6258
- runPass,
6259
- createWatcher: () => watcher,
6260
- createReceiver: ({ syncBatchFn }) => {
6261
- receiverSync = syncBatchFn;
6262
- return { connected: true, start: async () => { }, dispose: async () => { } };
6263
- },
6264
- monotonicNow: () => monotonicMs,
6265
- sleep: () => new Promise(() => { }),
6266
- onShutdownSignal: (handler) => {
6267
- shutdown = handler;
6268
- return () => { };
6269
- },
6270
- });
6271
- const event = (relativePath) => ({
6272
- kind: "upsert",
6273
- relativePath,
6274
- contentHash: "sha256:remote",
6275
- mtime: "2026-09-23T00:00:00.000Z",
6276
- originDeviceId: "peer-device",
6277
- originTenantId: "tenant-indigo",
6278
- sequenceNumber: 1,
6279
- eventTimestamp: "2026-09-23T00:00:00.000Z",
6280
- });
6281
- try {
6282
- await flushLoopMicrotasks(30);
6283
- runPass.mockClear();
6284
- const guardedPass = receiverSync({
6285
- events: [event("companies/alpha/guard.md")],
6286
- signal: new AbortController().signal,
6287
- });
6288
- await guardStarted;
6289
- const targetAbsolutePath = path.join(hqRoot, "companies", "beta", "target.md");
6290
- fs.mkdirSync(path.dirname(targetAbsolutePath), { recursive: true });
6291
- fs.writeFileSync(targetAbsolutePath, "x");
6292
- watcher.emit("companies/beta/target.md", {
6293
- paths: new Map([
6294
- [targetAbsolutePath, "companies/beta/target.md"],
6295
- ]),
6296
- changes: new Map(),
6297
- });
6298
- await flushLoopMicrotasks(30);
6299
- expect(targetStartedAt).toBeUndefined();
6300
- monotonicMs += 25;
6301
- guardReleased = true;
6302
- releaseGuard();
6303
- await guardedPass;
6304
- await targetStarted;
6305
- await flushLoopMicrotasks(30);
6306
- expect(targetStartedAt).toBe(25);
6307
- expect(timings).toHaveLength(2);
6308
- const targetTiming = timings.find((record) => record.source === "watcher" &&
6309
- record.direction === "push" &&
6310
- record.outcome === "durable");
6311
- expect(targetTiming).toBeDefined();
6312
- expect(targetTiming?.timingMs?.operationGate).toBe(25);
6313
- expect(targetTiming?.timingMs?.appendLock).toBe(0);
6314
- }
6315
- finally {
6316
- guardReleased = true;
6317
- releaseGuard();
6318
- shutdown();
6319
- await loop;
6320
- stderr.mockRestore();
6321
- fs.rmSync(hqRoot, { recursive: true, force: true });
6322
- if (previousCoordinator === undefined)
6323
- delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6324
- else
6325
- process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
6326
- }
6327
- });
6328
6264
  it("REGRESSION: a fast receiver pull overtakes a slow watcher push and records admitted occupancy", async () => {
6329
6265
  const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6330
6266
  const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
@@ -9016,6 +8952,137 @@ describe("runRunnerWithLoop — event-push wiring", () => {
9016
8952
  await expect(loop).resolves.toBe(0);
9017
8953
  expect(watcher.disposed).toBe(false);
9018
8954
  });
8955
+ it("bounds repeated watch-pass exits with a fake-clock retry budget while keeping the runner alive", async () => {
8956
+ const watcher = makeWatcherStub();
8957
+ const runPass = vi.fn().mockResolvedValue(1);
8958
+ const delays = [];
8959
+ const waiters = [];
8960
+ let now = 0;
8961
+ let shutdown = () => { };
8962
+ let settled;
8963
+ const sleep = (ms) => {
8964
+ delays.push(ms);
8965
+ return new Promise((resolve) => {
8966
+ waiters.push(resolve);
8967
+ });
8968
+ };
8969
+ const tick = async (advanceMs) => {
8970
+ now += advanceMs;
8971
+ const resolve = waiters.shift();
8972
+ if (!resolve)
8973
+ throw new Error("retry sleep was not pending");
8974
+ resolve();
8975
+ await flushLoopMicrotasks();
8976
+ };
8977
+ const waitForRetrySleep = async () => {
8978
+ for (let attempt = 0; attempt < 200; attempt += 1) {
8979
+ if (waiters.length > 0)
8980
+ return;
8981
+ await new Promise((resolve) => setImmediate(resolve));
8982
+ }
8983
+ throw new Error("timed out waiting for retry sleep");
8984
+ };
8985
+ const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
8986
+ runPass,
8987
+ clock: new FakeClock(),
8988
+ createWatcher: () => watcher,
8989
+ monotonicNow: () => now,
8990
+ sleep,
8991
+ onShutdownSignal: (handler) => {
8992
+ shutdown = handler;
8993
+ return () => { };
8994
+ },
8995
+ }).then((code) => {
8996
+ settled = code;
8997
+ return code;
8998
+ });
8999
+ try {
9000
+ await waitForRetrySleep();
9001
+ expect(runPass).toHaveBeenCalledTimes(1);
9002
+ expect(delays).toEqual([1]);
9003
+ await tick(1_000);
9004
+ expect(runPass).toHaveBeenCalledTimes(2);
9005
+ expect(delays).toEqual([1, 1]);
9006
+ await tick(1_000);
9007
+ expect(runPass).toHaveBeenCalledTimes(3);
9008
+ expect(delays).toEqual([1, 1, 60_000]);
9009
+ await tick(60_000);
9010
+ await tick(120_000);
9011
+ await tick(240_000);
9012
+ await tick(480_000);
9013
+ expect(runPass).toHaveBeenCalledTimes(7);
9014
+ expect(delays).toEqual([
9015
+ 1,
9016
+ 1,
9017
+ 60_000,
9018
+ 120_000,
9019
+ 240_000,
9020
+ 480_000,
9021
+ 600_000,
9022
+ ]);
9023
+ // The retry window is quiet-time based, not anchored to the first
9024
+ // failure. Continuing failures after thirty minutes must remain capped.
9025
+ await tick(600_000);
9026
+ await tick(600_000);
9027
+ expect(runPass).toHaveBeenCalledTimes(9);
9028
+ expect(delays.slice(-2)).toEqual([600_000, 600_000]);
9029
+ expect(settled).toBeUndefined();
9030
+ expect(watcher.disposed).toBe(false);
9031
+ }
9032
+ finally {
9033
+ shutdown();
9034
+ await expect(loop).resolves.toBe(0);
9035
+ }
9036
+ });
9037
+ it("does not let a watcher interrupt bypass a failed-pass retry deadline", async () => {
9038
+ const watcher = makeWatcherStub();
9039
+ let passes = 0;
9040
+ const runPass = vi.fn().mockImplementation(async () => {
9041
+ passes += 1;
9042
+ return passes === 1 ? 0 : 1;
9043
+ });
9044
+ const waiters = [];
9045
+ const now = 0;
9046
+ let shutdown = () => { };
9047
+ const sleep = vi.fn(() => new Promise((resolve) => {
9048
+ waiters.push(resolve);
9049
+ }));
9050
+ const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--poll-remote-ms", "1", "--hq-root", "/tmp/hq"], {
9051
+ runPass,
9052
+ clock: new FakeClock(),
9053
+ createWatcher: () => watcher,
9054
+ monotonicNow: () => now,
9055
+ sleep,
9056
+ onShutdownSignal: (handler) => {
9057
+ shutdown = handler;
9058
+ return () => { };
9059
+ },
9060
+ });
9061
+ try {
9062
+ await waitForPollSleep({
9063
+ pendingCount: () => waiters.length,
9064
+ });
9065
+ expect(runPass).toHaveBeenCalledTimes(1);
9066
+ watcher.emit();
9067
+ await flushLoopMicrotasks(30);
9068
+ expect(runPass).toHaveBeenCalledTimes(2);
9069
+ watcher.emit();
9070
+ await flushLoopMicrotasks(30);
9071
+ expect(runPass).toHaveBeenCalledTimes(3);
9072
+ watcher.emit();
9073
+ await flushLoopMicrotasks(30);
9074
+ expect(runPass).toHaveBeenCalledTimes(4);
9075
+ expect(sleep).toHaveBeenLastCalledWith(60_000);
9076
+ watcher.emit();
9077
+ await flushLoopMicrotasks(30);
9078
+ expect(runPass).toHaveBeenCalledTimes(4);
9079
+ expect(sleep).toHaveBeenLastCalledWith(60_000);
9080
+ }
9081
+ finally {
9082
+ shutdown();
9083
+ await expect(loop).resolves.toBe(0);
9084
+ }
9085
+ });
9019
9086
  it("alerts when a partial pass failed before a company's download leg", async () => {
9020
9087
  const watcher = makeWatcherStub();
9021
9088
  const stderrWrite = vi