@indigoai-us/hq-cloud 6.14.12 → 6.14.14

package/src/bin/sync-runner.test.ts

@@ -23,6 +23,7 @@ import {
   readPinnedPrefixes,
   SESSIONS_SCOPE_PREFIX,
   defaultCollectTelemetry,
+  AUTH_REQUIRED_PASS_EXIT,
 } from "./sync-runner.js";
 import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import type { EntityContext, VaultServiceConfig } from "../types.js";
@@ -33,6 +34,7 @@ import type {
   WatcherSurface,
 } from "./sync-runner.js";
 import { FakeClock } from "../watcher.js";
+import { adaptivePollMs } from "./sync-runner-watch-loop.js";
 import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
 import { HQ_CLOUD_VERSION } from "../version.js";
 import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
@@ -48,6 +50,7 @@ import type {
   TelemetryEventsBatch,
 } from "../vault-client.js";
 import { VaultAuthError, VaultNotFoundError } from "../vault-client.js";
+import { CognitoRefreshError } from "../cognito-auth.js";
 import type { PushEvent } from "../sync/push-event.js";
 
 // ---------------------------------------------------------------------------
@@ -209,6 +212,11 @@ function makeDeps(overrides: Partial<RunnerDeps> = {}): TestDeps {
   // deps without each override re-wrapping.
   return {
     getAccessToken: vi.fn().mockResolvedValue("test-access-token"),
+    clearSession: vi.fn(),
+    // Keep tests isolated from the developer/agent token cache. Without this,
+    // an agent-shaped real idToken enables the post-sync AGENTS materializer
+    // and adds ambient diagnostic events to otherwise-hermetic assertions.
+    getIdTokenClaims: vi.fn().mockReturnValue(null),
     createVaultClient: vi.fn().mockImplementation(() => makeVaultStub()),
     sync: vi.fn().mockResolvedValue(defaultSyncResult()),
     reindex: vi.fn(() => ({ status: 0 })),
@@ -288,11 +296,37 @@ describe("auth", () => {
     expect(code).toBe(0);
     // auth-error is an error-class event → stderr, not stdout
     expect(deps.stderr.events()).toEqual([
-      { type: "auth-error", message: "no cached tokens" },
+      {
+        type: "auth-error",
+        message:
+          "Your HQ session needs a quick refresh. Sign in again to keep sync moving.",
+      },
     ]);
     expect(deps.stdout.events()).toEqual([]);
   });
 
+  it("keeps the watch pass retryable when initial refresh fails transiently", async () => {
+    const deps = makeDeps({
+      getAccessToken: vi
+        .fn()
+        .mockRejectedValue(
+          new CognitoRefreshError("Cognito refresh request failed: ECONNRESET", false),
+        ),
+    });
+
+    const code = await runRunner(["--companies"], deps);
+
+    expect(code).toBe(TRANSIENT_NETWORK_EXIT);
+    expect(deps.stderr.events()).toEqual([
+      {
+        type: "error",
+        path: "(auth)",
+        message: "Cognito refresh request failed: ECONNRESET",
+      },
+    ]);
+    expect(deps.clearSession).not.toHaveBeenCalled();
+  });
+
   it("emits auth-error on stderr when VaultAuthError thrown during discovery", async () => {
     const deps = makeDeps({
       createVaultClient: () => ({
@@ -304,11 +338,38 @@ describe("auth", () => {
     const code = await runRunner(["--companies"], deps);
     expect(code).toBe(0);
     expect(deps.stderr.events()).toEqual([
-      { type: "auth-error", message: "token expired" },
+      {
+        type: "auth-error",
+        message:
+          "Your HQ session needs a quick refresh. Sign in again to keep sync moving.",
+      },
     ]);
+    expect(deps.clearSession).toHaveBeenCalledTimes(1);
     expect(deps.stdout.events()).toEqual([]);
   });
 
+  it("handles a typed 401 while planning the personal target", async () => {
+    const deps = makeDeps({
+      createVaultClient: () =>
+        makeVaultStub({
+          listPersons: () =>
+            Promise.reject(new VaultAuthError("personal lookup unauthorized")),
+        }),
+    });
+
+    const code = await runRunner(["--personal"], deps);
+
+    expect(code).toBe(0);
+    expect(deps.clearSession).toHaveBeenCalledTimes(1);
+    expect(deps.stderr.events()).toEqual([
+      {
+        type: "auth-error",
+        message:
+          "Your HQ session needs a quick refresh. Sign in again to keep sync moving.",
+      },
+    ]);
+  });
+
   it("emits error event on stderr and returns 1 on non-auth discovery failure (after 3 attempts)", async () => {
     let calls = 0;
     const deps = makeDeps({
@@ -459,8 +520,13 @@ describe("memberships retry", () => {
     // Auth failures don't retry — re-vending creds is the caller's job.
     expect(listCalls).toBe(1);
     expect(deps.stderr.events()).toEqual([
-      { type: "auth-error", message: "token expired" },
+      {
+        type: "auth-error",
+        message:
+          "Your HQ session needs a quick refresh. Sign in again to keep sync moving.",
+      },
     ]);
+    expect(deps.clearSession).toHaveBeenCalledTimes(1);
   });
 
   it("single-company mode bypasses listMyMemberships entirely (no retry path triggered)", async () => {
@@ -1512,6 +1578,31 @@ describe("per-company fanout", () => {
     expect(betaComplete?.filesDownloaded).toBe(1);
   });
 
+  it("turns a typed mid-fanout 401 into one terminal auth-error", async () => {
+    const deps = makeDeps({
+      createVaultClient: () =>
+        makeVaultStub({
+          memberships: [{ companyUid: "cmp_auth" }],
+          entityGet: (uid: string) =>
+            Promise.resolve({ uid, slug: "auth-co" } as unknown as EntityInfo),
+        }),
+      sync: vi.fn().mockRejectedValue(new VaultAuthError("expired access token")),
+    });
+
+    const code = await runRunner(["--companies"], deps);
+
+    expect(code).toBe(0);
+    expect(deps.clearSession).toHaveBeenCalledTimes(1);
+    expect(deps.stderr.events()).toEqual([
+      {
+        type: "auth-error",
+        message:
+          "Your HQ session needs a quick refresh. Sign in again to keep sync moving.",
+      },
+    ]);
+    expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
+  });
+
   /**
    * Regression test: AWS SDK v3 raises an opaque `UnknownError` (name and
    * message both literally "UnknownError") when it cannot match a Node-side
@@ -3375,6 +3466,12 @@ describe("routeChangeToTarget", () => {
     expect(routeChangeToTarget("companies")).toBeNull();
     expect(routeChangeToTarget("companies/")).toBeNull();
   });
+
+  it("does not route the company scaffold to cloud sync", () => {
+    expect(
+      routeChangeToTarget("companies/_template/knowledge/README.md"),
+    ).toBeNull();
+  });
 });
 
 describe("buildTargetedPushArgv", () => {
@@ -3481,6 +3578,127 @@ function makeSteppableSleep() {
   };
 }
 
+// ---------------------------------------------------------------------------
+// Adaptive remote-poll backoff — whole-box CPU aware (no --poll-remote-ms)
+// ---------------------------------------------------------------------------
+
+describe("adaptivePollMs (whole-box CPU backoff)", () => {
+  const FLOOR = 60_000;
+  const CEIL = 600_000;
+
+  it("returns the floor when the box is idle (load 0)", () => {
+    expect(adaptivePollMs(0, 8)).toBe(FLOOR);
+  });
+
+  it("returns the ceiling when load equals the core count (fully saturated)", () => {
+    expect(adaptivePollMs(8, 8)).toBe(CEIL);
+  });
+
+  it("clamps to the ceiling when load exceeds the core count", () => {
+    expect(adaptivePollMs(64, 8)).toBe(CEIL);
+  });
+
+  it("scales linearly between floor and ceiling at half saturation", () => {
+    // load 4 on 8 cores → ratio 0.5 → midpoint of [60s, 600s] = 330s.
+    expect(adaptivePollMs(4, 8)).toBe(330_000);
+  });
+
+  it("treats a negative load sample as idle (defensive)", () => {
+    expect(adaptivePollMs(-1, 8)).toBe(FLOOR);
+  });
+
+  it("guards a zero/absent core count without dividing by zero", () => {
+    // cores coerced to 1; load 1 → fully saturated → ceil.
+    expect(adaptivePollMs(1, 0)).toBe(CEIL);
+  });
+
+  it("honors custom floor/ceil bounds", () => {
+    expect(adaptivePollMs(0, 4, 30_000, 300_000)).toBe(30_000);
+    expect(adaptivePollMs(4, 4, 30_000, 300_000)).toBe(300_000);
+  });
+});
+
+describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", () => {
+  // Capture the ms handed to sleep on the FIRST poll, then hang so the loop
+  // parks until shutdown (mirrors the never-resolving sleep the other loop
+  // tests use).
+  function makeCapturingSleep() {
+    const calls: number[] = [];
+    return {
+      calls,
+      sleep: (ms: number) => {
+        calls.push(ms);
+        return new Promise<void>(() => {});
+      },
+    };
+  }
+
+  it("sleeps the floor interval when the box is idle", async () => {
+    const { calls, sleep } = makeCapturingSleep();
+    let triggerShutdown = () => {};
+    const loop = runRunnerWithLoop(
+      ["--companies", "--watch", "--hq-root", "/tmp/hq"],
+      {
+        runPass: vi.fn().mockResolvedValue(0),
+        sleep,
+        sampleLoadAvg: () => 0,
+        onShutdownSignal: (handler) => {
+          triggerShutdown = handler;
+          return () => {};
+        },
+      },
+    );
+    await flushLoopMicrotasks();
+    expect(calls[0]).toBe(60_000);
+    triggerShutdown();
+    await loop;
+  });
+
+  it("backs off to the ceiling interval when the box is fully loaded", async () => {
+    const { calls, sleep } = makeCapturingSleep();
+    let triggerShutdown = () => {};
+    const loop = runRunnerWithLoop(
+      ["--companies", "--watch", "--hq-root", "/tmp/hq"],
+      {
+        runPass: vi.fn().mockResolvedValue(0),
+        sleep,
+        sampleLoadAvg: () => 1_000, // far above any core count → clamps to ceil
+        onShutdownSignal: (handler) => {
+          triggerShutdown = handler;
+          return () => {};
+        },
+      },
+    );
+    await flushLoopMicrotasks();
+    expect(calls[0]).toBe(600_000);
+    triggerShutdown();
+    await loop;
+  });
+
+  it("an explicit --poll-remote-ms pins a fixed interval and ignores load", async () => {
+    const { calls, sleep } = makeCapturingSleep();
+    const sampleLoadAvg = vi.fn(() => 1_000);
+    let triggerShutdown = () => {};
+    const loop = runRunnerWithLoop(
+      ["--companies", "--watch", "--poll-remote-ms", "123456", "--hq-root", "/tmp/hq"],
+      {
+        runPass: vi.fn().mockResolvedValue(0),
+        sleep,
+        sampleLoadAvg,
+        onShutdownSignal: (handler) => {
+          triggerShutdown = handler;
+          return () => {};
+        },
+      },
+    );
+    await flushLoopMicrotasks();
+    expect(calls[0]).toBe(123456);
+    expect(sampleLoadAvg).not.toHaveBeenCalled();
+    triggerShutdown();
+    await loop;
+  });
+});
+
 describe("runRunnerWithLoop — event-push wiring", () => {
   it("starts the watcher alongside the poll loop when --event-push is on", async () => {
     const watcher = makeWatcherStub();
@@ -3810,6 +4028,28 @@ describe("runRunnerWithLoop — event-push wiring", () => {
     expect(watcher.disposed).toBe(true);
   });
 
+  it("an exit-0 auth-required pass stops the unattended loop without reporting a crash", async () => {
+    const watcher = makeWatcherStub();
+    const runPass = vi.fn().mockResolvedValue(AUTH_REQUIRED_PASS_EXIT);
+    const sleep = vi.fn().mockResolvedValue(undefined);
+
+    const code = await runRunnerWithLoop(
+      ["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"],
+      {
+        runPass,
+        sleep,
+        clock: new FakeClock(),
+        createWatcher: () => watcher,
+        onShutdownSignal: () => () => {},
+      },
+    );
+
+    expect(code).toBe(0);
+    expect(runPass).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+    expect(watcher.disposed).toBe(true);
+  });
+
   it("a TRANSIENT_NETWORK_EXIT poll pass does NOT surface as a crash — the loop retries (HQ-SYNC-1W)", async () => {
     // A transient offline blip must keep the watcher alive: the loop logs and
     // polls again instead of returning the code and letting the process exit
@@ -3845,7 +4085,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
 
   it("a PARTIAL_SYNC_EXIT poll pass does NOT surface as a crash — the loop retries", async () => {
     // A partial pass (some per-file transfers threw: oversized object on an
-    // old runner, presign denial, expired token, corrupt vault key) must keep
+    // old runner, presign denial, corrupt vault key) must keep
     // the watcher alive. Exiting on it got the watcher hot-respawned by the
     // menubar supervisor and Sentry-captured on EVERY failing pass — the
     // July 2026 "watcher exited unexpectedly (code=Some(2))" alert flood. The

package/src/bin/sync-runner.ts

@@ -62,7 +62,9 @@ import * as fs from "fs";
 import { randomUUID } from "node:crypto";
 import { fileURLToPath } from "url";
 import {
+  CognitoRefreshError,
   getValidAccessToken,
+  invalidateCachedTokensByFingerprint,
   loadCachedTokens,
   VaultClient,
   VaultAuthError,
@@ -532,6 +534,16 @@ export type { PullScope };
  */
 const MEMBERSHIPS_RETRY_BACKOFFS_MS: readonly number[] = [50, 100];
 
+const REAUTH_MESSAGE =
+  "Your HQ session needs a quick refresh. Sign in again to keep sync moving.";
+
+/**
+ * Internal-only result used between a watch pass and its owning loop. The
+ * outer process still exits 0 (auth-required is a handled UX state), while
+ * the loop stops instead of retrying a known-unusable session forever.
+ */
+export const AUTH_REQUIRED_PASS_EXIT = 18;
+
 /**
  * Call `listMyMemberships()` with up to 3 attempts and a small linear
  * backoff between them. The single network call that drives every cloud
@@ -589,6 +601,8 @@ export interface RunnerDeps {
   stderr?: { write: (chunk: string) => boolean | void };
   /** Resolve a valid access token. Defaults to `getValidAccessToken` non-interactive. */
   getAccessToken?: () => Promise<string>;
+  /** Invalidate only the rejected token generation. Injectable for tests. */
+  clearSession?: (tokenFingerprint?: string) => void;
   /**
    * Read the caller's identity claims (sub/email/name) off the cached Cognito
    * idToken. Defaults to decoding `loadCachedTokens().idToken`. Returns `null`
@@ -621,6 +635,8 @@ export interface RunnerDeps {
   ) => Promise<void>;
   /** Internal: set when runRunner is invoked under the per-root operation lock. */
   operationLockAlreadyHeld?: boolean;
+  /** Internal watch-loop result override; one-shot callers keep exit 0. */
+  authRequiredExitCode?: number;
   /** Share function (push phase). Defaults to `cli/share.share`. */
   share?: (options: ShareOptions) => Promise<ShareResult>;
   /**
@@ -1109,14 +1125,27 @@ export async function runRunner(
   const getAccessToken =
     deps.getAccessToken ??
     (() => getValidAccessToken(DEFAULT_COGNITO, { interactive: false }));
+  const clearSession =
+    deps.clearSession ?? invalidateCachedTokensByFingerprint;
   try {
     await getAccessToken();
   } catch (err) {
+    if (
+      (err instanceof CognitoRefreshError && !err.requiresReauth) ||
+      isTransientNetworkError(err)
+    ) {
+      emit({
+        type: "error",
+        path: "(auth)",
+        message: err instanceof Error ? err.message : String(err),
+      });
+      return TRANSIENT_NETWORK_EXIT;
+    }
     emit({
       type: "auth-error",
-      message: err instanceof Error ? err.message : String(err),
+      message: REAUTH_MESSAGE,
     });
-    return 0;
+    return deps.authRequiredExitCode ?? 0;
   }
 
   // ---- vault client -----------------------------------------------------
@@ -1257,11 +1286,12 @@ export async function runRunner(
     memberships = resolution.memberships;
   } catch (err) {
     if (err instanceof VaultAuthError) {
+      clearSession(err.tokenFingerprint);
       emit({
         type: "auth-error",
-        message: err.message,
+        message: REAUTH_MESSAGE,
       });
-      return 0;
+      return deps.authRequiredExitCode ?? 0;
     }
     // Surface the failure as an error event for observability either way.
     emit({
@@ -1279,15 +1309,31 @@ export async function runRunner(
     return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
   }
 
-  const targetPlan = await buildFanoutPlan({
-    memberships,
-    companies: parsed.companies,
-    personal: parsed.personal,
-    skipPersonal: parsed.skipPersonal,
-    client,
-    claims,
-    resolveSkipPersonal,
-  });
+  let targetPlan: Awaited<ReturnType<typeof buildFanoutPlan>>;
+  try {
+    targetPlan = await buildFanoutPlan({
+      memberships,
+      companies: parsed.companies,
+      personal: parsed.personal,
+      skipPersonal: parsed.skipPersonal,
+      client,
+      claims,
+      resolveSkipPersonal,
+    });
+  } catch (err) {
+    if (err instanceof VaultAuthError) {
+      clearSession(err.tokenFingerprint);
+      emit({ type: "auth-error", message: REAUTH_MESSAGE });
+      return deps.authRequiredExitCode ?? 0;
+    }
+    emit({
+      type: "error",
+      message: err instanceof Error ? err.message : String(err),
+      path: "(discovery)",
+    });
+    emitRunnerTerminalTelemetry({ status: "failed" });
+    return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
+  }
   if (targetPlan.status === "setup-needed") {
     emit({ type: "setup-needed" });
     return 0;
@@ -1306,24 +1352,34 @@ export async function runRunner(
   // ---- fanout -----------------------------------------------------------
   const syncFn = deps.sync ?? defaultSync;
   const shareFn = deps.share ?? defaultShare;
-  const fanout = await executeCompanyFanout({
-    plan,
-    direction: parsed.direction,
-    hqRoot: parsed.hqRoot,
-    onConflict: parsed.onConflict,
-    client,
-    vaultConfig,
-    ...(uploadAuthor ? { uploadAuthor } : {}),
-    ...(deps.operationLockAlreadyHeld ? { operationLockAlreadyHeld: true } : {}),
-    syncFn,
-    shareFn,
-    resolveDeletePolicy,
-    emit,
-    scopePaths: parsed.scopePaths,
-    telemetryClient: client,
-    telemetryClaims: claims,
-    telemetrySessionId,
-  });
+  let fanout: Awaited<ReturnType<typeof executeCompanyFanout>>;
+  try {
+    fanout = await executeCompanyFanout({
+      plan,
+      direction: parsed.direction,
+      hqRoot: parsed.hqRoot,
+      onConflict: parsed.onConflict,
+      client,
+      vaultConfig,
+      ...(uploadAuthor ? { uploadAuthor } : {}),
+      ...(deps.operationLockAlreadyHeld ? { operationLockAlreadyHeld: true } : {}),
+      syncFn,
+      shareFn,
+      resolveDeletePolicy,
+      emit,
+      scopePaths: parsed.scopePaths,
+      telemetryClient: client,
+      telemetryClaims: claims,
+      telemetrySessionId,
+    });
+  } catch (err) {
+    if (err instanceof VaultAuthError) {
+      clearSession(err.tokenFingerprint);
+      emit({ type: "auth-error", message: REAUTH_MESSAGE });
+      return deps.authRequiredExitCode ?? 0;
+    }
+    throw err;
+  }
   const { errors, allConflicts, meaningfulOutcomeCount } = fanout;
   const rollup = rollupAllComplete(plan, fanout.stateByCompany);
 
@@ -1685,6 +1741,13 @@ const isDirectInvocation = (() => {
 export interface RunnerLoopDeps {
   /** Sleep `ms` between passes. Default: host setTimeout. */
   sleep?: (ms: number) => Promise<void>;
+  /**
+   * 1-minute whole-box load average source for the adaptive remote-poll
+   * backoff (used only when `--poll-remote-ms` is omitted). Defaults to
+   * `os.loadavg()[0]`. Tests inject a fixed sample to assert the interval
+   * scales with load without depending on the host's real load.
+   */
+  sampleLoadAvg?: () => number;
   /**
    * Run a single sync pass. Defaults to {@link runRunner}. Injected by tests
    * (and the event-push wiring) so the poll loop and the watcher-triggered
@@ -1787,7 +1850,11 @@ export async function runRunnerWithLoop(
   const parsed = parseArgs(argv);
   const runtime = {
     runPassWithOperationLockAlreadyHeld: (passArgv: string[]) =>
-      runRunner(passArgv, { operationLockAlreadyHeld: true }),
+      runRunner(passArgv, {
+        operationLockAlreadyHeld: true,
+        authRequiredExitCode: AUTH_REQUIRED_PASS_EXIT,
+      }),
+    authRequiredPassExit: AUTH_REQUIRED_PASS_EXIT,
     defaultGetIdTokenClaims,
     defaultGetAccessToken: () =>
       getValidAccessToken(DEFAULT_COGNITO, { interactive: false }),