@indigoai-us/hq-cloud 6.12.2 → 6.12.4

package/src/bin/sync-runner-events.ts

@@ -23,3 +23,33 @@ export function createRunnerEmitter(
 ): (event: RunnerEvent) => void {
   return (event) => routeEvents(event, streams);
 }
+
+/**
+ * Emit a CLASSIFIED `error` event for a top-level uncaught runner rejection,
+ * BEFORE the process exits non-zero.
+ *
+ * The menubar (hq-sync) treats a non-zero runner exit with NO error event seen
+ * at all as an "unexplained crash" and ALWAYS alerts on it. That is correct for
+ * a real panic/OOM, but the historical top-level catch wrote only a plain
+ * stderr line (not a protocol event) for EVERY uncaught rejection — so even a
+ * benign cause that bubbled to the top (transient network blip, a
+ * not-yet-provisioned 404, an expected ACL-scope skip) drove a false alert
+ * (HQ-SYNC-WEB-6). Emitting a structured `{type:"error"}` event here routes the
+ * failure through the menubar's `is_alertable_error` classifier, so the benign
+ * cases are suppressed while genuine defects still surface.
+ *
+ * Best-effort: if routing itself throws, fall back to a plain stderr line so the
+ * failure is never fully silent in the logs.
+ */
+export function emitUncaughtRunnerError(
+  err: unknown,
+  streams: RunnerEventStreams,
+): void {
+  const message =
+    err instanceof Error ? (err.stack ?? err.message) : String(err);
+  try {
+    routeEvents({ type: "error", message, path: "(runner)" }, streams);
+  } catch {
+    streams.stderr.write(`hq-sync-runner: uncaught error — ${message}\n`);
+  }
+}

package/src/bin/sync-runner.ts

@@ -115,7 +115,10 @@ import {
   type StartEventSyncOptions,
 } from "../sync/event-sync.js";
 import { migratePersonalVaultJournal } from "../journal.js";
-import { createRunnerEmitter } from "./sync-runner-events.js";
+import {
+  createRunnerEmitter,
+  emitUncaughtRunnerError,
+} from "./sync-runner-events.js";
 import {
   buildFanoutPlan,
   emitFanoutPlan,
@@ -1473,9 +1476,15 @@ if (isDirectInvocation) {
   runRunnerWithLoop(process.argv.slice(2))
     .then((code) => process.exit(code))
     .catch((err) => {
-      process.stderr.write(
-        `hq-sync-runner: uncaught error — ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`,
-      );
+      // A top-level rejection means the runner died before (or instead of)
+      // emitting its protocol stream. Emit a CLASSIFIED `error` event first so
+      // the menubar runs it through `is_alertable_error` and suppresses the
+      // benign cases, instead of treating every uncaught exit as an unexplained
+      // "silent death" crash (HQ-SYNC-WEB-6).
+      emitUncaughtRunnerError(err, {
+        stdout: process.stdout,
+        stderr: process.stderr,
+      });
       process.exit(1);
     });
 }

package/src/cli/reindex.test.ts

@@ -24,6 +24,11 @@ const hoisted = vi.hoisted(() => ({
   // lets a test simulate a mkdir failure at a NON-wrapper site (e.g. the
   // `.claude/skills` parent or a `core/<type>` mirror dir).
   failMkdirContaining: "" as string,
+  // HQ-CLI-4: when set, any symlink whose LINK path contains this substring
+  // throws EPERM — simulates Windows WITHOUT Developer Mode, where symlink
+  // creation is a privileged operation. Off by default so other tests see the
+  // real `symlinkSync`.
+  failSymlinkContaining: "" as string,
 }));
 vi.mock("fs", async (importOriginal) => {
   const actual = await importOriginal<typeof import("fs")>();
@@ -40,7 +45,24 @@ vi.mock("fs", async (importOriginal) => {
     }
     return (actual.mkdirSync as (p: unknown, opts: unknown) => unknown)(p, opts);
   }) as typeof actual.mkdirSync;
-  const patched = { ...actual, mkdirSync };
+  const symlinkSync = ((target: unknown, linkPath: unknown, type?: unknown) => {
+    if (
+      hoisted.failSymlinkContaining !== "" &&
+      typeof linkPath === "string" &&
+      linkPath.includes(hoisted.failSymlinkContaining)
+    ) {
+      throw Object.assign(
+        new Error(`EPERM: operation not permitted, symlink '${target}' -> '${linkPath}'`),
+        { code: "EPERM" },
+      );
+    }
+    return (actual.symlinkSync as (t: unknown, l: unknown, ty?: unknown) => unknown)(
+      target,
+      linkPath,
+      type,
+    );
+  }) as typeof actual.symlinkSync;
+  const patched = { ...actual, mkdirSync, symlinkSync };
   return { ...patched, default: patched };
 });
 
@@ -307,4 +329,49 @@ describe("reindex", () => {
     // The mirror for the failed type is skipped; skill surfacing still happened.
     expect(fs.existsSync(path.join(root, ".claude/skills/core:demo"))).toBe(true);
   });
+
+  // ── HQ-CLI-4: symlink creation is privileged on Windows. Without Developer
+  //    Mode, fs.symlinkSync throws EPERM. The HQ-B0 work guarded every mkdir
+  //    site but left the symlink sites unguarded, so `hq reindex` still aborted
+  //    on Windows. safeSymlink must degrade gracefully like safeMkdir.
+  it("does not abort reindex when a skill-wrapper symlink fails with EPERM (Windows no Developer Mode)", () => {
+    writeSkill("core/skills/demo");
+
+    hoisted.failSymlinkContaining = path.join(".claude", "skills");
+    try {
+      // Every per-file wrapper symlink throws EPERM, but reindex must still
+      // complete cleanly (exit 0) rather than crash + flood Sentry.
+      expect(reindex({ repoRoot: root }).status).toBe(0);
+    } finally {
+      hoisted.failSymlinkContaining = "";
+    }
+
+    // The wrapper DIR was still created; the symlink inside was just skipped.
+    expect(fs.existsSync(path.join(root, ".claude/skills/core:demo"))).toBe(true);
+    expect(
+      fs.existsSync(path.join(root, ".claude/skills/core:demo/SKILL.md")),
+    ).toBe(false);
+  });
+
+  it("does not abort reindex when a personal-mirror symlink fails with EPERM, and other work still runs", () => {
+    writeSkill("core/skills/demo");
+    fs.mkdirSync(path.join(root, "personal/knowledge/my-kb"), { recursive: true });
+    fs.writeFileSync(path.join(root, "personal/knowledge/my-kb/note.md"), "n\n");
+
+    // Fail ONLY the core/<type> personal-mirror symlinks; wrapper symlinks
+    // succeed, proving the skip is scoped and the reindex completes.
+    hoisted.failSymlinkContaining = path.join("core", "knowledge");
+    try {
+      expect(reindex({ repoRoot: root }).status).toBe(0);
+    } finally {
+      hoisted.failSymlinkContaining = "";
+    }
+
+    // The personal mirror was skipped (EPERM)...
+    expect(fs.existsSync(path.join(root, "core/knowledge/my-kb"))).toBe(false);
+    // ...but unaffected work still happened: the skill wrapper + its symlink.
+    expect(
+      fs.lstatSync(path.join(root, ".claude/skills/core:demo/SKILL.md")).isSymbolicLink(),
+    ).toBe(true);
+  });
 });

package/src/cli/reindex.ts

@@ -154,6 +154,31 @@ function safeMkdir(dir: string, label: string): boolean {
   }
 }
 
+// Create a symlink, tolerating platform-specific failures instead of aborting
+// the whole reindex — the symlink sibling of safeMkdir. The classic case
+// (HQ-CLI-4) is Windows WITHOUT Developer Mode (or an elevated shell), where
+// symlink creation is a privileged operation and `fs.symlinkSync` throws EPERM
+// (Win32 ERROR_PRIVILEGE_NOT_HELD). A single symlink we can't create should
+// degrade gracefully — warn with an actionable hint and skip just that link so
+// the rest of the reindex (other wrappers, mirrors, registry regen) still runs.
+// reindex is idempotent and re-run on hooks, so a later run re-creates the link
+// once the privilege is granted. Returns true on success, false on skip.
+function safeSymlink(target: string, linkPath: string, label: string): boolean {
+  try {
+    fs.symlinkSync(target, linkPath);
+    return true;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    warn(
+      `reindex: could not create ${label} '${linkPath}' -> '${target}' ` +
+        `(${code ?? "error"}: ${(err as Error).message}); skipping. On Windows, ` +
+        `creating symlinks needs Developer Mode (Settings -> Privacy & security ` +
+        `-> For developers) or an elevated shell.`,
+    );
+    return false;
+  }
+}
+
 // --- legacy `.claude/commands/<ns>/<skill>.md` symlink matcher --------------
 // Mirrors the bash `case` patterns; `*` matches any chars (including `/`).
 function isLegacyCommandTarget(t: string): boolean {
@@ -325,7 +350,7 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
           continue;
         }
 
-        fs.symlinkSync(relativeTarget, linkPath);
+        safeSymlink(relativeTarget, linkPath, `.claude/skills/${wrapperName}/${entry}`);
       }
 
       // Prune symlinks in the wrapper whose source entry no longer exists.
@@ -424,7 +449,7 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
         continue;
       }
 
-      fs.symlinkSync(relativeTarget, linkPath);
+      safeSymlink(relativeTarget, linkPath, `core/${type}/${entry}`);
     }
   }
 

package/src/cognito-auth.ts

@@ -299,15 +299,63 @@ interface InitiateAuthResponse {
   message?: string;
 }
 
+/** Tunables for the machine-mint retry-on-throttle loop. Defaults are the
+ *  production values; tests pass `baseDelayMs: 0` to retry without waiting. */
+export interface MintRetryOptions {
+  /** Retries AFTER the first attempt (so total attempts = maxRetries + 1). */
+  maxRetries?: number;
+  /** Backoff for the first retry in ms; doubles each subsequent attempt. */
+  baseDelayMs?: number;
+  /** Ceiling for any single backoff sleep. */
+  maxDelayMs?: number;
+}
+
+const DEFAULT_MINT_RETRY: Required<MintRetryOptions> = {
+  maxRetries: 5,
+  baseDelayMs: 250,
+  maxDelayMs: 5000,
+};
+
+/**
+ * Cognito signals a per-user mint throttle as `TooManyRequestsException` — and
+ * (observed in prod on the Infraredi agent box, 2026-06-26) returns it under
+ * HTTP 400, not 429. Match the typed error first and the status second so both
+ * shapes are treated as transient throttles to retry, not hard auth failures.
+ */
+function isCognitoThrottle(status: number, type: string | undefined): boolean {
+  if (typeof type === "string" && type.includes("TooManyRequests")) return true;
+  return status === 429;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Full-jitter exponential backoff: a random delay in
+ *  [0, min(maxDelayMs, baseDelayMs * 2^attempt)]. Jitter de-syncs a concurrent
+ *  burst of retries so they don't re-collide on the same per-user limit. */
+function backoffDelayMs(
+  attempt: number,
+  opts: Required<MintRetryOptions>,
+): number {
+  const ceiling = Math.min(opts.maxDelayMs, opts.baseDelayMs * 2 ** attempt);
+  return Math.floor(Math.random() * (ceiling + 1));
+}
+
 /**
  * Mint a fresh session for the machine identity via USER_PASSWORD_AUTH
  * against the Cognito IDP endpoint (plain unsigned HTTP — no AWS SDK
  * dependency). Caches BOTH tokens with correct field semantics and returns
  * them.
+ *
+ * Retries on a `TooManyRequestsException` throttle with exponential backoff +
+ * full jitter (transient, per-user rate limit); every other non-OK response
+ * (bad creds, disabled flow) is a hard error surfaced immediately.
  */
 export async function mintMachineTokens(
   config: CognitoAuthConfig,
   creds?: MachineCreds,
+  retry?: MintRetryOptions,
 ): Promise<CognitoTokens> {
   const machineCreds = creds ?? loadMachineCreds();
   if (!machineCreds) {
@@ -320,68 +368,131 @@ export async function mintMachineTokens(
   // which the CLI's default (browser) client may not.
   const region = machineCreds.region ?? config.region;
   const clientId = machineCreds.clientId ?? config.clientId;
-  const res = await fetch(
-    `https://cognito-idp.${region}.amazonaws.com/`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/x-amz-json-1.1",
-        "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
-      },
-      body: JSON.stringify({
-        AuthFlow: "USER_PASSWORD_AUTH",
-        ClientId: clientId,
-        AuthParameters: {
-          USERNAME: machineCreds.username,
-          PASSWORD: machineCreds.secret,
+  const retryOpts: Required<MintRetryOptions> = { ...DEFAULT_MINT_RETRY, ...retry };
+
+  let lastError: CognitoAuthError | null = null;
+  for (let attempt = 0; attempt <= retryOpts.maxRetries; attempt++) {
+    const res = await fetch(
+      `https://cognito-idp.${region}.amazonaws.com/`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-amz-json-1.1",
+          "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
         },
-      }),
-    },
-  );
-  const data = (await res.json().catch(() => ({}))) as InitiateAuthResponse;
-  if (!res.ok) {
-    throw new CognitoAuthError(
-      `Machine token mint failed (${res.status}): ${data.__type ?? ""} ${data.message ?? ""}`.trim(),
-    );
-  }
-  const result = data.AuthenticationResult;
-  if (!result?.AccessToken || !result?.IdToken) {
-    throw new CognitoAuthError(
-      `Machine token mint returned no tokens${data.ChallengeName ? ` (challenge: ${data.ChallengeName})` : ""}`,
+        body: JSON.stringify({
+          AuthFlow: "USER_PASSWORD_AUTH",
+          ClientId: clientId,
+          AuthParameters: {
+            USERNAME: machineCreds.username,
+            PASSWORD: machineCreds.secret,
+          },
+        }),
+      },
     );
+    const data = (await res.json().catch(() => ({}))) as InitiateAuthResponse;
+    if (!res.ok) {
+      const err = new CognitoAuthError(
+        `Machine token mint failed (${res.status}): ${data.__type ?? ""} ${data.message ?? ""}`.trim(),
+      );
+      // Back off and retry a transient per-user throttle rather than failing
+      // the whole sync run; surface every other auth error immediately.
+      if (
+        isCognitoThrottle(res.status, data.__type) &&
+        attempt < retryOpts.maxRetries
+      ) {
+        lastError = err;
+        await sleep(backoffDelayMs(attempt, retryOpts));
+        continue;
+      }
+      throw err;
+    }
+    const result = data.AuthenticationResult;
+    if (!result?.AccessToken || !result?.IdToken) {
+      throw new CognitoAuthError(
+        `Machine token mint returned no tokens${data.ChallengeName ? ` (challenge: ${data.ChallengeName})` : ""}`,
+      );
+    }
+    const tokens: CognitoTokens = {
+      accessToken: result.AccessToken,
+      idToken: result.IdToken,
+      // Machine creds never expire — expiry is handled by re-minting, so the
+      // refresh token (when Cognito returns one at all) is never exercised.
+      refreshToken: result.RefreshToken ?? "",
+      expiresAt: Date.now() + (result.ExpiresIn ?? 3600) * 1000,
+      tokenType: "Bearer",
+    };
+    saveCachedTokens(tokens);
+    return tokens;
   }
-  const tokens: CognitoTokens = {
-    accessToken: result.AccessToken,
-    idToken: result.IdToken,
-    // Machine creds never expire — expiry is handled by re-minting, so the
-    // refresh token (when Cognito returns one at all) is never exercised.
-    refreshToken: result.RefreshToken ?? "",
-    expiresAt: Date.now() + (result.ExpiresIn ?? 3600) * 1000,
-    tokenType: "Bearer",
-  };
-  saveCachedTokens(tokens);
-  return tokens;
+  // Unreachable in practice — the final throttled attempt throws inside the
+  // loop — but keeps the function total for the type checker.
+  throw lastError ?? new CognitoAuthError("Machine token mint failed: throttled");
 }
 
+// In-process machine-token memo (sync-runner mint-storm fix).
+//
+// `getValidAccessToken` runs on EVERY vault request, and a busy sync fans out
+// many requests concurrently (presign primes, STS vend, the per-file tombstone
+// HEAD verifies). The on-disk cache alone does NOT bound mints: at run start
+// the disk token is often within its expiry buffer, so N concurrent callers
+// each independently observe "expiring" and each fire USER_PASSWORD_AUTH —
+// bursting Cognito's per-user limit (TooManyRequestsException) on large vaults
+// (observed: 56 throttled mints over 314 files on the Infraredi box, 2026-06-26).
+//
+// This module-level memo makes mints O(1) per process:
+//   - hot path: a live in-memory token serves every later request with no disk
+//     read and no network — a 300-file sync does ONE mint, not one-per-request;
+//   - single-flight: a concurrent burst of cache-misses (the run-start fan-out)
+//     collapses onto a single in-flight mint instead of one mint per caller.
+// One machine identity per process is assumed (true for agent boxes), so a
+// single slot suffices.
+let inProcessMachineTokens: CognitoTokens | null = null;
+let inFlightMachineMint: Promise<CognitoTokens> | null = null;
+
 /**
  * Return a valid (non-expiring) machine session, re-minting on demand.
- * Cache-hit path never touches the network.
+ * Cache-hit path never touches the network — and after the first mint, the
+ * in-process memo serves every subsequent request for the life of the token.
  */
 export async function getValidMachineTokens(
   config: CognitoAuthConfig,
 ): Promise<CognitoTokens> {
-  const machineCreds = loadMachineCreds();
-  const cached = loadCachedTokens();
-  if (machineCreds && cached && !isExpiring(cached, 120)) {
-    // Compare against the client we'd actually mint with (creds-file clientId
-    // wins over config), and require the cached ID token to prove this exact
-    // agent identity. Opaque/missing/human-shaped claims are treated as stale.
-    const expectedClientId = machineCreds.clientId ?? config.clientId;
-    if (cachedTokensMatchMachineIdentity(cached, machineCreds, expectedClientId)) {
-      return cached;
+  // Hot path: a live in-memory token serves every request after the first.
+  if (inProcessMachineTokens && !isExpiring(inProcessMachineTokens, 120)) {
+    return inProcessMachineTokens;
+  }
+  // Single-flight: collapse a concurrent burst of misses onto one resolution.
+  // The assignment below completes before the first `await` yields, so sibling
+  // callers in the same tick see the in-flight promise and reuse it.
+  if (inFlightMachineMint) return inFlightMachineMint;
+
+  inFlightMachineMint = (async () => {
+    const machineCreds = loadMachineCreds();
+    const cached = loadCachedTokens();
+    if (machineCreds && cached && !isExpiring(cached, 120)) {
+      // Compare against the client we'd actually mint with (creds-file clientId
+      // wins over config), and require the cached ID token to prove this exact
+      // agent identity. Opaque/missing/human-shaped claims are treated as stale.
+      const expectedClientId = machineCreds.clientId ?? config.clientId;
+      if (
+        cachedTokensMatchMachineIdentity(cached, machineCreds, expectedClientId)
+      ) {
+        return cached;
+      }
     }
+    return mintMachineTokens(config, machineCreds ?? undefined);
+  })();
+
+  try {
+    const tokens = await inFlightMachineMint;
+    inProcessMachineTokens = tokens;
+    return tokens;
+  } finally {
+    // Clear on both success and failure: a rejected mint must not wedge later
+    // calls onto a permanently-failed promise.
+    inFlightMachineMint = null;
   }
-  return mintMachineTokens(config, machineCreds ?? undefined);
 }
 
 // ---------------------------------------------------------------------------

package/src/machine-auth.test.ts

@@ -355,6 +355,143 @@ describe("getValidMachineTokens", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// Regression: constant token mints over N files (sync-runner mint storm)
+// ---------------------------------------------------------------------------
+//
+// `getValidAccessToken` (→ getValidMachineTokens) is invoked on EVERY vault
+// request a sync makes, including the per-file tombstone HEAD verifies. The
+// 2026-06-26 prod incident was 56 throttled USER_PASSWORD_AUTH mints over 314
+// files on one box: with no in-process memo + single-flight, mints scaled with
+// request count and tripped Cognito's per-user limit. These lock the invariant
+// that mints are O(1) per process, independent of N.
+
+describe("regression: constant mints regardless of file count", () => {
+  it.each([10, 300])(
+    "performs exactly ONE token mint across %i sequential file-verify requests",
+    async (n) => {
+      writeCreds();
+      const { fetchMock } = stubMintFetch();
+      const { getValidMachineTokens } = await importModule();
+
+      for (let i = 0; i < n; i++) {
+        await getValidMachineTokens(CONFIG);
+      }
+
+      // One mint for N requests — NOT proportional to N. The first call mints;
+      // every later call is served by the in-process memo (no network).
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+    },
+  );
+
+  it.each([10, 300])(
+    "collapses a concurrent burst of %i requests onto a SINGLE mint (single-flight)",
+    async (n) => {
+      writeCreds();
+      const { fetchMock } = stubMintFetch();
+      const { getValidMachineTokens } = await importModule();
+
+      // Fire all N at once, as the run-start fan-out does. Without single-flight
+      // every concurrent miss would mint and burst the per-user limit.
+      const results = await Promise.all(
+        Array.from({ length: n }, () => getValidMachineTokens(CONFIG)),
+      );
+
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+      // All callers observe the same token.
+      for (const r of results) expect(r).toEqual(results[0]);
+
+      // Follow-up sequential requests stay on the memo — still one mint total.
+      await getValidMachineTokens(CONFIG);
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+    },
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Throttle handling: retry TooManyRequestsException with backoff
+// ---------------------------------------------------------------------------
+
+describe("mintMachineTokens throttle retry", () => {
+  /** Fetch mock that throttles (HTTP 400 + TooManyRequestsException, the shape
+   *  Cognito actually returns) for the first `throttleCount` calls, then 200s. */
+  function stubThrottleThenSuccess(throttleCount: number) {
+    let call = 0;
+    const fetchMock = vi.fn(async () => {
+      call++;
+      if (call <= throttleCount) {
+        return new Response(
+          JSON.stringify({
+            __type: "TooManyRequestsException",
+            message: "Too many requests",
+          }),
+          { status: 400 },
+        );
+      }
+      return new Response(
+        JSON.stringify({
+          AuthenticationResult: {
+            AccessToken: fakeJwt({ token_use: "access", client_id: CONFIG.clientId }),
+            IdToken: fakeJwt({ token_use: "id" }),
+            ExpiresIn: 3600,
+          },
+        }),
+        { status: 200 },
+      );
+    });
+    vi.stubGlobal("fetch", fetchMock);
+    return fetchMock;
+  }
+
+  it("retries a throttled mint with backoff and eventually succeeds", async () => {
+    writeCreds();
+    const fetchMock = stubThrottleThenSuccess(2);
+    const { mintMachineTokens } = await importModule();
+
+    // baseDelayMs: 0 keeps the test instant — jitter * 0 == 0.
+    const tokens = await mintMachineTokens(CONFIG, undefined, {
+      baseDelayMs: 0,
+      maxRetries: 5,
+    });
+
+    expect(fetchMock).toHaveBeenCalledTimes(3); // 2 throttles + 1 success
+    const idClaims = JSON.parse(
+      Buffer.from(tokens.idToken.split(".")[1], "base64url").toString(),
+    );
+    expect(idClaims.token_use).toBe("id");
+  });
+
+  it("gives up after maxRetries and throws when throttling never clears", async () => {
+    writeCreds();
+    const fetchMock = stubThrottleThenSuccess(Number.POSITIVE_INFINITY);
+    const { mintMachineTokens, CognitoAuthError } = await importModule();
+
+    await expect(
+      mintMachineTokens(CONFIG, undefined, { baseDelayMs: 0, maxRetries: 2 }),
+    ).rejects.toBeInstanceOf(CognitoAuthError);
+
+    // maxRetries: 2 → 3 total attempts (initial + 2 retries).
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+
+  it("does NOT retry a non-throttle auth failure (bad creds fail fast)", async () => {
+    writeCreds();
+    const fetchMock = vi.fn(async () =>
+      new Response(
+        JSON.stringify({ __type: "NotAuthorizedException", message: "nope" }),
+        { status: 400 },
+      ),
+    );
+    vi.stubGlobal("fetch", fetchMock);
+    const { mintMachineTokens, CognitoAuthError } = await importModule();
+
+    await expect(
+      mintMachineTokens(CONFIG, undefined, { baseDelayMs: 0, maxRetries: 5 }),
+    ).rejects.toBeInstanceOf(CognitoAuthError);
+    expect(fetchMock).toHaveBeenCalledTimes(1); // no retries
+  });
+});
+
 // ---------------------------------------------------------------------------
 // getValidAccessToken — machine-mode short circuit
 // ---------------------------------------------------------------------------