@indigoai-us/hq-cloud 6.14.12 → 6.14.14

package/src/cognito-auth.ts

@@ -68,18 +68,89 @@ export class CognitoAuthError extends Error {
   }
 }
 
+/**
+ * A failed refresh attempt with enough classification for callers to decide
+ * whether the cached session is still worth preserving. Network/5xx/429
+ * failures remain recoverable; an invalid/expired refresh token means the
+ * stored session can no longer make progress and should be cleared.
+ */
+export class CognitoRefreshError extends CognitoAuthError {
+  public readonly requiresReauth: boolean;
+  public readonly statusCode?: number;
+  public readonly sessionFingerprint?: string;
+
+  constructor(
+    message: string,
+    requiresReauth: boolean,
+    statusCode?: number,
+    sessionFingerprint?: string,
+  ) {
+    super(message);
+    this.name = "CognitoRefreshError";
+    this.requiresReauth = requiresReauth;
+    this.statusCode = statusCode;
+    this.sessionFingerprint = sessionFingerprint;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Token cache (~/.hq/cognito-tokens.json)
 // ---------------------------------------------------------------------------
 
 const HQ_DIR = path.join(os.homedir(), ".hq");
 const TOKEN_FILE = path.join(HQ_DIR, "cognito-tokens.json");
+const TOKEN_INVALIDATION_PREFIX = `${TOKEN_FILE}.invalid.`;
+
+/** Stable, non-secret identity for one access-token generation. */
+export function accessTokenFingerprint(accessToken: string): string {
+  return crypto.createHash("sha256").update(accessToken).digest("hex");
+}
+
+function invalidationPathForFingerprint(fingerprint: string): string | null {
+  return /^[a-f0-9]{64}$/i.test(fingerprint)
+    ? `${TOKEN_INVALIDATION_PREFIX}${fingerprint.toLowerCase()}`
+    : null;
+}
+
+/**
+ * Mark only the rejected token generation unusable. Automatic auth failures
+ * must not unlink the shared cache: another process may have completed a
+ * refresh or login after the failing request started.
+ */
+export function invalidateCachedTokensByFingerprint(
+  fingerprint: string | undefined,
+): void {
+  if (!fingerprint) return;
+  const invalidationPath = invalidationPathForFingerprint(fingerprint);
+  if (!invalidationPath) return;
+  if (!fs.existsSync(HQ_DIR)) {
+    fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
+  }
+  try {
+    fs.writeFileSync(invalidationPath, "", { mode: 0o600, flag: "wx" });
+  } catch (err) {
+    const code =
+      err && typeof err === "object" && "code" in err
+        ? (err as { code?: string }).code
+        : undefined;
+    if (code !== "EEXIST") throw err;
+  }
+}
+
+function tokenIsInvalidated(accessToken: string): boolean {
+  const marker = invalidationPathForFingerprint(
+    accessTokenFingerprint(accessToken),
+  );
+  return marker !== null && fs.existsSync(marker);
+}
 
 export function loadCachedTokens(): CognitoTokens | null {
   if (!fs.existsSync(TOKEN_FILE)) return null;
   try {
     const raw = fs.readFileSync(TOKEN_FILE, "utf-8");
-    return JSON.parse(raw) as CognitoTokens;
+    const tokens = JSON.parse(raw) as CognitoTokens;
+    if (!tokens.accessToken || tokenIsInvalidated(tokens.accessToken)) return null;
+    return tokens;
   } catch {
     return null;
   }
@@ -89,6 +160,12 @@ export function saveCachedTokens(tokens: CognitoTokens): void {
   if (!fs.existsSync(HQ_DIR)) {
     fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
   }
+  const invalidationPath = invalidationPathForFingerprint(
+    accessTokenFingerprint(tokens.accessToken),
+  );
+  if (invalidationPath && fs.existsSync(invalidationPath)) {
+    fs.unlinkSync(invalidationPath);
+  }
   const tmpPath = path.join(HQ_DIR, `.cognito-tokens.json.tmp.${process.pid}`);
   fs.writeFileSync(tmpPath, JSON.stringify(tokens, null, 2), { mode: 0o600 });
   fs.renameSync(tmpPath, TOKEN_FILE);
@@ -96,6 +173,128 @@ export function saveCachedTokens(tokens: CognitoTokens): void {
 
 export function clearCachedTokens(): void {
   if (fs.existsSync(TOKEN_FILE)) fs.unlinkSync(TOKEN_FILE);
+  if (!fs.existsSync(HQ_DIR)) return;
+  const markerPrefix = `${path.basename(TOKEN_FILE)}.invalid.`;
+  for (const entry of fs.readdirSync(HQ_DIR)) {
+    if (entry.startsWith(markerPrefix)) {
+      fs.unlinkSync(path.join(HQ_DIR, entry));
+    }
+  }
+}
+
+/** Advisory lockfile guarding the read→refresh→save cycle (see withRefreshLock). */
+const REFRESH_LOCK_FILE = `${TOKEN_FILE}.lock`;
+
+/** Thrown when the refresh lock can't be acquired within the deadline. */
+export class RefreshLockTimeoutError extends Error {
+  constructor() {
+    super("timed out acquiring the token refresh lock");
+    this.name = "RefreshLockTimeoutError";
+  }
+}
+
+/** The pid recorded in the lockfile, or null if absent/empty/malformed. */
+function readLockOwnerPid(): number | null {
+  try {
+    const pid = Number.parseInt(fs.readFileSync(REFRESH_LOCK_FILE, "utf-8").trim(), 10);
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Whether `pid` is a live process. All contenders for this lock share one host
+ * (the cache is `~/.hq`), so pid liveness is authoritative. `EPERM` means the
+ * process exists but is owned by another user — still alive; any other error
+ * (`ESRCH`, bad pid) means it's gone.
+ */
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return (err as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+/**
+ * Serialize the token refresh/mint cycle across every process sharing this
+ * cache (~/.hq/cognito-tokens.json). Boxes run several hq processes (the
+ * auth-refresh timer, the sync runner, replica-sync) that refresh or mint the
+ * same session concurrently; without coordination they race on the shared
+ * cache. And if the app client ever issues single-use tokens (refresh-token
+ * rotation), two unsynchronized refreshers each send the same token — the first
+ * consumes it, the second gets `invalid_grant`. An advisory lockfile (atomic
+ * hard-link publication) makes them take turns; the caller MUST re-read the cache
+ * inside `fn` so a waiter reuses the winner's fresh token, never a consumed one.
+ * This is also the lock the per-box machine-identity mint path reuses (ADR-0009).
+ *
+ * Correctness over availability at the edges:
+ *  - A held lock is reclaimed ONLY when its recorded holder pid is dead (a real
+ *    crash) or the lockfile is malformed — NEVER because a live holder is merely
+ *    slow (a slow Cognito response must not let a second refresher race in).
+ *  - Release unlinks ONLY our own lock (pid re-checked), so we can't delete a
+ *    successor's lock.
+ *  - If we can't acquire within `timeoutMs` (a live holder never released — a
+ *    hung network), we THROW rather than refresh unlocked; the caller prefers a
+ *    still-valid cached token and otherwise retries next cycle. Proceeding
+ *    unlocked would reintroduce the exact rotation race this guards.
+ */
+async function withRefreshLock<T>(fn: () => Promise<T>): Promise<T> {
+  const timeoutMs = 20_000;
+  const pollMs = 50;
+  if (!fs.existsSync(HQ_DIR)) {
+    fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
+  }
+  // Publish a fully-written inode with an atomic hard link. Creating the lock
+  // itself and then writing the pid leaves an observable empty-file window in
+  // which a contender can misclassify and unlink a live holder's lock.
+  const candidate = `${REFRESH_LOCK_FILE}.candidate.${process.pid}.${crypto.randomBytes(8).toString("hex")}`;
+  fs.writeFileSync(candidate, String(process.pid), {
+    flag: "wx",
+    mode: 0o600,
+  });
+  const deadline = Date.now() + timeoutMs;
+  let held = false;
+  try {
+    for (;;) {
+      try {
+        fs.linkSync(candidate, REFRESH_LOCK_FILE);
+        held = true;
+        break;
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+        const owner = readLockOwnerPid();
+        if (owner === null || (owner !== process.pid && !processAlive(owner))) {
+          try {
+            fs.unlinkSync(REFRESH_LOCK_FILE); // crashed/malformed holder — reclaim
+          } catch {
+            /* raced with another reclaimer */
+          }
+          continue;
+        }
+        if (Date.now() >= deadline) throw new RefreshLockTimeoutError();
+        await sleep(pollMs);
+      }
+    }
+    return await fn();
+  } finally {
+    // Release ONLY our own lock — never clobber a successor's.
+    if (held && readLockOwnerPid() === process.pid) {
+      try {
+        fs.unlinkSync(REFRESH_LOCK_FILE);
+      } catch {
+        /* already released */
+      }
+    }
+    // Remove the private proposal inode name on success or acquisition failure.
+    try {
+      fs.unlinkSync(candidate);
+    } catch {
+      /* already removed */
+    }
+  }
 }
 
 /**
@@ -155,7 +354,7 @@ function decodeJwtClaims(token: string): Record<string, unknown> | null {
   }
 }
 
-function cachedTokensMatchMachineIdentity(
+function machineTokenPairMatchesIdentity(
   tokens: CognitoTokens,
   creds: MachineCreds,
   expectedClientId: string,
@@ -181,8 +380,25 @@ function cachedTokensMatchMachineIdentity(
   const idUsername =
     stringClaim(idClaims, "cognito:username") ?? stringClaim(idClaims, "username");
   const idSub = stringClaim(idClaims, "sub");
-  const entityType = idClaims["custom:entityType"];
-  const entityUid = idClaims["custom:entityUid"];
+  const entityType = stringClaim(idClaims, "custom:entityType");
+  const entityUid = stringClaim(idClaims, "custom:entityUid");
+  const delegatedSub = stringClaim(idClaims, "custom:delegatedSub");
+  const delegatedEmail = stringClaim(idClaims, "custom:delegatedEmail");
+  const hasValidMachineClaimShape =
+    (entityType === "agent" &&
+      entityUid !== null &&
+      entityUid.startsWith("agt_")) ||
+    (entityType === "outpost" &&
+      entityUid !== null &&
+      entityUid.startsWith("out_") &&
+      delegatedSub !== null &&
+      delegatedEmail !== null);
+  // Legacy agent credentials did not persist entityType. If a newer creds
+  // file adds only its agt_* uid, infer the only valid type for that uid.
+  const expectedEntityType = creds.entityType ?? "agent";
+  const entityMatchesCreds =
+    entityType === expectedEntityType &&
+    (creds.entityUid === undefined || entityUid === creds.entityUid);
   const idTokenMatchesCreds =
     idUsername === creds.username || idSub === creds.username;
   const subjectBindings: boolean[] = [];
@@ -200,33 +416,32 @@ function cachedTokensMatchMachineIdentity(
     accessClientId === expectedClientId &&
     idClaims.token_use === "id" &&
     idAudience === expectedClientId &&
-    entityType === "agent" &&
-    typeof entityUid === "string" &&
-    entityUid.startsWith("agt_") &&
+    hasValidMachineClaimShape &&
+    entityMatchesCreds &&
     idTokenMatchesCreds &&
     tokensShareSubject
   );
 }
 
 // ---------------------------------------------------------------------------
-// Machine identity (company agents)
+// Machine identity (company agents and Outpost boxes)
 // ---------------------------------------------------------------------------
 //
-// HQ company agents run headless on their own boxes with long-lived Cognito
+// HQ agents and Outposts run headless on their own boxes with long-lived Cognito
 // MACHINE credentials ({username: "machine-agt_<ulid>", secret}) provisioned
 // by hq-pro's agent bootstrap and stored at ~/.hq-agent/machine-creds.json.
 // There is no browser, no Hosted UI, and no refresh-token dance: the creds
 // never expire, so the CLI simply re-mints a session via USER_PASSWORD_AUTH
 // whenever the cached tokens are missing or expiring.
 //
-// Token semantics matter here. The agent's identity claims
-// (custom:entityType=agent, custom:entityUid=agt_*) ride the ID token only;
-// APIs that verify token_use=access (e.g. hq-deploy) need the real access
-// token. Both are cached with correct field semantics — callers pick the
-// token type each API actually validates.
+// Token semantics matter here. Machine identity claims ride the ID token only:
+// agents carry custom:entityType=agent/custom:entityUid=agt_*, while Outposts
+// carry outpost/out_* plus delegatedSub and delegatedEmail. APIs that verify
+// token_use=access (e.g. hq-deploy) need the real access token. Both are cached
+// with correct field semantics — callers pick the token type each API validates.
 
 export interface MachineCreds {
-  /** Cognito username (live boxes: "agt-<ulid>@agents.getindigo.ai"). */
+  /** Cognito username for the machine user. */
   username: string;
   /** Long-lived machine secret (USER_PASSWORD_AUTH password). */
   secret: string;
@@ -235,6 +450,10 @@ export interface MachineCreds {
   clientId?: string;
   /** Cognito region for the mint endpoint; falls back to config. */
   region?: string;
+  /** Expected machine kind carried in the ID token's custom:entityType claim. */
+  entityType?: "agent" | "outpost";
+  /** Exact expected machine uid carried in custom:entityUid. */
+  entityUid?: string;
 }
 
 /** Resolve the machine-creds file path (HQ_MACHINE_CREDS_FILE overrides). */
@@ -258,22 +477,45 @@ export function loadMachineCreds(): MachineCreds | null {
       secret?: unknown;
       clientId?: unknown;
       region?: unknown;
+      entityType?: unknown;
+      entityUid?: unknown;
     };
     // The creds FILE is the machine-identity signal — no username-shape
     // gate. (6.7.0 required a "machine-" prefix, but live boxes are
     // provisioned with "agt-<ulid>@agents.getindigo.ai", so detection
     // never engaged on the first real agent box, 2026-06-12.)
+    const validEntityIdentity =
+      (raw.entityType === undefined && raw.entityUid === undefined) ||
+      (raw.entityType === undefined &&
+        typeof raw.entityUid === "string" &&
+        raw.entityUid.startsWith("agt_") &&
+        raw.entityUid.length > "agt_".length) ||
+      (raw.entityType === "agent" &&
+        typeof raw.entityUid === "string" &&
+        raw.entityUid.startsWith("agt_") &&
+        raw.entityUid.length > "agt_".length) ||
+      (raw.entityType === "outpost" &&
+        typeof raw.entityUid === "string" &&
+        raw.entityUid.startsWith("out_") &&
+        raw.entityUid.length > "out_".length);
     if (
       typeof raw.username === "string" &&
       raw.username.length > 0 &&
       typeof raw.secret === "string" &&
-      raw.secret.length > 0
+      raw.secret.length > 0 &&
+      validEntityIdentity
     ) {
       return {
         username: raw.username,
         secret: raw.secret,
         clientId: typeof raw.clientId === "string" ? raw.clientId : undefined,
         region: typeof raw.region === "string" ? raw.region : undefined,
+        ...(raw.entityType === "agent" || raw.entityType === "outpost"
+          ? { entityType: raw.entityType }
+          : {}),
+        ...(typeof raw.entityUid === "string"
+          ? { entityUid: raw.entityUid }
+          : {}),
       };
     }
     return null;
@@ -352,7 +594,7 @@ function backoffDelayMs(
  * full jitter (transient, per-user rate limit); every other non-OK response
  * (bad creds, disabled flow) is a hard error surfaced immediately.
  */
-export async function mintMachineTokens(
+async function mintMachineTokensUnlocked(
   config: CognitoAuthConfig,
   creds?: MachineCreds,
   retry?: MintRetryOptions,
@@ -417,11 +659,16 @@ export async function mintMachineTokens(
       accessToken: result.AccessToken,
       idToken: result.IdToken,
       // Machine creds never expire — expiry is handled by re-minting, so the
-      // refresh token (when Cognito returns one at all) is never exercised.
-      refreshToken: result.RefreshToken ?? "",
+      // refresh token (when Cognito returns one at all) is never retained.
+      refreshToken: "",
       expiresAt: Date.now() + (result.ExpiresIn ?? 3600) * 1000,
       tokenType: "Bearer",
     };
+    if (!machineTokenPairMatchesIdentity(tokens, machineCreds, clientId)) {
+      throw new CognitoAuthError(
+        "Machine token mint returned tokens that do not match the requested machine identity",
+      );
+    }
     saveCachedTokens(tokens);
     return tokens;
   }
@@ -430,6 +677,19 @@ export async function mintMachineTokens(
   throw lastError ?? new CognitoAuthError("Machine token mint failed: throttled");
 }
 
+/**
+ * Mint and persist a fresh machine session while holding the process-shared
+ * token lock. The unlocked implementation is private so public callers cannot
+ * race another process's cache write.
+ */
+export async function mintMachineTokens(
+  config: CognitoAuthConfig,
+  creds?: MachineCreds,
+  retry?: MintRetryOptions,
+): Promise<CognitoTokens> {
+  return withRefreshLock(() => mintMachineTokensUnlocked(config, creds, retry));
+}
+
 // In-process machine-token memo (sync-runner mint-storm fix).
 //
 // `getValidAccessToken` runs on EVERY vault request, and a busy sync fans out
@@ -445,10 +705,37 @@ export async function mintMachineTokens(
 //     read and no network — a 300-file sync does ONE mint, not one-per-request;
 //   - single-flight: a concurrent burst of cache-misses (the run-start fan-out)
 //     collapses onto a single in-flight mint instead of one mint per caller.
-// One machine identity per process is assumed (true for agent boxes), so a
-// single slot suffices.
-let inProcessMachineTokens: CognitoTokens | null = null;
-let inFlightMachineMint: Promise<CognitoTokens> | null = null;
+// The memo carries an exact credential/client key, and in-flight work is keyed,
+// so reprovisioning a long-running process cannot reuse the prior identity.
+interface MachineTokenMemo {
+  identityKey: string;
+  tokens: CognitoTokens;
+}
+
+let inProcessMachineTokens: MachineTokenMemo | null = null;
+const inFlightMachineMints = new Map<string, Promise<CognitoTokens>>();
+
+function machineIdentityKey(
+  config: CognitoAuthConfig,
+  creds: MachineCreds,
+): string {
+  // Include every credential value that can change the effective mint. Hashing
+  // keeps the long-lived secret out of diagnostic heap strings while retaining
+  // exact-key behavior when a creds file is reprovisioned in place.
+  return crypto
+    .createHash("sha256")
+    .update(
+      JSON.stringify({
+        username: creds.username,
+        secret: creds.secret,
+        clientId: creds.clientId ?? config.clientId,
+        region: creds.region ?? config.region,
+        entityType: creds.entityType ?? "agent",
+        entityUid: creds.entityUid ?? null,
+      }),
+    )
+    .digest("hex");
+}
 
 /**
  * Return a valid (non-expiring) machine session, re-minting on demand.
@@ -458,40 +745,99 @@ let inFlightMachineMint: Promise<CognitoTokens> | null = null;
 export async function getValidMachineTokens(
   config: CognitoAuthConfig,
 ): Promise<CognitoTokens> {
+  const machineCreds = loadMachineCreds();
+  if (!machineCreds) {
+    throw new CognitoAuthError(
+      `No machine credentials found at ${machineCredsFilePath()}`,
+    );
+  }
+  const expectedClientId = machineCreds.clientId ?? config.clientId;
+  const identityKey = machineIdentityKey(config, machineCreds);
   // Hot path: a live in-memory token serves every request after the first.
-  if (inProcessMachineTokens && !isExpiring(inProcessMachineTokens, 120)) {
-    return inProcessMachineTokens;
+  if (
+    inProcessMachineTokens?.identityKey === identityKey &&
+    !isExpiring(inProcessMachineTokens.tokens, 120) &&
+    machineTokenPairMatchesIdentity(
+      inProcessMachineTokens.tokens,
+      machineCreds,
+      expectedClientId,
+    )
+  ) {
+    return inProcessMachineTokens.tokens;
   }
   // Single-flight: collapse a concurrent burst of misses onto one resolution.
   // The assignment below completes before the first `await` yields, so sibling
   // callers in the same tick see the in-flight promise and reuse it.
-  if (inFlightMachineMint) return inFlightMachineMint;
+  const existingResolution = inFlightMachineMints.get(identityKey);
+  if (existingResolution) return existingResolution;
+
+  const resolution = (async () => {
+    const resolveUnderLock = async (): Promise<CognitoTokens> => {
+      // Re-read only after acquiring the process-shared lock. A different hq
+      // process may have minted and saved this machine's session while this
+      // caller was waiting.
+      const cached = loadCachedTokens();
+      if (cached && !isExpiring(cached, 120)) {
+        // Compare against the client we'd actually mint with (creds-file
+        // clientId wins over config), and require the cached ID token to prove
+        // the exact machine identity. Opaque/missing/human-shaped claims are
+        // treated as stale.
+        if (
+          machineTokenPairMatchesIdentity(
+            cached,
+            machineCreds,
+            expectedClientId,
+          )
+        ) {
+          if (cached.refreshToken === "") return cached;
+          const normalized = { ...cached, refreshToken: "" };
+          saveCachedTokens(normalized);
+          return normalized;
+        }
+      }
+      // mintMachineTokens persists before returning, so the complete shared
+      // cache read -> USER_PASSWORD_AUTH mint -> save sequence is serialized.
+      return mintMachineTokensUnlocked(config, machineCreds);
+    };
 
-  inFlightMachineMint = (async () => {
-    const machineCreds = loadMachineCreds();
-    const cached = loadCachedTokens();
-    if (machineCreds && cached && !isExpiring(cached, 120)) {
-      // Compare against the client we'd actually mint with (creds-file clientId
-      // wins over config), and require the cached ID token to prove this exact
-      // agent identity. Opaque/missing/human-shaped claims are treated as stale.
-      const expectedClientId = machineCreds.clientId ?? config.clientId;
-      if (
-        cachedTokensMatchMachineIdentity(cached, machineCreds, expectedClientId)
-      ) {
-        return cached;
+    try {
+      return await withRefreshLock(resolveUnderLock);
+    } catch (err) {
+      if (!(err instanceof RefreshLockTimeoutError)) throw err;
+      // Never mint unlocked. If the holder managed to persist a matching fresh
+      // session, reuse it; otherwise surface a retryable auth failure.
+      const cached = loadCachedTokens();
+      if (cached && !isExpiring(cached, 120)) {
+        if (
+          machineTokenPairMatchesIdentity(
+            cached,
+            machineCreds,
+            expectedClientId,
+          )
+        ) {
+          // The successful path always leaves a machine cache with no refresh
+          // token. If this is legacy state, only normalize it while holding the
+          // lock; after a timeout we must not race the live holder's write.
+          if (cached.refreshToken === "") return cached;
+        }
       }
+      throw new CognitoAuthError(
+        "Machine token mint is held by another process and no fresh matching session is available; retry shortly.",
+      );
     }
-    return mintMachineTokens(config, machineCreds ?? undefined);
   })();
+  inFlightMachineMints.set(identityKey, resolution);
 
   try {
-    const tokens = await inFlightMachineMint;
-    inProcessMachineTokens = tokens;
+    const tokens = await resolution;
+    inProcessMachineTokens = { identityKey, tokens };
     return tokens;
   } finally {
     // Clear on both success and failure: a rejected mint must not wedge later
     // calls onto a permanently-failed promise.
-    inFlightMachineMint = null;
+    if (inFlightMachineMints.get(identityKey) === resolution) {
+      inFlightMachineMints.delete(identityKey);
+    }
   }
 }
 
@@ -700,42 +1046,146 @@ async function exchangeCodeForTokens(
   };
 }
 
+/** Whether the cached session was minted by `config`'s app client. */
+function cachedIsForClient(
+  cached: CognitoTokens,
+  config: CognitoAuthConfig,
+): boolean {
+  const clientId = decodeAccessTokenClientId(cached.accessToken);
+  return clientId !== null && clientId === config.clientId;
+}
+
 /**
  * Use the refresh token to obtain a fresh access token without user interaction.
- * The refresh token itself is NOT rotated by Cognito on the refresh grant, so
- * we preserve the existing one in the result.
+ *
+ * Concurrency-safe. Several hq processes on a box refresh the same cached
+ * session at once, so the whole read→refresh→save cycle runs under a
+ * cross-process lock, and inside the lock we (a) reuse a token another refresher
+ * just produced if it's still fresh, and (b) otherwise refresh with the FRESHEST
+ * token on disk rather than the caller's snapshot. This keeps the shared cache
+ * consistent, and stays correct if the app client uses single-use (rotated)
+ * tokens — a stale snapshot would otherwise be an already-consumed token
+ * (`invalid_grant`). NOTE: rotation is NOT how boxes stay signed in — a rotated
+ * token inherits the original's remaining lifetime, so it does not extend the
+ * session (AWS). Durable never-log-out is per-box machine identity (ADR-0009).
  */
 export async function refreshTokens(
   config: CognitoAuthConfig,
   currentRefreshToken: string,
+  currentAccessToken?: string,
 ): Promise<CognitoTokens> {
-  const body = new URLSearchParams({
-    grant_type: "refresh_token",
-    client_id: config.clientId,
-    refresh_token: currentRefreshToken,
-  });
+  const doRefresh = async (): Promise<CognitoTokens> => {
+    const cached = loadCachedTokens();
+    // Reuse a token another refresher just produced — but ONLY when it belongs
+    // to THIS app client (a direct caller may target a different pool than the
+    // shared cache holds) and is still fresh (a fresh access token has ~1h, so
+    // >300s of life means "someone already refreshed", not the caller's own
+    // expiring token).
+    if (cached && cachedIsForClient(cached, config) && !isExpiring(cached, 300)) {
+      return cached;
+    }
 
-  const res = await fetch(`${authBaseUrl(config)}/oauth2/token`, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: body.toString(),
-  });
-  if (!res.ok) {
-    const text = await res.text();
-    throw new CognitoAuthError(
-      `Refresh failed (${res.status}): ${text}`,
+    // Prefer the freshest refresh token on disk over the caller's snapshot when
+    // it's for this client — a concurrent rotation may have consumed the
+    // snapshot. Otherwise (different client / no cache) use the supplied token.
+    const refreshToken =
+      cached && cachedIsForClient(cached, config)
+        ? cached.refreshToken
+        : currentRefreshToken;
+    const sessionFingerprint = cached && cachedIsForClient(cached, config)
+      ? accessTokenFingerprint(cached.accessToken)
+      : currentAccessToken
+        ? accessTokenFingerprint(currentAccessToken)
+        : undefined;
+
+    const body = new URLSearchParams({
+      grant_type: "refresh_token",
+      client_id: config.clientId,
+      refresh_token: refreshToken,
+    });
+
+    let lastFailure: CognitoRefreshError | undefined;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      let res: Response;
+      try {
+        res = await fetch(`${authBaseUrl(config)}/oauth2/token`, {
+          method: "POST",
+          headers: { "Content-Type": "application/x-www-form-urlencoded" },
+          body: body.toString(),
+        });
+      } catch (err) {
+        lastFailure = new CognitoRefreshError(
+          `Cognito refresh request failed: ${err instanceof Error ? err.message : String(err)}`,
+          false,
+          undefined,
+          sessionFingerprint,
+        );
+        if (attempt === 0) continue;
+        throw lastFailure;
+      }
+
+      if (!res.ok) {
+        const text = await res.text();
+        const retryable =
+          res.status === 401 ||
+          res.status === 408 ||
+          res.status === 429 ||
+          res.status >= 500;
+        const requiresReauth =
+          res.status === 401 ||
+          (res.status >= 400 &&
+            res.status < 500 &&
+            ![408, 429].includes(res.status));
+        lastFailure = new CognitoRefreshError(
+          requiresReauth
+            ? "Your HQ session needs a quick refresh. Sign in again to keep sync moving."
+            : `Cognito refresh failed (${res.status}): ${text}`,
+          requiresReauth,
+          res.status,
+          sessionFingerprint,
+        );
+        if (retryable && attempt === 0) continue;
+        throw lastFailure;
+      }
+
+      const data = (await res.json()) as CognitoTokenResponse;
+      const tokens: CognitoTokens = {
+        accessToken: data.access_token,
+        idToken: data.id_token,
+        // Persist a rotated refresh token if the endpoint returns one (single-use
+        // clients); otherwise keep the freshest token we sent. This is for cache
+        // correctness, not longevity; rotation does not extend session lifetime.
+        refreshToken: data.refresh_token ?? refreshToken,
+        expiresAt: Date.now() + data.expires_in * 1000,
+        tokenType: "Bearer",
+      };
+      saveCachedTokens(tokens);
+      return tokens;
+    }
+
+    throw (
+      lastFailure ??
+      new CognitoRefreshError("Cognito refresh failed after retry", false)
     );
-  }
-  const data = (await res.json()) as CognitoTokenResponse;
-  const tokens: CognitoTokens = {
-    accessToken: data.access_token,
-    idToken: data.id_token,
-    refreshToken: data.refresh_token ?? currentRefreshToken,
-    expiresAt: Date.now() + data.expires_in * 1000,
-    tokenType: "Bearer",
   };
-  saveCachedTokens(tokens);
-  return tokens;
+
+  try {
+    return await withRefreshLock(doRefresh);
+  } catch (err) {
+    if (!(err instanceof RefreshLockTimeoutError)) throw err;
+    // A live holder never released within the window (pathological — e.g. a
+    // hung network). Prefer whatever it may have persisted for this client; do
+    // NOT refresh unlocked (that reintroduces the rotation race). If nothing
+    // usable is cached, surface the failure so the caller retries next cycle.
+    const cached = loadCachedTokens();
+    if (cached && cachedIsForClient(cached, config) && !isExpiring(cached)) {
+      return cached;
+    }
+    throw new CognitoRefreshError(
+      "Token refresh is held by another process and no fresh cached session is available; retry shortly.",
+      false,
+    );
+  }
 }
 
 /**
@@ -778,7 +1228,9 @@ export async function getValidAccessToken(
   if (cached) {
     const cachedClientId = decodeAccessTokenClientId(cached.accessToken);
     if (cachedClientId !== null && cachedClientId !== config.clientId) {
-      clearCachedTokens();
+      invalidateCachedTokensByFingerprint(
+        accessTokenFingerprint(cached.accessToken),
+      );
       cached = null;
     }
   }
@@ -787,9 +1239,19 @@ export async function getValidAccessToken(
 
   if (cached) {
     try {
-      const refreshed = await refreshTokens(config, cached.refreshToken);
+      const refreshed = await refreshTokens(
+        config,
+        cached.refreshToken,
+        cached.accessToken,
+      );
       return refreshed.accessToken;
-    } catch {
+    } catch (err) {
+      if (err instanceof CognitoRefreshError && err.requiresReauth) {
+        invalidateCachedTokensByFingerprint(
+          err.sessionFingerprint ?? accessTokenFingerprint(cached.accessToken),
+        );
+      }
+      if (!interactive) throw err;
       // fall through to interactive login
     }
   }