@indigoai-us/hq-cloud 6.14.11 → 6.14.13

package/src/cli/rescue-core.ts

@@ -65,6 +65,9 @@ const ALWAYS_PRESERVED_NAMES = [
   ".hq-sync-journal.json",
   ".hq",
   ".hq-conflicts",
+  // Machine-local state. SSM/root invocations can leave credentials here
+  // unreadable to the HQ user; rescue must neither snapshot nor classify it.
+  ".local",
   // Package-manager caches: never release-shipped, never drift-tracked, and
   // huge (the pnpm content-addressed store alone is tens of thousands of
   // files). Walking them just to classify everything user-only dominates the
@@ -531,7 +534,7 @@ function doRescue(
     out("==> Mode:       preserve-list (default)\n");
     const extra = cfg.extraPreserve.length ? `, ${cfg.extraPreserve.join(" ")}` : "";
     out(
-      `==> Preserved:  .git, companies (except companies/_template), personal, workspace, repos, worktrees, .worktrees, .sst-sandbox-home, .tmp-codex-asar, .github, .leak-scan, .hq-sync-journal.json, .hq, .hq-conflicts${extra}\n`,
+      `==> Preserved:  .git, companies (except companies/_template), personal, workspace, repos, worktrees, .worktrees, .sst-sandbox-home, .tmp-codex-asar, .github, .leak-scan, .hq-sync-journal.json, .hq, .hq-conflicts, .local${extra}\n`,
     );
   }
   if (cfg.preserveSubpaths.length !== 0) {
@@ -720,6 +723,7 @@ function doRescue(
       "--exclude=.hq-sync-journal.json",
       "--exclude=.hq",
       "--exclude=.hq-conflicts",
+      "--exclude=/.local",
       "--exclude=/worktrees",
       "--exclude=/.worktrees",
       "--exclude=/.sst-sandbox-home",

package/src/cli/rescue.reindex.test.ts

@@ -1,7 +1,8 @@
 /**
  * Wiring tests: `rescue()` runs `reindex()` after a successful, non-dry-run
- * rescue so the generated skill wrappers / personal mirrors / workers registry
- * are refreshed once core/ has been re-laid-down.
+ * rescue so the generated skill wrappers / workers registry are refreshed once
+ * core/ has been re-laid-down (the personal overlay is read directly, not
+ * mirrored, so there are no personal mirror symlinks to refresh).
  *
  * The rescue algorithm (./rescue-core.js) is mocked so no real clone/overlay
  * runs, and ./reindex.js is mocked to a spy so we assert the call without

package/src/cli/share.test.ts

@@ -41,6 +41,7 @@ import * as readline from "readline";
 import { share, _testing as shareTesting } from "./share.js";
 import { deleteRemoteFile, downloadFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
 import type { EntityContext } from "../types.js";
+import { VaultAuthError } from "../vault-client.js";
 
 const mockConfig: VaultServiceConfig = {
   apiUrl: "https://vault-api.test",
@@ -207,6 +208,25 @@ describe("share", () => {
     expect(uploadFile).toHaveBeenCalledWith(expect.anything(), testFile, "test.md", undefined, expect.anything());
   });
 
+  it("propagates a per-file vault 401 to the runner auth boundary", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const testFile = path.join(companyRoot, "private.md");
+    fs.writeFileSync(testFile, "private");
+    vi.mocked(uploadFile).mockRejectedValueOnce(
+      new VaultAuthError("upload unauthorized"),
+    );
+
+    await expect(
+      share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      }),
+    ).rejects.toBeInstanceOf(VaultAuthError);
+  });
+
   it("respects ignore rules", async () => {
     const companyRoot = path.join(tmpDir, "companies", "acme");
     fs.mkdirSync(path.join(companyRoot, ".git"), { recursive: true });

package/src/cli/share.ts

@@ -67,6 +67,7 @@ import {
 } from "../lib/conflict-file.js";
 import { appendConflictEntry } from "../lib/conflict-index.js";
 import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
+import { VaultAuthError } from "../vault-client.js";
 
 /**
  * Push-side fresh-collision convergence probe.
@@ -110,7 +111,8 @@ async function remoteContentDiffers(
       ? hashSymlinkTarget(fs.readlinkSync(probePath))
       : hashFile(probePath);
     return remoteHash !== localHash;
-  } catch {
+  } catch (err) {
+    if (err instanceof VaultAuthError) throw err;
     return true;
   } finally {
     try {
@@ -1388,6 +1390,7 @@ async function executeUploads(
     try {
       await performUpload(precondition);
     } catch (err) {
+      if (err instanceof VaultAuthError) throw err;
       if (isPreconditionFailed(err)) {
         conflictPaths.push(relativePath);
         const resolution = await resolveConflictSerialized({
@@ -1410,6 +1413,7 @@ async function executeUploads(
           try {
             await performUpload(undefined);
           } catch (retryErr) {
+            if (retryErr instanceof VaultAuthError) throw retryErr;
             run.emit({
               type: "error",
               path: relativePath,
@@ -1501,6 +1505,7 @@ async function writePushConflictMirror(
       });
     }
   } catch (mirrorErr) {
+    if (mirrorErr instanceof VaultAuthError) throw mirrorErr;
     run.emit({
       type: "error",
       path: item.relativePath,
@@ -1562,6 +1567,7 @@ async function executeDeletes(
         deleted: true,
       });
     } catch (err) {
+      if (err instanceof VaultAuthError) throw err;
       run.emit({
         type: "error",
         path: relativePath,

package/src/cli/sync.test.ts

@@ -58,6 +58,7 @@ import { share } from "./share.js";
 import * as s3Module from "../s3.js";
 import { reindex } from "./reindex.js";
 import { readSyncProgress } from "../sync-progress.js";
+import { VaultAuthError } from "../vault-client.js";
 
 const mockConfig: VaultServiceConfig = {
   apiUrl: "https://vault-api.test",
@@ -154,6 +155,19 @@ describe("sync", () => {
     expect(s3Module.classifyVaultKey("docs/readme.md", "company")).toBeNull();
   });
 
+  it("propagates a per-file vault 401 to the runner auth boundary", async () => {
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "docs/private.md", size: 10, lastModified: new Date(), etag: '"private"' },
+    ]);
+    vi.mocked(s3Module.downloadFile).mockRejectedValueOnce(
+      new VaultAuthError("download unauthorized"),
+    );
+
+    await expect(
+      sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir }),
+    ).rejects.toBeInstanceOf(VaultAuthError);
+  });
+
   it("runs reindex against hqRoot after a sync that downloaded files", async () => {
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
     // skipLock: the surrounding sync run already holds the per-root lock.

package/src/cli/sync.ts

@@ -12,7 +12,7 @@ import type {
   VaultServiceConfig,
   SyncJournal,
 } from "../types.js";
-import { VaultClient, type SyncMode } from "../vault-client.js";
+import { VaultAuthError, VaultClient, type SyncMode } from "../vault-client.js";
 import {
   emitCloudTelemetry,
   type TelemetryClaims,
@@ -1311,6 +1311,7 @@ async function executeConflictItem(
       : (downloaded.contentHash ?? hashFile(conflictAbs));
     converged = remoteHash === item.localHash;
   } catch (probeErr) {
+    if (probeErr instanceof VaultAuthError) throw probeErr;
     run.emit({
       type: "error",
       path: remoteFile.key,
@@ -1563,6 +1564,7 @@ async function downloadOne(
     counters.bytesDownloaded += size;
     counters.changedPaths.push(remoteFile.key);
   } catch (err) {
+    if (err instanceof VaultAuthError) throw err;
     if (err instanceof LocalSnapshotChangedError) {
       reportLocalSnapshotConflict(run, counters, remoteFile.key);
     } else if (isAccessDenied(err)) {
@@ -1617,6 +1619,7 @@ async function emitAndReportNewFiles(
             addedBy = head.metadata["created-by"];
           }
         } catch (headErr) {
+          if (headErr instanceof VaultAuthError) throw headErr;
           try {
             console.error(
               `[hq-sync] HeadObject failed for ${nf.path}: ${
@@ -1683,6 +1686,7 @@ async function verifyPlannedJournalTombstones(
           const head = await headRemoteFile(run.ctx, tombstone.key);
           return head === null ? tombstone : null;
         } catch (err) {
+          if (err instanceof VaultAuthError) throw err;
           if (isAccessDenied(err)) return null;
           run.emit({
             type: "error",

package/src/cognito-auth.test.ts

@@ -338,3 +338,230 @@ describe("expiresAt shape round-trip", () => {
     expect(typeof onDisk?.expiresAt).toBe("number");
   });
 });
+
+// ---------------------------------------------------------------------------
+// Concurrency-safe token refresh (cross-process lock)
+//
+// Boxes run several hq processes (`hq auth refresh` on a timer, the sync runner,
+// replica-sync) that refresh the SAME cached session at once. Without
+// coordination they race on the shared cache; and if the app client issues
+// single-use (rotated) tokens, concurrent refreshers invalidate each other —
+// the first consumes the token, the second gets `invalid_grant`. These pin the
+// fix: a cross-process lock + a re-read of the freshest token inside the lock,
+// so refreshes serialize and a consumed token is never reused. (Rotation does
+// NOT reset the 30-day lifetime — see ADR-0009; durable never-log-out is per-box
+// machine identity, which reuses this same lock for its mint sequence.)
+// ---------------------------------------------------------------------------
+describe("refreshTokens — concurrency-safe refresh (cross-process lock)", () => {
+  const config = {
+    region: "us-east-1",
+    userPoolDomain: "vault-indigo-hq-dev",
+    clientId: "test-client",
+  };
+
+  // A minimal unsigned JWT carrying a client_id claim, so the cache's
+  // client-ownership check (decodeAccessTokenClientId) resolves in tests.
+  const jwtFor = (clientId: string, extra: Record<string, unknown> = {}) =>
+    "eyJhbGciOiJub25lIn0." +
+    Buffer.from(JSON.stringify({ client_id: clientId, ...extra })).toString(
+      "base64url",
+    ) +
+    ".sig";
+
+  // Models Cognito refresh-token ROTATION with single-use enforcement: a valid
+  // refresh mints a new token and consumes the old one; reusing a consumed
+  // token returns 400 invalid_grant (exactly what took the outposts offline).
+  function rotatingCognitoFetch(initialToken: string) {
+    let current = initialToken;
+    const consumed = new Set<string>();
+    let counter = 0;
+    return vi.fn(async (_url: string, init?: { body?: string }) => {
+      const sent = new URLSearchParams(init?.body ?? "").get("refresh_token") ?? "";
+      if (sent !== current || consumed.has(sent)) {
+        return new Response(JSON.stringify({ error: "invalid_grant" }), {
+          status: 400,
+          headers: { "Content-Type": "application/json" },
+        });
+      }
+      consumed.add(sent);
+      counter += 1;
+      current = `refresh-${counter}`;
+      return new Response(
+        JSON.stringify({
+          access_token: jwtFor("test-client", { n: counter }),
+          id_token: jwtFor("test-client", { n: counter }),
+          refresh_token: current,
+          expires_in: 3600,
+          token_type: "Bearer",
+        }),
+        { status: 200, headers: { "Content-Type": "application/json" } },
+      );
+    });
+  }
+
+  it("persists the rotated refresh token when Cognito returns one", async () => {
+    vi.stubGlobal("fetch", rotatingCognitoFetch("R0"));
+    const { refreshTokens, loadCachedTokens } = await importModule();
+    const result = await refreshTokens(config, "R0");
+    expect(result.refreshToken).toBe("refresh-1");
+    expect(loadCachedTokens()?.refreshToken).toBe("refresh-1");
+  });
+
+  it("serializes concurrent refreshers: ONE network refresh, neither gets invalid_grant", async () => {
+    const fetchMock = rotatingCognitoFetch("R0");
+    vi.stubGlobal("fetch", fetchMock);
+    const { refreshTokens, saveCachedTokens, loadCachedTokens } =
+      await importModule();
+    // Seed an EXPIRED session (for THIS client) so both callers attempt refresh.
+    saveCachedTokens({
+      ...baseTokens,
+      accessToken: jwtFor("test-client"),
+      refreshToken: "R0",
+      expiresAt: Date.now() - 1000,
+    });
+
+    // Two processes refresh the same expiring session at once. Without the lock
+    // both send "R0"; the second would hit the mock's invalid_grant and throw.
+    const [a, b] = await Promise.all([
+      refreshTokens(config, "R0"),
+      refreshTokens(config, "R0"),
+    ]);
+
+    // Neither threw; the network was hit exactly once (the waiter reused the
+    // winner's fresh token under the lock); both hold that same token.
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(a.refreshToken).toBe("refresh-1");
+    expect(b.refreshToken).toBe("refresh-1");
+    expect(a.accessToken).toBe(b.accessToken);
+    expect(loadCachedTokens()?.refreshToken).toBe("refresh-1");
+  });
+
+  it("reclaims a lock whose holder process has died (no deadlock)", async () => {
+    vi.stubGlobal("fetch", rotatingCognitoFetch("R0"));
+    const { refreshTokens } = await importModule();
+    // A crashed refresher left its lock behind, tagged with a pid that is not
+    // running — reclaimed by pid-liveness, not by an mtime heuristic.
+    const hqDir = path.join(tmpHome, ".hq");
+    fs.mkdirSync(hqDir, { recursive: true });
+    const lockPath = path.join(hqDir, "cognito-tokens.json.lock");
+    fs.writeFileSync(lockPath, "2147483646");
+
+    const result = await refreshTokens(config, "R0");
+    expect(result.refreshToken).toBe("refresh-1");
+    // Our own lock is released after the refresh completes (no leak).
+    expect(fs.existsSync(lockPath)).toBe(false);
+  });
+
+  it("does NOT reuse a cached session minted by a different app client", async () => {
+    const fetchMock = rotatingCognitoFetch("R-B");
+    vi.stubGlobal("fetch", fetchMock);
+    const { refreshTokens, saveCachedTokens } = await importModule();
+    // Cache holds a FRESH token, but for a DIFFERENT client.
+    saveCachedTokens({
+      ...baseTokens,
+      accessToken: jwtFor("other-client"),
+      refreshToken: "R-other",
+      expiresAt: Date.now() + 3600 * 1000,
+    });
+
+    // A caller for "test-client" must NOT be handed the other client's cached
+    // token; it must exchange the token it supplied.
+    const result = await refreshTokens(config, "R-B");
+    expect(fetchMock).toHaveBeenCalledTimes(1); // did not short-circuit on the wrong-client cache
+    expect(result.refreshToken).toBe("refresh-1");
+  });
+});
+
+describe("refresh expiry hardening", () => {
+  it("retries one transient refresh failure and preserves the refresh token", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(
+        new Response(JSON.stringify({ message: "temporarily unavailable" }), {
+          status: 503,
+        }),
+      )
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            access_token: "fresh-access",
+            id_token: "fresh-id",
+            expires_in: 3600,
+            token_type: "Bearer",
+          }),
+          { status: 200, headers: { "Content-Type": "application/json" } },
+        ),
+      );
+    vi.stubGlobal("fetch", fetchMock);
+
+    const { refreshTokens } = await importModule();
+    const result = await refreshTokens(baseConfig, "preserved-refresh");
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(result.accessToken).toBe("fresh-access");
+    expect(result.refreshToken).toBe("preserved-refresh");
+  });
+
+  it("clears an expired refresh session after one 401 retry", async () => {
+    const fetchMock = vi.fn(async () =>
+      new Response(JSON.stringify({ message: "refresh token expired" }), {
+        status: 401,
+      }),
+    );
+    vi.stubGlobal("fetch", fetchMock);
+
+    const { saveCachedTokens, loadCachedTokens, getValidAccessToken } =
+      await importModule();
+    saveCachedTokens({ ...baseTokens, expiresAt: 0 });
+
+    await expect(
+      getValidAccessToken(baseConfig, { interactive: false }),
+    ).rejects.toThrow(/quick refresh/i);
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(loadCachedTokens()).toBeNull();
+  });
+
+  it("keeps the stored session after two transient network failures", async () => {
+    const fetchMock = vi.fn(async () => {
+      throw new Error("ECONNRESET");
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    const { saveCachedTokens, loadCachedTokens, getValidAccessToken } =
+      await importModule();
+    saveCachedTokens({ ...baseTokens, expiresAt: 0 });
+
+    await expect(
+      getValidAccessToken(baseConfig, { interactive: false }),
+    ).rejects.toThrow(/ECONNRESET/);
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(loadCachedTokens()?.refreshToken).toBe("refresh");
+  });
+
+  it("does not invalidate a newer login when an older token fails late", async () => {
+    const {
+      accessTokenFingerprint,
+      invalidateCachedTokensByFingerprint,
+      loadCachedTokens,
+      saveCachedTokens,
+    } = await importModule();
+
+    saveCachedTokens({
+      ...baseTokens,
+      accessToken: "old-access",
+      expiresAt: Date.now() + 60_000,
+    });
+    const oldFingerprint = accessTokenFingerprint("old-access");
+
+    saveCachedTokens({
+      ...baseTokens,
+      accessToken: "new-access",
+      expiresAt: Date.now() + 60_000,
+    });
+    invalidateCachedTokensByFingerprint(oldFingerprint);
+
+    expect(loadCachedTokens()?.accessToken).toBe("new-access");
+  });
+});