@indigoai-us/hq-cloud 6.14.42 → 6.14.43

package/src/lib/net-errors.test.ts

@@ -8,7 +8,7 @@ describe("isTransientNetworkError", () => {
     expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
   });
 
-  it("treats known transport `.code`s as transient", () => {
+  it("treats direct transport `.code`s as transient", () => {
     for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
       const err = Object.assign(new Error("boom"), { code });
       expect(isTransientNetworkError(err)).toBe(true);
@@ -23,6 +23,18 @@ describe("isTransientNetworkError", () => {
     expect(isTransientNetworkError(err)).toBe(true);
   });
 
+  it("walks an AWS SDK UnknownError cause chain to the underlying DNS error", () => {
+    const cause = Object.assign(
+      new Error("getaddrinfo ENOTFOUND hq-vault-prs-test.s3.us-east-1.amazonaws.com"),
+      { code: "ENOTFOUND", syscall: "getaddrinfo" },
+    );
+    const err = Object.assign(new Error("UnknownError"), {
+      name: "UnknownError",
+      cause,
+    });
+    expect(isTransientNetworkError(err)).toBe(true);
+  });
+
   it("walks AggregateError.errors (happy-eyeballs)", () => {
     const agg = Object.assign(new AggregateError([
       Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),

package/src/lib/net-errors.ts

@@ -2,9 +2,10 @@
  * Transient network-failure classification for the sync runner.
  *
  * The auto-sync watcher runs an unattended poll loop. At the top of every pass
- * it calls `GET /membership/me` to resolve which companies to sync. When the
- * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
- * vault API momentarily unreachable) that call fails at the transport layer —
+ * it calls `GET /membership/me` to resolve which companies to sync, then it
+ * runs a per-company fanout. When the machine is briefly offline (wifi drop,
+ * waking from sleep, a DNS blip, the vault API or a vault S3 endpoint
+ * momentarily unreachable), either boundary can fail at the transport layer —
  * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
  * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
  *
@@ -12,9 +13,11 @@
  * back. But the runner used to `return 1` for it, the watch loop propagated
  * that non-zero exit, the process exited, and the menubar supervisor reported
  * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
- * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
- * retry instead of surfacing a false crash, while a one-shot `hq sync` still
- * exits non-zero so a human running it by hand sees the failure.
+ * HQ-SYNC-1W cluster. A company-leg failure was also previously counted as an
+ * exit-2 partial sync, producing the HQ-SYNC-X symptom even though it was just
+ * retryable DNS/transport loss. We classify both boundaries so the watch loop
+ * can stay alive and retry instead of surfacing a false crash, while a one-shot
+ * `hq sync` still exits non-zero so a human running it by hand sees the failure.
  */
 
 /**

package/test/e2e/sync/transient-company-leg.test.ts

@@ -0,0 +1,380 @@
+/**
+ * Artifact-level regression for HQ-SYNC-X.
+ *
+ * This starts the compiled runner as a child process. The local vault stub
+ * completes authentication, membership discovery, entity lookup, and the
+ * normal company-vault list. Only the personal vault's direct-S3 endpoint uses
+ * an unresolvable `.invalid` host. That makes the real sync function reject
+ * from its fanout leg, rather than synthesizing a per-file error or failing at
+ * the initial GET /membership/me boundary (which already returns 75).
+ */
+
+import { createServer, type Server } from "node:http";
+import { once } from "node:events";
+import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+const repoRoot = fileURLToPath(new URL("../../../", import.meta.url));
+const artifactPath = process.env.HQ_CLOUD_E2E_ARTIFACT
+  ? path.resolve(process.env.HQ_CLOUD_E2E_ARTIFACT)
+  : path.join(repoRoot, "dist", "bin", "sync-runner.js");
+
+const companyUid = "cmp_clean_e2e";
+const companySlug = "clean-e2e";
+const personalUid = "prs_transient_e2e";
+const invalidS3Endpoint = "http://hq-vault-transient-company-leg.invalid";
+
+let fixtureRoot: string | undefined;
+let hqRoot: string;
+let testHome: string;
+let server: Server | undefined;
+let vaultUrl: string;
+const requests: string[] = [];
+let personListReads = 0;
+
+function json(response: import("node:http").ServerResponse, body: unknown): void {
+  response.writeHead(200, { "content-type": "application/json" });
+  response.end(JSON.stringify(body));
+}
+
+async function ensureArtifact(): Promise<void> {
+  try {
+    await access(artifactPath);
+    return;
+  } catch (err) {
+    // An explicit artifact path is used only by the base-commit proof. Do not
+    // hide a bad path there by building the candidate artifact instead.
+    if (process.env.HQ_CLOUD_E2E_ARTIFACT) throw err;
+  }
+
+  await new Promise<void>((resolve, reject) => {
+    const npm = process.platform === "win32" ? "npm.cmd" : "npm";
+    const child = spawn(npm, ["run", "build"], {
+      cwd: repoRoot,
+      stdio: "inherit",
+    });
+    child.once("error", reject);
+    child.once("close", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`npm run build exited ${code ?? "without a status"}`));
+    });
+  });
+  await access(artifactPath);
+}
+
+async function runArtifact(): Promise<{
+  code: number | null;
+  stdout: string;
+  stderr: string;
+}> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(
+      process.execPath,
+      [artifactPath, "--companies", "--hq-root", hqRoot, "--json"],
+      {
+        cwd: path.dirname(artifactPath),
+        env: {
+          ...process.env,
+          HOME: testHome,
+          HQ_MACHINE_ID: "e2e-transient-company-leg",
+          HQ_QMD_REINDEX_ON_SYNC: "0",
+          HQ_STATE_DIR: path.join(fixtureRoot, "state"),
+          HQ_VAULT_API_URL: vaultUrl,
+          AWS_ENDPOINT_URL: invalidS3Endpoint,
+        },
+        stdio: ["ignore", "pipe", "pipe"],
+      },
+    );
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    const timeout = setTimeout(() => {
+      child.kill("SIGTERM");
+      reject(new Error("compiled sync runner did not exit within 15 seconds"));
+    }, 15_000);
+    child.once("error", (err) => {
+      clearTimeout(timeout);
+      reject(err);
+    });
+    child.once("close", (code) => {
+      clearTimeout(timeout);
+      resolve({ code, stdout, stderr });
+    });
+  });
+}
+
+async function runWatchArtifact(): Promise<{
+  code: number | null;
+  signal: NodeJS.Signals | null;
+  stdout: string;
+  stderr: string;
+}> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(
+      process.execPath,
+      [
+        artifactPath,
+        "--companies",
+        "--hq-root",
+        hqRoot,
+        "--json",
+        "--watch",
+        "--poll-remote-ms",
+        "25",
+      ],
+      {
+        cwd: path.dirname(artifactPath),
+        env: {
+          ...process.env,
+          HOME: testHome,
+          HQ_MACHINE_ID: "e2e-transient-company-leg",
+          HQ_QMD_REINDEX_ON_SYNC: "0",
+          HQ_STATE_DIR: path.join(fixtureRoot, "state"),
+          HQ_VAULT_API_URL: vaultUrl,
+          AWS_ENDPOINT_URL: invalidS3Endpoint,
+        },
+        stdio: ["ignore", "pipe", "pipe"],
+      },
+    );
+    let stdout = "";
+    let stderr = "";
+    let stopping = false;
+    const stop = () => {
+      if (stopping || child.exitCode !== null) return;
+      stopping = true;
+      child.kill("SIGTERM");
+    };
+    const timeout = setTimeout(() => {
+      stop();
+      reject(new Error("watching compiled sync runner did not complete two polls within 20 seconds"));
+    }, 20_000);
+    child.stdout.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+      const completePasses = (stdout.match(/"type":"all-complete"/g) ?? []).length;
+      if (completePasses >= 2) stop();
+    });
+    child.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    child.once("error", (err) => {
+      clearTimeout(timeout);
+      reject(err);
+    });
+    child.once("close", (code, signal) => {
+      clearTimeout(timeout);
+      resolve({ code, signal, stdout, stderr });
+    });
+  });
+}
+
+beforeEach(async () => {
+  await ensureArtifact();
+  fixtureRoot = await mkdtemp(path.join(tmpdir(), "hqcloud-transient-company-leg-e2e-"));
+  hqRoot = path.join(fixtureRoot, "hq");
+  testHome = path.join(fixtureRoot, "home");
+  await mkdir(path.join(hqRoot, "companies", companySlug), { recursive: true });
+  await mkdir(path.join(testHome, ".hq"), { recursive: true });
+  const fakeJwtPayload = Buffer.from(
+    JSON.stringify({ sub: "e2e-subject", exp: Math.floor(Date.now() / 1000) + 3600 }),
+  ).toString("base64url");
+  await writeFile(
+    path.join(testHome, ".hq", "cognito-tokens.json"),
+    JSON.stringify({
+      accessToken: `header.${fakeJwtPayload}.signature`,
+      idToken: `header.${fakeJwtPayload}.signature`,
+      refreshToken: "e2e-only",
+      expiresAt: Date.now() + 60 * 60 * 1000,
+      tokenType: "Bearer",
+    }),
+  );
+
+  requests.length = 0;
+  personListReads = 0;
+  server = createServer((request, response) => {
+    const url = new URL(request.url ?? "/", "http://127.0.0.1");
+    requests.push(`${request.method} ${url.pathname}`);
+    if (request.method === "GET" && url.pathname === "/membership/me") {
+      json(response, { memberships: [{ companyUid }] });
+      return;
+    }
+    if (request.method === "GET" && url.pathname === `/entity/${companyUid}`) {
+      json(response, {
+        entity: {
+          uid: companyUid,
+          slug: companySlug,
+          type: "company",
+          bucketName: "hq-vault-e2e",
+          status: "active",
+        },
+      });
+      return;
+    }
+    if (request.method === "GET" && url.pathname === `/entity/${personalUid}`) {
+      json(response, {
+        entity: {
+          uid: personalUid,
+          slug: "personal-e2e",
+          type: "person",
+          bucketName: "hq-vault-e2e-personal",
+          status: "active",
+        },
+      });
+      return;
+    }
+    if (request.method === "GET" && url.pathname === "/v1/files/list") {
+      json(response, {
+        objects: [],
+        cursor: null,
+        truncated: false,
+      });
+      return;
+    }
+    if (request.method === "POST" && url.pathname === "/sts/vend-self") {
+      json(response, {
+        credentials: {
+          accessKeyId: "e2e-access-key",
+          secretAccessKey: "e2e-secret-key",
+          sessionToken: "e2e-session-token",
+          expiration: "2026-07-30T01:00:00.000Z",
+        },
+        expiresAt: "2026-07-30T01:00:00.000Z",
+      });
+      return;
+    }
+    if (request.method === "GET" && url.pathname === "/entity/by-type/person") {
+      personListReads += 1;
+      json(response, {
+        entities:
+          personListReads % 2 === 1
+            ? [
+                {
+                  uid: personalUid,
+                  slug: "personal-e2e",
+                  type: "person",
+                  bucketName: "hq-vault-e2e-personal",
+                  status: "active",
+                },
+              ]
+            : [],
+      });
+      return;
+    }
+    if (request.method === "GET" && url.pathname === "/membership/pending-by-email") {
+      json(response, { invites: [] });
+      return;
+    }
+    if (request.method === "POST") {
+      json(response, { ok: true, written: 0, skipped: [] });
+      return;
+    }
+    response.writeHead(404, { "content-type": "application/json" });
+    response.end(JSON.stringify({ error: `unexpected ${request.method} ${url.pathname}` }));
+  });
+  server.listen(0, "127.0.0.1");
+  await once(server, "listening");
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("e2e vault stub did not bind a TCP port");
+  }
+  vaultUrl = `http://127.0.0.1:${address.port}`;
+});
+
+afterEach(async () => {
+  if (server?.listening) {
+    server.close();
+    await once(server, "close");
+  }
+  if (fixtureRoot) {
+    await rm(fixtureRoot, { recursive: true, force: true });
+  }
+});
+
+describe("compiled sync runner transient company leg (HQ-SYNC-X)", () => {
+  it(
+    "returns 75 and emits transient-network when only the personal vault S3 leg cannot resolve",
+    async () => {
+      const result = await runArtifact();
+
+      expect(
+        result.code,
+        `stdout=${result.stdout}\nstderr=${result.stderr}`,
+      ).toBe(75);
+      expect(requests).toEqual(
+        expect.arrayContaining([
+          "GET /membership/me",
+          `GET /entity/${companyUid}`,
+          "GET /v1/files/list",
+          "POST /sts/vend-self",
+        ]),
+      );
+      const stdoutEvents = result.stdout
+        .trim()
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => JSON.parse(line) as Record<string, unknown>);
+      expect(
+        stdoutEvents.filter((event) => event.type === "transient-network"),
+      ).toEqual([
+        expect.objectContaining({
+          company: "personal",
+          path: "(company)",
+          message: expect.stringContaining("ENOTFOUND"),
+        }),
+      ]);
+      expect(
+        stdoutEvents.some(
+          (event) => event.type === "error" && event.path === "(company)",
+        ),
+      ).toBe(false);
+      expect(stdoutEvents.find((event) => event.type === "all-complete")).toEqual(
+        expect.objectContaining({
+          errors: [],
+          transient: [expect.objectContaining({ company: "personal" })],
+        }),
+      );
+      expect(result.stderr).not.toContain('"path":"(company)"');
+    },
+    30_000,
+  );
+
+  it(
+    "stays alive through two transient company-leg passes until SIGTERM stops it",
+    async () => {
+      const result = await runWatchArtifact();
+      expect(
+        { code: result.code, signal: result.signal },
+        `stdout=${result.stdout}\nstderr=${result.stderr}`,
+      ).toEqual({ code: null, signal: "SIGTERM" });
+
+      const stdoutEvents = result.stdout
+        .trim()
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => JSON.parse(line) as Record<string, unknown>);
+      expect(
+        stdoutEvents.filter((event) => event.type === "all-complete"),
+      ).toHaveLength(2);
+      expect(
+        stdoutEvents.filter(
+          (event) =>
+            event.type === "transient-network" && event.company === "personal",
+        ),
+      ).toHaveLength(2);
+      expect(result.stderr).toContain(
+        "watch pass skipped — transient network failure; retrying next poll",
+      );
+      expect(result.stderr).not.toContain('"path":"(company)"');
+    },
+    30_000,
+  );
+});