@indigoai-us/hq-cloud 5.31.0 → 5.32.0

package/src/bin/sync-runner.test.ts

@@ -915,6 +915,64 @@ describe("per-company fanout", () => {
     expect(betaComplete?.filesDownloaded).toBe(1);
   });
 
+  /**
+   * Regression test: AWS SDK v3 raises an opaque `UnknownError` (name and
+   * message both literally "UnknownError") when it cannot match a Node-side
+   * networking failure to a modeled exception. The actual root cause —
+   * ENOTFOUND from a stale DNS cache, ECONNRESET on a flapping link, etc. —
+   * is parked on `err.cause`. Pre-fix the per-company catch extracted only
+   * `err.message`, so an unactionable bare "UnknownError" reached operators
+   * and the underlying syscall/hostname were lost. The catch now runs the
+   * thrown value through `describeError`, which walks the cause chain and
+   * stitches a one-line diagnostic the operator can act on.
+   */
+  it("per-company error event surfaces cause-chain (ENOTFOUND from SDK UnknownError)", async () => {
+    const deps = makeDeps({
+      createVaultClient: () =>
+        makeVaultStub({
+          memberships: [{ companyUid: "cmp_x" }],
+          entityGet: (uid: string) =>
+            Promise.resolve({ uid, slug: "privy" } as unknown as EntityInfo),
+        }),
+      sync: vi
+        .fn<(opts: SyncOptions) => Promise<SyncResult>>()
+        .mockImplementationOnce(async () => {
+          // Shape mirrors @aws-sdk/client-s3 v3 behaviour when the underlying
+          // node-http-handler raises a getaddrinfo failure: opaque wrapper
+          // with the real error on `.cause`.
+          const sdkErr = new Error("UnknownError");
+          sdkErr.name = "UnknownError";
+          const dnsErr = new Error("getaddrinfo ENOTFOUND privy.s3.us-east-1.amazonaws.com");
+          (dnsErr as Error & { code?: string; syscall?: string; hostname?: string }).code =
+            "ENOTFOUND";
+          (dnsErr as Error & { code?: string; syscall?: string; hostname?: string }).syscall =
+            "getaddrinfo";
+          (dnsErr as Error & { code?: string; syscall?: string; hostname?: string }).hostname =
+            "privy.s3.us-east-1.amazonaws.com";
+          (sdkErr as Error & { cause?: unknown }).cause = dnsErr;
+          throw sdkErr;
+        }),
+    });
+
+    const code = await runRunner(["--companies"], deps);
+    expect(code).toBe(2);
+
+    const companyErr = deps.stderr
+      .events()
+      .find(
+        (e): e is Extract<RunnerEvent, { type: "error" }> =>
+          e.type === "error" && e.company === "privy",
+      );
+    expect(companyErr).toBeDefined();
+    expect(companyErr?.path).toBe("(company)");
+    // The diagnostic must carry the original networking signal — without
+    // this the operator only sees "UnknownError" and the run is unactionable.
+    expect(companyErr?.message).toContain("UnknownError");
+    expect(companyErr?.message).toContain("ENOTFOUND");
+    expect(companyErr?.message).toContain("getaddrinfo");
+    expect(companyErr?.message).toContain("privy.s3.us-east-1.amazonaws.com");
+  });
+
   /**
    * Regression test for the rollup-bug from the personal-sync 401 incident.
    *
@@ -1638,6 +1696,312 @@ describe("personal slot fanout", () => {
     }
   });
 
+  it("G: personal slot includes companies/{slug}/ subdirs when cloud:false marker is set AND HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1, excluding _template + team-synced slugs + missing/cloud:true markers", async () => {
+    // Gate the new behavior: without this env var the runner falls back to
+    // the legacy "companies/ never enumerated" personal vault. Test H below
+    // pins the OFF case.
+    vi.stubEnv("HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL", "1");
+    const shareSpy = vi.fn().mockResolvedValue(defaultShareResult());
+    const tmpHqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-runner-test-"));
+    try {
+      // Top-level personal-vault content — anchors the assertion that the
+      // existing top-level walk still runs alongside the new company-subdir
+      // discovery.
+      fs.mkdirSync(path.join(tmpHqRoot, "knowledge"));
+
+      // companies/ subdirs covering every branch of the eligibility filter.
+      const mkCompany = (slug: string, yaml: string | null): void => {
+        const dir = path.join(tmpHqRoot, "companies", slug);
+        fs.mkdirSync(dir, { recursive: true });
+        if (yaml !== null) {
+          fs.writeFileSync(path.join(dir, "company.yaml"), yaml);
+        }
+      };
+      mkCompany("free-co", "cloud: false\nname: Free Co\n"); // INCLUDED
+      mkCompany("acme", "cloud: false\n"); // EXCLUDED — team-synced (membership below)
+      mkCompany("_template", "cloud: false\n"); // EXCLUDED — hard-listed slug
+      mkCompany("cloud-team", "cloud: true\n"); // EXCLUDED — wrong marker
+      mkCompany("zilch", null); // EXCLUDED — no company.yaml at all
+
+      const deps = makeDeps({
+        createVaultClient: () =>
+          makeVaultStub({
+            memberships: [{ companyUid: "cmp_a" }],
+            entityGet: (uid: string) =>
+              Promise.resolve({ uid, slug: "acme" } as unknown as EntityInfo),
+            listPersons: () => Promise.resolve([olderPerson]),
+          }),
+        share: shareSpy,
+      });
+
+      const code = await runRunner(
+        ["--companies", "--direction", "push", "--hq-root", tmpHqRoot],
+        deps,
+      );
+      expect(code).toBe(0);
+
+      const personalCall = (shareSpy.mock.calls as Array<[ShareOptions]>).find(
+        (c) => c[0].company?.startsWith("prs_"),
+      );
+      expect(personalCall).toBeDefined();
+      const personalArgs = personalCall![0];
+
+      // Convert paths to a normalized form (relative to tmpHqRoot) for
+      // easier matching. Comparing as a set since fs.readdirSync order is
+      // not guaranteed.
+      const rel = personalArgs.paths
+        .map((p) => path.relative(tmpHqRoot, p))
+        .sort();
+
+      // free-co MUST be present (cloud:false, no membership, not _template).
+      expect(rel).toContain(path.join("companies", "free-co"));
+      // knowledge/ MUST still be present (top-level walk unchanged).
+      expect(rel).toContain("knowledge");
+
+      // All other companies MUST be filtered out.
+      for (const forbidden of [
+        path.join("companies", "acme"), // team-synced — goes to cmp_a's bucket
+        path.join("companies", "_template"), // hard-listed
+        path.join("companies", "cloud-team"), // cloud:true
+        path.join("companies", "zilch"), // no marker
+      ]) {
+        expect(rel).not.toContain(forbidden);
+      }
+
+      // The top-level `companies/` directory itself MUST NOT be in the
+      // share list — only specific opt-in subdirs are. This preserves the
+      // invariant that share() never walks the whole companies/ tree.
+      expect(rel).not.toContain("companies");
+
+      // The team target for cmp_a must still receive the unchanged
+      // `companies/acme` path (not double-routed to the personal slot).
+      const acmeCall = (shareSpy.mock.calls as Array<[ShareOptions]>).find(
+        (c) => c[0].company === "cmp_a",
+      );
+      expect(acmeCall).toBeDefined();
+      expect(acmeCall![0].paths).toEqual([
+        path.join(tmpHqRoot, "companies", "acme"),
+      ]);
+    } finally {
+      fs.rmSync(tmpHqRoot, { recursive: true, force: true });
+      vi.unstubAllEnvs();
+    }
+  });
+
+  it("H: with HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL unset, companies/ subdirs are NEVER added to the personal slot — even with a valid cloud:false marker", async () => {
+    // Explicitly assert the OFF case: the env var is the gate. Without it,
+    // a `cloud: false` marker is necessary-but-insufficient; the personal
+    // vault stays in its pre-5.20 shape (top-level entries only, never
+    // any company subdir). Defensive `unstub` in case a leaked stub from
+    // an earlier test bleeds into this one.
+    vi.unstubAllEnvs();
+    const shareSpy = vi.fn().mockResolvedValue(defaultShareResult());
+    const tmpHqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-runner-test-"));
+    try {
+      fs.mkdirSync(path.join(tmpHqRoot, "knowledge"));
+      // Same fixture as G — but the gate is off, so free-co must NOT appear.
+      const dir = path.join(tmpHqRoot, "companies", "free-co");
+      fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(path.join(dir, "company.yaml"), "cloud: false\n");
+
+      const deps = makeDeps({
+        createVaultClient: () =>
+          makeVaultStub({
+            memberships: [{ companyUid: "cmp_a" }],
+            entityGet: (uid: string) =>
+              Promise.resolve({ uid, slug: "acme" } as unknown as EntityInfo),
+            listPersons: () => Promise.resolve([olderPerson]),
+          }),
+        share: shareSpy,
+      });
+
+      const code = await runRunner(
+        ["--companies", "--direction", "push", "--hq-root", tmpHqRoot],
+        deps,
+      );
+      expect(code).toBe(0);
+
+      const personalCall = (shareSpy.mock.calls as Array<[ShareOptions]>).find(
+        (c) => c[0].company?.startsWith("prs_"),
+      );
+      expect(personalCall).toBeDefined();
+      const rel = personalCall![0].paths
+        .map((p) => path.relative(tmpHqRoot, p))
+        .sort();
+
+      // free-co must NOT appear because the env-var gate is off.
+      expect(rel).not.toContain(path.join("companies", "free-co"));
+      // knowledge/ must still appear — the legacy top-level walk is unchanged.
+      expect(rel).toContain("knowledge");
+    } finally {
+      fs.rmSync(tmpHqRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("I: personal slot receives decommissionPrefixes = ['companies/{slug}'] for every team-synced slug; company slots receive none", async () => {
+    // Decommission cleanup runs UNCONDITIONALLY — not gated on
+    // HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL — because the off-ramp must
+    // always be safe to traverse. A user who turned the feature off after
+    // syncing some companies to personal would otherwise be stuck with
+    // permanent orphans.
+    vi.unstubAllEnvs();
+    const shareSpy = vi.fn().mockResolvedValue(defaultShareResult());
+    const tmpHqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-runner-test-"));
+    try {
+      const deps = makeDeps({
+        createVaultClient: () =>
+          makeVaultStub({
+            // Two team-backed companies — both slugs should land in the
+            // personal slot's decommissionPrefixes list.
+            memberships: [
+              { companyUid: "cmp_a" },
+              { companyUid: "cmp_b" },
+            ],
+            entityGet: (uid: string) =>
+              Promise.resolve({
+                uid,
+                slug: uid === "cmp_a" ? "acme" : "beta",
+              } as unknown as EntityInfo),
+            listPersons: () => Promise.resolve([olderPerson]),
+          }),
+        share: shareSpy,
+      });
+
+      const code = await runRunner(
+        ["--companies", "--direction", "push", "--hq-root", tmpHqRoot],
+        deps,
+      );
+      expect(code).toBe(0);
+
+      // Personal slot: decommissionPrefixes covers both team-synced slugs.
+      const personalCall = (shareSpy.mock.calls as Array<[ShareOptions]>).find(
+        (c) => c[0].company?.startsWith("prs_"),
+      );
+      expect(personalCall).toBeDefined();
+      const personalArgs = personalCall![0] as ShareOptions & {
+        decommissionPrefixes?: string[];
+      };
+      expect(personalArgs.decommissionPrefixes?.sort()).toEqual([
+        "companies/acme",
+        "companies/beta",
+      ]);
+
+      // Company slots: NO decommissionPrefixes key at all (preserves the
+      // contract that company-target args stay identical to pre-decommission
+      // shape — symmetric to test F's personalMode/journalSlug pin).
+      const companyCalls = (shareSpy.mock.calls as Array<[ShareOptions]>).filter(
+        (c) => c[0].company?.startsWith("cmp_"),
+      );
+      expect(companyCalls.length).toBeGreaterThan(0);
+      for (const [args] of companyCalls) {
+        expect(Object.keys(args)).not.toContain("decommissionPrefixes");
+      }
+    } finally {
+      fs.rmSync(tmpHqRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("J: personal slot's syncFn (pull) receives includeLocalCompanies + teamSyncedSlugs; company slots receive neither", async () => {
+    // Symmetric to test I (push-side decommissionPrefixes). The pull side
+    // needs the same context to:
+    //   • allow `companies/{cloud-false-slug}/...` keys through when the
+    //     operator opts in (HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1)
+    //   • drop `companies/{team-synced-slug}/...` orphans even when push-
+    //     side decommission hasn't run (e.g. pull-only mode)
+    //
+    // Default direction for runRunner is "pull" so this test exercises
+    // the pull call without needing --direction.
+    vi.stubEnv("HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL", "1");
+    const syncSpy = vi.fn().mockResolvedValue(defaultSyncResult());
+    try {
+      const deps = makeDeps({
+        createVaultClient: () =>
+          makeVaultStub({
+            memberships: [
+              { companyUid: "cmp_a" },
+              { companyUid: "cmp_b" },
+            ],
+            entityGet: (uid: string) =>
+              Promise.resolve({
+                uid,
+                slug: uid === "cmp_a" ? "acme" : "beta",
+              } as unknown as EntityInfo),
+            listPersons: () => Promise.resolve([olderPerson]),
+          }),
+        sync: syncSpy,
+      });
+
+      const code = await runRunner(["--companies"], deps);
+      expect(code).toBe(0);
+
+      // Personal slot: both args present, populated with the team-synced
+      // slug set + the env-var-derived gate flag.
+      const personalCall = (syncSpy.mock.calls as Array<[SyncOptions]>).find(
+        (c) => c[0].company?.startsWith("prs_"),
+      );
+      expect(personalCall).toBeDefined();
+      const personalArgs = personalCall![0] as SyncOptions & {
+        includeLocalCompanies?: boolean;
+        teamSyncedSlugs?: ReadonlySet<string>;
+      };
+      expect(personalArgs.includeLocalCompanies).toBe(true);
+      expect(Array.from(personalArgs.teamSyncedSlugs ?? []).sort()).toEqual([
+        "acme",
+        "beta",
+      ]);
+
+      // Company slots: NEITHER key present (preserves the contract that
+      // company-target args stay identical to pre-Slice-2 shape — symmetric
+      // to test C's personalMode/journalSlug pin and test I's
+      // decommissionPrefixes pin).
+      const companyCalls = (syncSpy.mock.calls as Array<[SyncOptions]>).filter(
+        (c) => c[0].company?.startsWith("cmp_"),
+      );
+      expect(companyCalls.length).toBeGreaterThan(0);
+      for (const [args] of companyCalls) {
+        expect(Object.keys(args)).not.toContain("includeLocalCompanies");
+        expect(Object.keys(args)).not.toContain("teamSyncedSlugs");
+      }
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
+  it("K: personal slot's syncFn (pull) sets includeLocalCompanies=false when env var is unset — pull stays in legacy shape", async () => {
+    // Symmetric to test H (env-var-OFF case for push). Without the gate,
+    // the pull side preserves the pre-5.20 "drop all companies/... keys"
+    // contract regardless of what's actually in the personal bucket.
+    // Defensive unstub in case a previous test leaked a stub.
+    vi.unstubAllEnvs();
+    const syncSpy = vi.fn().mockResolvedValue(defaultSyncResult());
+    const deps = makeDeps({
+      createVaultClient: () =>
+        makeVaultStub({
+          memberships: [{ companyUid: "cmp_a" }],
+          entityGet: (uid: string) =>
+            Promise.resolve({ uid, slug: "acme" } as unknown as EntityInfo),
+          listPersons: () => Promise.resolve([olderPerson]),
+        }),
+      sync: syncSpy,
+    });
+
+    const code = await runRunner(["--companies"], deps);
+    expect(code).toBe(0);
+
+    const personalCall = (syncSpy.mock.calls as Array<[SyncOptions]>).find(
+      (c) => c[0].company?.startsWith("prs_"),
+    );
+    expect(personalCall).toBeDefined();
+    const personalArgs = personalCall![0] as SyncOptions & {
+      includeLocalCompanies?: boolean;
+    };
+    // Explicitly false — not "undefined" — because the runner always
+    // computes the value from the env var, then spreads it for the
+    // personal slot regardless of the value (allows downstream to know
+    // the gate was evaluated, not just "caller forgot to set it").
+    expect(personalArgs.includeLocalCompanies).toBe(false);
+  });
+
   it("F: shareFn paths for company slots stay [hqRoot/companies/{slug}] with no personalMode/journalSlug keys", async () => {
     const shareSpy = vi.fn().mockResolvedValue(defaultShareResult());
     const tmpHqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-runner-test-"));

package/src/bin/sync-runner.ts

@@ -88,6 +88,7 @@ import type { ShareOptions, ShareResult } from "../cli/share.js";
 import type { ConflictStrategy } from "../cli/conflict.js";
 import type { UploadAuthor } from "../s3.js";
 import { collectAndSendTelemetry } from "../telemetry.js";
+import { describeError } from "../lib/describe-error.js";
 import {
   TreeWatcher,
   WatchPushDriver,
@@ -973,11 +974,59 @@ export async function runRunner(
       // target walks every top-level entry under hqRoot minus the exclusion
       // list (see PERSONAL_VAULT_EXCLUDED_TOP_LEVEL). `skipUnchanged: true`
       // keeps both cases efficient on re-runs.
+      // Shared between push and pull for the personal slot. Hoisted out of
+      // the if-blocks below so doPush + doPull see the same set:
+      //
+      //   `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL=1` opts INTO the
+      //   cloud:false → personal-bucket fallback. Default OFF keeps the
+      //   personal vault identical to the pre-5.20 shape until operators
+      //   explicitly enable it.
+      //
+      //   `teamSyncedSlugs` is the slug set the operator currently has
+      //   active team-bucket Memberships for, derived from the live plan.
+      //   Used by:
+      //     • push: filter `companies/` subdir enumeration so a team-synced
+      //       company never rides the personal slot; build
+      //       `decommissionPrefixes` so share() sweeps orphan keys for
+      //       any slug that's now team-backed.
+      //     • pull: filter `listRemoteFiles` so pre-decommission orphan
+      //       keys at `companies/{team-synced-slug}/...` don't get
+      //       re-downloaded into the disk paths the team-bucket pull
+      //       now manages.
+      const includeLocalCompanies =
+        process.env.HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL === "1";
+      const teamSyncedSlugs = new Set(
+        plan
+          .filter((p) => p.personalMode !== true)
+          .map((p) => p.slug),
+      );
+
       if (doPush) {
         activePhase = "push";
+        // For the personal slot we hand share() both (a) the top-level
+        // hqRoot entries that are part of the personal vault and (b) any
+        // `companies/{slug}/` subdirs the user has opted into the personal
+        // bucket via `cloud: false` in `company.yaml`. Slugs that already
+        // own a team bucket (anything else in `plan`) are excluded so a
+        // company is never double-written.
         const pushPaths = target.personalMode === true
-          ? computePersonalVaultPaths(parsed.hqRoot)
+          ? computePersonalVaultPaths(parsed.hqRoot, {
+              includeLocalCompanies,
+              teamSyncedSlugs,
+            })
           : [path.join(parsed.hqRoot, "companies", target.slug)];
+        // For the personal slot, hand share() a list of `companies/{slug}/`
+        // prefixes for every team-synced slug. If the personal-bucket
+        // journal still holds entries under any of those prefixes — i.e.
+        // the company used to ride the personal-bucket fallback and was
+        // since promoted to its own team bucket — share() will sweep
+        // those orphans via DeleteObject + journal removal. Unconditional
+        // (not gated on `HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL`) so cleanup
+        // still runs after an operator turns the feature off: the on-ramp
+        // is opt-in but the off-ramp is always safe to traverse.
+        const decommissionPrefixes = target.personalMode === true
+          ? Array.from(teamSyncedSlugs).map((slug) => `companies/${slug}`)
+          : undefined;
         pushResult = await shareFn({
           paths: pushPaths,
           company: target.uid,
@@ -1007,6 +1056,9 @@ export async function runRunner(
           // in sync-runner.test.ts pins that contract).
           ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
           ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
+          ...(decommissionPrefixes && decommissionPrefixes.length > 0
+            ? { decommissionPrefixes }
+            : {}),
         });
       }
 
@@ -1022,6 +1074,22 @@ export async function runRunner(
           onConflict: parsed.onConflict,
           ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
           ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
+          // Symmetric to the push side: for the personal slot, tell sync()
+          // how to interpret `companies/{slug}/...` keys in the personal
+          // bucket. With `includeLocalCompanies: true`, keys for slugs NOT
+          // in `teamSyncedSlugs` are downloaded (legitimate cloud:false
+          // opt-in content from another machine); keys for slugs IN that
+          // set are dropped as orphans (pre-decommission residue from a
+          // promoted company). With `includeLocalCompanies: false` the
+          // legacy pre-5.20 behavior holds: all `companies/...` keys are
+          // dropped. Only spread for the personal slot so company-target
+          // args stay identical to pre-Slice-2 shape (test C pin).
+          ...(target.personalMode === true
+            ? {
+                includeLocalCompanies,
+                teamSyncedSlugs,
+              }
+            : {}),
           onEvent: tagAndEmit,
         });
       }
@@ -1086,7 +1154,10 @@ export async function runRunner(
         allConflicts.push({ company: companyLabel, path: p, direction: "push" });
       }
     } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
+      // describeError walks the cause chain so AWS SDK v3's "UnknownError"
+      // wrapper surfaces the underlying Node networking error (ENOTFOUND,
+      // ECONNRESET, …) instead of an unactionable bare "UnknownError".
+      const message = describeError(err);
       errors.push({ company: companyLabel, message });
       // `state.status` was seeded as "errored" at loop entry — the throw
       // path leaves it there, and `state.files{Down,Up}loaded` reflects the