@indigoai-us/hq-cloud 6.14.41 → 6.14.43

package/src/cli/share.test.ts

@@ -50,6 +50,7 @@ import type { SyncProgressEvent } from "./sync.js";
 import { deleteRemoteFile, downloadFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
 import type { EntityContext } from "../types.js";
 import { VaultAuthError } from "../vault-client.js";
+import { hashFile } from "../journal.js";
 
 const mockConfig: VaultServiceConfig = {
   apiUrl: "https://vault-api.test",
@@ -1276,12 +1277,14 @@ describe("share", () => {
       });
 
       expect(result.filesUploaded).toBe(1);
+      // Prefer journal-baseline If-Match when present (stale-writer guard);
+      // quote form is optional (object-io quoteEtag accepts both).
       expect(uploadFile).toHaveBeenCalledWith(
         expect.anything(),
         testFile,
         "fenced.md",
         undefined,
-        { ifMatch: '"baseline-etag"' },
+        { ifMatch: "baseline-etag" },
       );
     });
 
@@ -1401,6 +1404,448 @@ describe("share", () => {
       expect(result.aborted).toBe(true);
       expect(result.conflictPaths).toEqual(["halting.md"]);
     });
+
+    // ── Stale-writer guard (Ace / vault multi-writer LWW, 2026-08-03) ──────
+    //
+    // Incident: Ace held Jul-28 worker bytes (journal still at that baseline)
+    // while a peer had already advanced the vault object. Push conflict
+    // requires localChanged && remoteChanged, so an unchanged-but-stale local
+    // (skipUnchanged=false full re-push) fell through to PUT with
+    // ifMatch: *live HEAD etag* — which succeeds and last-write-wins the old
+    // body over the peer. Fix: fence with journal remoteEtag so peer advance
+    // yields PreconditionFailed → existing keep/abort/overwrite path.
+
+    it("Ace-shaped stale local under keep: journal-baseline If-Match fails closed (no silent LWW overwrite)", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(path.join(companyRoot, "workers", "fleet-agent-monitoring", "scripts"), {
+        recursive: true,
+      });
+      const rel =
+        "workers/fleet-agent-monitoring/scripts/hourly-enrich.py";
+      const testFile = path.join(companyRoot, rel);
+      // Stale local body (Ace-shaped: old smaller worker artifact).
+      const staleBytes = "# old Jul-28 enricher\n".repeat(50);
+      fs.writeFileSync(testFile, staleBytes);
+      const staleHash = hashFile(testFile);
+
+      const JOURNAL_ETAG = "etag-jul28-old";
+      const PEER_ETAG = "etag-reviewed-new";
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce({
+        lastModified: new Date(),
+        etag: `"${PEER_ETAG}"`,
+        size: 34768,
+      });
+
+      // S3-shaped fence: only land the PUT when If-Match equals live HEAD.
+      vi.mocked(uploadFile).mockImplementation(async (_ctx, _p, _k, _a, pc) => {
+        if (pc?.ifMatch) {
+          const want = String(pc.ifMatch).replace(/^"|"$/g, "");
+          if (want !== PEER_ETAG) {
+            throw fence412();
+          }
+        }
+        return { etag: '"should-not-land"' };
+      });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date(Date.now() - 60_000).toISOString(),
+          files: {
+            [rel]: {
+              // Local matches journal → localChanged=false; remote advanced →
+              // remoteChanged=true. Pre-fix 3-way did NOT conflict, and
+              // ifMatch used live HEAD so the stale PUT succeeded.
+              hash: staleHash,
+              size: staleBytes.length,
+              syncedAt: new Date(Date.now() - 60_000).toISOString(),
+              direction: "down",
+              remoteEtag: JOURNAL_ETAG,
+            },
+          },
+        }),
+      );
+
+      const events: Array<{ type?: string; path?: string; resolution?: string }> = [];
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        // Explicit full re-push path (models skipUnchanged !== true).
+        skipUnchanged: false,
+        onConflict: "keep",
+        onEvent: (e) => events.push(e as { type?: string }),
+      });
+
+      // Must not land the stale body under keep.
+      expect(result.filesUploaded).toBe(0);
+      expect(result.conflictPaths).toEqual([rel]);
+      expect(
+        events.some(
+          (e) => e.type === "conflict" && e.path === rel && e.resolution === "keep",
+        ),
+      ).toBe(true);
+
+      // Any PUT attempt must fence on the journal baseline, not live HEAD.
+      // (If the planner conflicts pre-PUT that is also fine — no unconditional
+      // overwrite of the peer-advanced object.)
+      for (const call of vi.mocked(uploadFile).mock.calls) {
+        const pc = call[4] as { ifMatch?: string; ifNoneMatch?: string } | undefined;
+        expect(pc).toBeDefined();
+        expect(pc!.ifNoneMatch).toBeUndefined();
+        const want = String(pc!.ifMatch).replace(/^"|"$/g, "");
+        expect(want).toBe(JOURNAL_ETAG);
+      }
+      // No unfenced retry under keep.
+      expect(
+        vi.mocked(uploadFile).mock.calls.some((c) => c[4] === undefined),
+      ).toBe(false);
+      // Local stale bytes preserved on disk (keep does not pull).
+      expect(fs.readFileSync(testFile, "utf-8")).toBe(staleBytes);
+    });
+
+    it("legitimate local-ahead still uploads when remote remains at journal baseline", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "workers", "legit-edit.md");
+      fs.mkdirSync(path.dirname(testFile), { recursive: true });
+      fs.writeFileSync(testFile, "my ahead edit");
+      const BASE = "still-current-etag";
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce({
+        lastModified: new Date(Date.now() - 120_000),
+        etag: `"${BASE}"`,
+        size: 4,
+      });
+      vi.mocked(uploadFile).mockResolvedValueOnce({ etag: '"new-upload-etag"' });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date(Date.now() - 60_000).toISOString(),
+          files: {
+            "workers/legit-edit.md": {
+              hash: "stale-journal-hash",
+              size: 4,
+              syncedAt: new Date(Date.now() - 60_000).toISOString(),
+              direction: "up",
+              remoteEtag: BASE,
+            },
+          },
+        }),
+      );
+
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        onConflict: "keep",
+      });
+
+      expect(result.filesUploaded).toBe(1);
+      expect(result.conflictPaths).toEqual([]);
+      expect(uploadFile).toHaveBeenCalledWith(
+        expect.anything(),
+        testFile,
+        "workers/legit-edit.md",
+        undefined,
+        { ifMatch: BASE },
+      );
+    });
+
+    it("true concurrent edit (both sides changed) still conflicts under keep", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "both-changed.md");
+      fs.writeFileSync(testFile, "local concurrent");
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce({
+        lastModified: new Date(),
+        etag: '"remote-concurrent"',
+        size: 20,
+      });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date().toISOString(),
+          files: {
+            "both-changed.md": {
+              hash: "stale-hash",
+              size: 10,
+              syncedAt: new Date().toISOString(),
+              direction: "up",
+              remoteEtag: "baseline-before-both",
+            },
+          },
+        }),
+      );
+
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        onConflict: "keep",
+      });
+
+      expect(result.conflictPaths).toEqual(["both-changed.md"]);
+      expect(result.filesUploaded).toBe(0);
+      expect(uploadFile).not.toHaveBeenCalled();
+    });
+
+    it("localDiverges under keep: pull-kept divergent local cannot silently become write authority", async () => {
+      // After pull --on-conflict keep, journal stamps remoteEtag of the peer
+      // version AND localDiverges=true (local never matched that etag). A later
+      // push must not treat "remote still at stamped etag" as license to PUT
+      // the divergent local body (journal-baseline If-Match would succeed).
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "kept-divergent.md");
+      const localBytes = "my kept divergent local";
+      fs.writeFileSync(testFile, localBytes);
+      const localHash = hashFile(testFile);
+      const REMOTE = "peer-version-etag";
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce({
+        lastModified: new Date(),
+        etag: `"${REMOTE}"`,
+        size: 40,
+      });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date().toISOString(),
+          files: {
+            "kept-divergent.md": {
+              hash: localHash,
+              size: localBytes.length,
+              syncedAt: new Date().toISOString(),
+              direction: "down",
+              remoteEtag: REMOTE,
+              localDiverges: true,
+            },
+          },
+        }),
+      );
+
+      const events: Array<{ type?: string; path?: string; resolution?: string }> = [];
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipUnchanged: false,
+        onConflict: "keep",
+        onEvent: (e) => events.push(e as { type?: string }),
+      });
+
+      expect(result.filesUploaded).toBe(0);
+      expect(result.conflictPaths).toEqual(["kept-divergent.md"]);
+      expect(uploadFile).not.toHaveBeenCalled();
+      expect(
+        events.some(
+          (e) =>
+            e.type === "conflict" &&
+            e.path === "kept-divergent.md" &&
+            e.resolution === "keep",
+        ),
+      ).toBe(true);
+      expect(fs.readFileSync(testFile, "utf-8")).toBe(localBytes);
+    });
+
+    it("localDiverges under overwrite: explicit intent may force-write", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "force-divergent.md");
+      fs.writeFileSync(testFile, "force my local");
+      const localHash = hashFile(testFile);
+      const REMOTE = "peer-etag";
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce({
+        lastModified: new Date(),
+        etag: `"${REMOTE}"`,
+        size: 10,
+      });
+      vi.mocked(uploadFile).mockResolvedValueOnce({ etag: '"forced-etag"' });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date().toISOString(),
+          files: {
+            "force-divergent.md": {
+              hash: localHash,
+              size: 14,
+              syncedAt: new Date().toISOString(),
+              direction: "down",
+              remoteEtag: REMOTE,
+              localDiverges: true,
+            },
+          },
+        }),
+      );
+
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipUnchanged: false,
+        onConflict: "overwrite",
+      });
+
+      expect(result.filesUploaded).toBe(1);
+      // Overwrite path may PUT unfenced after conflict resolution, or with a
+      // fence then retry — either is consent. Must not leave the file unuploaded.
+      expect(uploadFile).toHaveBeenCalled();
+    });
+
+    // Journal still carries remoteEtag from a prior sync, but the remote key
+    // is gone (HEAD null) — e.g. peer delete or operator purge. Recreation
+    // must use If-None-Match:* (create-only), NOT If-Match on the stale
+    // journal etag (which cannot match a missing object and would false-
+    // conflict forever under keep). Concurrent create still 412s via the
+    // existing fence path.
+    it("deleted journaled remote under keep: recreates once with If-None-Match:* and updates journal", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "recreate-deleted.md");
+      const localBytes = "local edit after remote was deleted";
+      fs.writeFileSync(testFile, localBytes);
+      const localHash = hashFile(testFile);
+      const STALE_JOURNAL_ETAG = "etag-before-delete";
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce(null);
+      vi.mocked(uploadFile).mockResolvedValueOnce({ etag: '"recreated-etag"' });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date(Date.now() - 60_000).toISOString(),
+          files: {
+            "recreate-deleted.md": {
+              hash: "stale-journal-hash",
+              size: 8,
+              syncedAt: new Date(Date.now() - 60_000).toISOString(),
+              direction: "up",
+              remoteEtag: STALE_JOURNAL_ETAG,
+            },
+          },
+        }),
+      );
+
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipUnchanged: false,
+        onConflict: "keep",
+      });
+
+      expect(result.filesUploaded).toBe(1);
+      expect(result.conflictPaths).toEqual([]);
+      expect(result.filesSkipped).toBe(0);
+      // Exactly one create-fenced PUT — not journal If-Match, not unfenced.
+      expect(uploadFile).toHaveBeenCalledTimes(1);
+      expect(uploadFile).toHaveBeenCalledWith(
+        expect.anything(),
+        testFile,
+        "recreate-deleted.md",
+        undefined,
+        { ifNoneMatch: "*" },
+      );
+      const pc = vi.mocked(uploadFile).mock.calls[0][4] as
+        | { ifMatch?: string; ifNoneMatch?: string }
+        | undefined;
+      expect(pc?.ifMatch).toBeUndefined();
+      expect(pc?.ifNoneMatch).toBe("*");
+
+      const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+      expect(journal.files["recreate-deleted.md"]).toMatchObject({
+        hash: localHash,
+        remoteEtag: "recreated-etag",
+        direction: "up",
+      });
+    });
+
+    it("deleted journaled remote + concurrent create 412 under keep: existing conflict semantics", async () => {
+      // Same setup as recreation, but a peer creates the key between HEAD and
+      // PUT → If-None-Match:* 412s → keep surfaces conflict, no journal stamp.
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const testFile = path.join(companyRoot, "race-recreate.md");
+      fs.writeFileSync(testFile, "my recreated local");
+
+      vi.mocked(headRemoteFile).mockResolvedValueOnce(null);
+      vi.mocked(uploadFile).mockRejectedValueOnce(fence412());
+      vi.mocked(downloadFile).mockImplementationOnce(async (_ctx, _key, dest) => {
+        fs.writeFileSync(dest as string, "peer created first");
+        return undefined as never;
+      });
+
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date(Date.now() - 60_000).toISOString(),
+          files: {
+            "race-recreate.md": {
+              hash: "old-hash",
+              size: 4,
+              syncedAt: new Date(Date.now() - 60_000).toISOString(),
+              direction: "up",
+              remoteEtag: "etag-before-delete",
+            },
+          },
+        }),
+      );
+
+      const events: Array<{ type?: string; path?: string }> = [];
+      const result = await share({
+        paths: [testFile],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipUnchanged: false,
+        onConflict: "keep",
+        onEvent: (e) => events.push(e as { type?: string }),
+      });
+
+      expect(result.filesUploaded).toBe(0);
+      expect(result.conflictPaths).toEqual(["race-recreate.md"]);
+      expect(result.aborted).toBe(false);
+      expect(
+        events.some((e) => e.type === "conflict" && e.path === "race-recreate.md"),
+      ).toBe(true);
+      // First (only) attempt must be create-fenced; no unfenced retry under keep.
+      expect(uploadFile).toHaveBeenCalledTimes(1);
+      expect(vi.mocked(uploadFile).mock.calls[0][4]).toEqual({ ifNoneMatch: "*" });
+      // Journal not advanced to a successful upload etag (entry may still be
+      // the prior baseline — next pass re-evaluates).
+      const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+      expect(journal.files["race-recreate.md"]?.remoteEtag).toBe("etag-before-delete");
+      expect(journal.files["race-recreate.md"]?.hash).toBe("old-hash");
+      expect(fs.readFileSync(testFile, "utf-8")).toBe("my recreated local");
+    });
   });
 
   it("scoped push (plan-exceeds-grant): syncs the in-scope subset, skips out-of-scope paths, never aborts (feedback_ded09d56)", async () => {

package/src/cli/share.ts

@@ -1482,8 +1482,14 @@ async function executeUploads(
       }
       throw headErr;
     }
+    // Journal entry for this key (if any). Used both for 3-way conflict
+    // classification and for the write fence below: Prefer last-synced
+    // `remoteEtag` over live HEAD so a peer-advanced object cannot be
+    // silently LWW-overwritten by a journaled stale local body (Ace-shaped
+    // vault regression 2026-08-03).
+    const journalEntry = run.journal.files[relativePath];
+
     if (remoteMeta) {
-      const journalEntry = run.journal.files[relativePath];
       const localChanged = !!journalEntry && journalEntry.hash !== localHash;
       const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
 
@@ -1523,7 +1529,14 @@ async function executeUploads(
         return;
       }
 
-      if ((localChanged && remoteChanged) || isFreshCollision) {
+      // `localDiverges` means a prior pull-keep stamped the remote etag for
+      // re-fire silence (#137) while local never matched that remote. The
+      // journal-baseline If-Match fence alone would SUCCEED when remote is
+      // still at that etag, silently promoting the divergent local to write
+      // authority. Route through conflict (keep/abort/overwrite) instead.
+      const divergentLocalHonesty = !!journalEntry?.localDiverges;
+
+      if ((localChanged && remoteChanged) || isFreshCollision || divergentLocalHonesty) {
         // Cloud-authoritative paths are already skipped at the top of
         // processUploadItem (before HEAD), so anything reaching here is a genuine
         // conflict on a client-owned file.
@@ -1555,13 +1568,24 @@ async function executeUploads(
           counters.filesSkipped++;
           return;
         }
+        // overwrite: fall through to conditional PUT (or unfenced retry on 412).
       }
     }
 
     if (aborted) return;
 
+    // Write fence: when remote exists, prefer last-synced journal remoteEtag
+    // over live HEAD so a peer-advanced object fails closed (412 → keep/
+    // abort/overwrite). Live-HEAD If-Match alone only covers HEAD→PUT TOCTOU.
+    // When remote is missing (HEAD null), always create-fence with
+    // If-None-Match:* — even if the journal still carries a stale remoteEtag
+    // from a prior sync (deleted key recreation under keep). Using journal
+    // If-Match against a missing object cannot succeed and false-conflicts.
+    // Legacy entries without remoteEtag fall back to live HEAD (prior behavior).
     const precondition: PutPrecondition = remoteMeta
-      ? { ifMatch: remoteMeta.etag }
+      ? journalEntry?.remoteEtag
+        ? { ifMatch: journalEntry.remoteEtag }
+        : { ifMatch: remoteMeta.etag }
       : { ifNoneMatch: "*" };
 
     const performUpload = async (pc: PutPrecondition | undefined): Promise<void> => {

package/src/lib/net-errors.test.ts

@@ -8,7 +8,7 @@ describe("isTransientNetworkError", () => {
     expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
   });
 
-  it("treats known transport `.code`s as transient", () => {
+  it("treats direct transport `.code`s as transient", () => {
     for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
       const err = Object.assign(new Error("boom"), { code });
       expect(isTransientNetworkError(err)).toBe(true);
@@ -23,6 +23,18 @@ describe("isTransientNetworkError", () => {
     expect(isTransientNetworkError(err)).toBe(true);
   });
 
+  it("walks an AWS SDK UnknownError cause chain to the underlying DNS error", () => {
+    const cause = Object.assign(
+      new Error("getaddrinfo ENOTFOUND hq-vault-prs-test.s3.us-east-1.amazonaws.com"),
+      { code: "ENOTFOUND", syscall: "getaddrinfo" },
+    );
+    const err = Object.assign(new Error("UnknownError"), {
+      name: "UnknownError",
+      cause,
+    });
+    expect(isTransientNetworkError(err)).toBe(true);
+  });
+
   it("walks AggregateError.errors (happy-eyeballs)", () => {
     const agg = Object.assign(new AggregateError([
       Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),

package/src/lib/net-errors.ts

@@ -2,9 +2,10 @@
  * Transient network-failure classification for the sync runner.
  *
  * The auto-sync watcher runs an unattended poll loop. At the top of every pass
- * it calls `GET /membership/me` to resolve which companies to sync. When the
- * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
- * vault API momentarily unreachable) that call fails at the transport layer —
+ * it calls `GET /membership/me` to resolve which companies to sync, then it
+ * runs a per-company fanout. When the machine is briefly offline (wifi drop,
+ * waking from sleep, a DNS blip, the vault API or a vault S3 endpoint
+ * momentarily unreachable), either boundary can fail at the transport layer —
  * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
  * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
  *
@@ -12,9 +13,11 @@
  * back. But the runner used to `return 1` for it, the watch loop propagated
  * that non-zero exit, the process exited, and the menubar supervisor reported
  * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
- * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
- * retry instead of surfacing a false crash, while a one-shot `hq sync` still
- * exits non-zero so a human running it by hand sees the failure.
+ * HQ-SYNC-1W cluster. A company-leg failure was also previously counted as an
+ * exit-2 partial sync, producing the HQ-SYNC-X symptom even though it was just
+ * retryable DNS/transport loss. We classify both boundaries so the watch loop
+ * can stay alive and retry instead of surfacing a false crash, while a one-shot
+ * `hq sync` still exits non-zero so a human running it by hand sees the failure.
  */
 
 /**