@indigoai-us/hq-cloud 5.17.0 → 5.19.0

package/src/s3.test.ts

@@ -20,6 +20,16 @@ const sentCommands: Array<{ name: string; input: Record<string, unknown> }> = []
 // is an empty bucket. Reset in beforeEach so cross-test leakage is impossible.
 let nextListObjectsResponse: Record<string, unknown> = { Contents: [] };
 
+// Per-test override for the GetObjectCommand response. The default fake
+// doesn't model GET (downloadFile previously had no s3.test.ts coverage);
+// downloadFile tests push the Body + Metadata shape they want returned.
+let nextGetObjectResponse: Record<string, unknown> = {
+  Body: (async function* () {
+    yield new Uint8Array();
+  })(),
+  Metadata: {},
+};
+
 vi.mock("@aws-sdk/client-s3", () => {
   class FakeS3Client {
     async send(command: { constructor: { name: string }; input: Record<string, unknown> }): Promise<Record<string, unknown>> {
@@ -35,6 +45,9 @@ vi.mock("@aws-sdk/client-s3", () => {
       if (command.constructor.name === "ListObjectsV2Command") {
         return nextListObjectsResponse;
       }
+      if (command.constructor.name === "GetObjectCommand") {
+        return nextGetObjectResponse;
+      }
       return {};
     }
   }
@@ -66,7 +79,16 @@ vi.mock("@aws-sdk/client-s3", () => {
   };
 });
 
-import { uploadFile, listRemoteFiles } from "./s3.js";
+import {
+  uploadFile,
+  uploadSymlink,
+  downloadFile,
+  listRemoteFiles,
+  SYMLINK_BODY_PREFIX,
+  SYMLINK_MARKER_META_VALUE,
+  encodeSymlinkMetadataValue,
+  decodeSymlinkMetadataValue,
+} from "./s3.js";
 import type { EntityContext } from "./types.js";
 
 function makeCtx(): EntityContext {
@@ -215,4 +237,396 @@ describe("listRemoteFiles", () => {
     expect(files).toHaveLength(1);
     expect(files[0].key).toBe("real.md");
   });
+
+  // Regression: pre-fix, the 0-byte-guard relaxation (`if (!obj.Key) continue;`
+  // replacing `if (!obj.Key || !obj.Size) continue;`) was correct for
+  // legitimate placeholder files like `.gitkeep` but accidentally also let
+  // S3 directory-marker objects (0-byte, key ends in `/`) through to the
+  // planner. Two downstream sites blow up on those:
+  //   1. pull planner (cli/sync.ts `computePullPlan`): for any marker whose
+  //      `path.join(root, key)` resolves to an existing local directory,
+  //      `localExists=true` → `hashFile(localPath)` → `readFileSync` on dir
+  //      → EISDIR "read". The runner reported this for `personal:(company)`.
+  //   2. download path (s3.ts `downloadFile`): for any marker whose local
+  //      path doesn't exist, the planner classifies it `download`;
+  //      `mkdirSync(path.dirname(localPath), recursive)` strips the trailing
+  //      slash and creates the leaf as a directory, then `writeFileSync`
+  //      on the trailing-slash path → EISDIR "open". The runner reported
+  //      this for `indigo:lambda/`.
+  // The fix drops the *definitional* marker shape (trailing-slash AND
+  // size 0) at this site so neither path ever sees them. Real 0-byte
+  // placeholders (`.gitkeep`) never end in `/` and continue to flow
+  // through — covered by the test immediately above.
+  it("drops S3 directory-marker keys (trailing '/', size 0) from list results", async () => {
+    nextListObjectsResponse = {
+      Contents: [
+        { Key: "real.md", Size: 42, LastModified: new Date(), ETag: '"e1"' },
+        { Key: "projects/.gitkeep", Size: 0, LastModified: new Date(), ETag: '"e2"' },
+        { Key: "lambda/", Size: 0, LastModified: new Date(), ETag: '"e3"' },
+        { Key: "core/", Size: 0, LastModified: new Date(), ETag: '"e4"' },
+        { Key: "deeply/nested/marker/", Size: 0, LastModified: new Date(), ETag: '"e5"' },
+      ],
+    };
+
+    const files = await listRemoteFiles(makeCtx());
+
+    // Real files (including the 0-byte .gitkeep placeholder) pass through;
+    // every 0-byte trailing-slash marker is dropped.
+    expect(files.map((f) => f.key).sort()).toEqual([
+      "projects/.gitkeep",
+      "real.md",
+    ]);
+  });
+
+  // Codex P2 follow-up: the marker filter must be definitionally
+  // constrained to size-0 trailing-slash keys. A hypothetical non-empty
+  // object whose key ends in `/` (no hq-cloud code path creates one,
+  // but `ListObjectsV2` returns whatever lives in the bucket) must
+  // NOT be silently hidden — that would mask real bucket state from
+  // the operator. Such an object will still surface EISDIR "open" at
+  // downloadFile when the local path is materialized, which is the
+  // signal needed to reconcile.
+  it("does NOT drop non-empty trailing-slash keys (preserves operator visibility)", async () => {
+    nextListObjectsResponse = {
+      Contents: [
+        { Key: "real.md", Size: 42, LastModified: new Date(), ETag: '"e1"' },
+        // 0-byte marker — filtered.
+        { Key: "marker/", Size: 0, LastModified: new Date(), ETag: '"e2"' },
+        // Non-zero, trailing slash — pathological but unfiltered so it
+        // stays visible. (Cannot be materialized locally; downloadFile
+        // will raise an explicit error pointing at this exact key.)
+        { Key: "exports/report/", Size: 1024, LastModified: new Date(), ETag: '"e3"' },
+      ],
+    };
+
+    const files = await listRemoteFiles(makeCtx());
+
+    expect(files.map((f) => f.key).sort()).toEqual([
+      "exports/report/",
+      "real.md",
+    ]);
+    // Spot-check that the non-empty marker carried its real size.
+    expect(files.find((f) => f.key === "exports/report/")!.size).toBe(1024);
+  });
+});
+
+describe("uploadSymlink", () => {
+  beforeEach(() => {
+    sentCommands.length = 0;
+  });
+
+  it("PUTs the prefixed target as the body bytes and a marker-only metadata flag", async () => {
+    // Body = SYMLINK_BODY_PREFIX + target is the source of truth for
+    // the target string and is load-bearing for two distinct
+    // drift-detection properties:
+    //   1. S3 ETag = MD5(body) varies with target → cross-machine
+    //      target-change propagation works through LIST etags.
+    //   2. The prefix means a symlink to "bar.md" doesn't share an
+    //      ETag with a regular file containing "bar.md" → symlink ↔
+    //      regular-file transitions on the same key are visible to
+    //      peers via LIST.
+    // The metadata is now a fixed marker only — we used to store the
+    // base64'd target there, but S3's 2 KiB header limit made that
+    // brittle for long POSIX targets. Marker-only is bounded.
+    const target = "../real-target.md";
+    await uploadSymlink(makeCtx(), target, "policies/link.md");
+
+    const put = sentCommands.find((c) => c.name === "PutObjectCommand");
+    expect(put).toBeDefined();
+    const meta = put!.input.Metadata as Record<string, string>;
+    expect(meta["hq-symlink-target"]).toBe(SYMLINK_MARKER_META_VALUE);
+    expect(put!.input.Body).toBeDefined();
+    const body = put!.input.Body as Buffer | Uint8Array;
+    expect(Buffer.from(body).toString("utf-8")).toBe(SYMLINK_BODY_PREFIX + target);
+    expect(put!.input.Key).toBe("policies/link.md");
+  });
+
+  it("handles arbitrary Unicode targets via the body (metadata stays marker-only)", async () => {
+    // Codex round-9 P2 follow-up: S3 user-metadata is HTTP-header-
+    // bound. A POSIX symlink target can include any Unicode bytes;
+    // storing the target in metadata risked PutObject rejecting it.
+    // Now the body carries the target — UTF-8 bytes pass through
+    // S3 object content unmolested, no ASCII restriction, no header
+    // limit. Metadata stays as the constant marker.
+    sentCommands.length = 0;
+    const unicodeTarget = "café/🌳/Übung.md";
+    await uploadSymlink(makeCtx(), unicodeTarget, "weird-link.md");
+
+    const put = sentCommands.find((c) => c.name === "PutObjectCommand");
+    expect(put).toBeDefined();
+    const meta = put!.input.Metadata as Record<string, string>;
+    expect(meta["hq-symlink-target"]).toBe(SYMLINK_MARKER_META_VALUE);
+    const body = Buffer.from(put!.input.Body as Buffer | Uint8Array).toString(
+      "utf-8",
+    );
+    expect(body).toBe(SYMLINK_BODY_PREFIX + unicodeTarget);
+  });
+
+  it("handles long targets without exceeding S3's 2 KiB user-metadata limit", async () => {
+    // Codex round-13 P2 follow-up: pre-fix, the metadata value was
+    // base64(target) plus author fields plus the metadata key name.
+    // A target around 1.5 KiB after base64 inflation could push the
+    // total over S3's 2 KiB user-metadata cap and PutObject would
+    // reject the upload. With marker-only metadata, the target's
+    // size only contributes to the body — bounded by S3's 5 GB
+    // object size, not the header limit. Verify a 4 KiB target (well
+    // beyond the old breaking point) round-trips cleanly.
+    sentCommands.length = 0;
+    const longTarget = "deeply/nested/" + "x".repeat(4096);
+    await uploadSymlink(makeCtx(), longTarget, "long-link.md");
+
+    const put = sentCommands.find((c) => c.name === "PutObjectCommand");
+    const meta = put!.input.Metadata as Record<string, string>;
+    expect(meta["hq-symlink-target"]).toBe(SYMLINK_MARKER_META_VALUE);
+    const body = Buffer.from(put!.input.Body as Buffer | Uint8Array).toString(
+      "utf-8",
+    );
+    expect(body).toBe(SYMLINK_BODY_PREFIX + longTarget);
+    // Sanity: marker fits in the header limit by an enormous margin
+    // (1 byte vs 2048).
+    expect(meta["hq-symlink-target"].length).toBeLessThan(10);
+  });
+
+  it("encodeSymlinkMetadataValue / decodeSymlinkMetadataValue round-trip arbitrary UTF-8", async () => {
+    // Direct property test of the helpers — covers the legacy-fallback
+    // path via the round-trip check inside decodeSymlinkMetadataValue.
+    for (const sample of [
+      "",
+      "real.md",
+      "../sibling/file.md",
+      "café/🌳/Übung.md",
+      "/absolute/path/with spaces.md",
+      "deeply/nested/path/" + "x".repeat(200),
+    ]) {
+      expect(decodeSymlinkMetadataValue(encodeSymlinkMetadataValue(sample))).toBe(sample);
+    }
+    // Legacy-fallback: a value that isn't valid base64 of UTF-8 (e.g.
+    // a raw target written by a pre-PR uploader) round-trips as-is.
+    // "real.md" happens to NOT be valid base64 (length not multiple
+    // of 4), so the decoder falls through to returning it raw.
+    expect(decodeSymlinkMetadataValue("real.md")).toBe("real.md");
+  });
+
+  it("produces a different body (and therefore a different S3 ETag) when target changes", async () => {
+    // Direct regression for the cross-machine target-propagation bug:
+    // two uploads with different targets must produce different bodies
+    // so S3-side etag comparison can detect the change. We assert on
+    // the body bytes rather than asking S3 for an ETag (the fake
+    // returns a constant ETag) — equality of body bytes is a necessary
+    // and sufficient property for ETag equality on PutObject.
+    sentCommands.length = 0;
+    await uploadSymlink(makeCtx(), "old-target.md", "policies/link.md");
+    await uploadSymlink(makeCtx(), "new-target.md", "policies/link.md");
+    const puts = sentCommands.filter((c) => c.name === "PutObjectCommand");
+    expect(puts).toHaveLength(2);
+    const bodies = puts.map((p) =>
+      Buffer.from(p.input.Body as Buffer | Uint8Array).toString("utf-8"),
+    );
+    expect(bodies[0]).not.toBe(bodies[1]);
+    expect(bodies[0]).toBe(SYMLINK_BODY_PREFIX + "old-target.md");
+    expect(bodies[1]).toBe(SYMLINK_BODY_PREFIX + "new-target.md");
+  });
+
+  it("produces a different body from a regular file containing the same target string (ETag distinguishability)", async () => {
+    // Codex round-4 P2 follow-up: pre-fix, a symlink whose target
+    // string equaled some regular file's exact contents produced an
+    // identical S3 ETag (since ETag = MD5(body) and bodies were equal).
+    // The LIST-based pull planner can't see per-object metadata, so
+    // it would classify a symlink ↔ regular-file transition on the
+    // same key as "no change" and never replace the local
+    // representation. The SYMLINK_BODY_PREFIX makes the bodies
+    // distinguishable for any realistic regular-file content.
+    sentCommands.length = 0;
+    const sharedString = "real.md";
+    // Make a regular file whose contents are exactly the target string
+    // — the worst-case collision shape pre-prefix.
+    const localFile = path.join(
+      os.tmpdir(),
+      `etag-distinguishability-${Date.now()}-${Math.random()}.md`,
+    );
+    fs.writeFileSync(localFile, sharedString);
+
+    await uploadSymlink(makeCtx(), sharedString, "case-key.md");
+    await uploadFile(makeCtx(), localFile, "regular-key.md");
+    fs.rmSync(localFile, { force: true });
+
+    const puts = sentCommands.filter((c) => c.name === "PutObjectCommand");
+    expect(puts).toHaveLength(2);
+    const symlinkBody = Buffer.from(
+      puts[0].input.Body as Buffer | Uint8Array,
+    ).toString("utf-8");
+    const regularBody = Buffer.from(
+      puts[1].input.Body as Buffer | Uint8Array,
+    ).toString("utf-8");
+
+    // The point of the prefix: symlink record body ≠ regular file body
+    // even when they describe the same string. Different bytes ⇒
+    // different MD5 ⇒ different S3 ETag ⇒ pull planner detects the
+    // transition via LIST without needing a per-object HEAD.
+    expect(symlinkBody).not.toBe(regularBody);
+    expect(regularBody).toBe(sharedString);
+    expect(symlinkBody).toBe(SYMLINK_BODY_PREFIX + sharedString);
+  });
+
+  it("forwards UploadAuthor alongside the symlink-target metadata", async () => {
+    await uploadSymlink(makeCtx(), "real.md", "policies/link.md", {
+      userSub: "abc-123",
+      email: "alice@example.com",
+    });
+
+    const put = sentCommands.find((c) => c.name === "PutObjectCommand");
+    const meta = put!.input.Metadata as Record<string, string>;
+    // Marker-only metadata; target lives in the body.
+    expect(meta["hq-symlink-target"]).toBe(SYMLINK_MARKER_META_VALUE);
+    const body = Buffer.from(put!.input.Body as Buffer | Uint8Array).toString(
+      "utf-8",
+    );
+    expect(body).toBe(SYMLINK_BODY_PREFIX + "real.md");
+    expect(meta["created-by"]).toBe("alice@example.com");
+    expect(meta["created-by-sub"]).toBe("abc-123");
+  });
+});
+
+describe("downloadFile", () => {
+  let tmpRoot: string;
+
+  beforeEach(() => {
+    sentCommands.length = 0;
+    tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "s3-download-test-"));
+  });
+
+  it("materializes a symlink record as a real symlink, sourcing the target from the body", async () => {
+    // Round-13 contract: metadata is just a marker; the body
+    // (after stripping SYMLINK_BODY_PREFIX) is the source of truth
+    // for the target string. downloadFile must consume the body
+    // and use the post-prefix slice as the link target.
+    const target = "../real-target.md";
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new TextEncoder().encode(SYMLINK_BODY_PREFIX + target);
+      })(),
+      Metadata: { "hq-symlink-target": SYMLINK_MARKER_META_VALUE },
+    };
+
+    const localPath = path.join(tmpRoot, "policies", "link.md");
+    await downloadFile(makeCtx(), "policies/link.md", localPath);
+
+    const lstat = fs.lstatSync(localPath);
+    expect(lstat.isSymbolicLink()).toBe(true);
+    expect(fs.readlinkSync(localPath)).toBe(target);
+  });
+
+  it("recovers the target from legacy metadata-only uploads (body lacks the prefix)", async () => {
+    // Backward-compat: an in-flight upload from earlier in this
+    // PR's lifetime stored the target in metadata (raw or base64'd)
+    // rather than in the body. downloadFile must fall back to
+    // decoding the metadata value when the body doesn't start with
+    // SYMLINK_BODY_PREFIX. Tested with a base64'd metadata value
+    // and an empty body — the round-trip-validating decoder
+    // should recover the original target.
+    const target = "legacy/target.md";
+    const legacyMetadata = encodeSymlinkMetadataValue(target);
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new Uint8Array(); // empty body, like a v1-of-this-PR upload
+      })(),
+      Metadata: { "hq-symlink-target": legacyMetadata },
+    };
+
+    const localPath = path.join(tmpRoot, "legacy-link.md");
+    await downloadFile(makeCtx(), "legacy-link.md", localPath);
+
+    expect(fs.lstatSync(localPath).isSymbolicLink()).toBe(true);
+    expect(fs.readlinkSync(localPath)).toBe(target);
+  });
+
+  it("recovers a target longer than the legacy 2 KiB metadata limit from the body", async () => {
+    // Codex round-13 P2 motivating case: targets between ~1.5 KiB
+    // (after base64 inflation) and ~5 GiB (S3 object cap) used to
+    // fail PutObject under the metadata-based design but now
+    // round-trip cleanly through the body. We pick 600 chars: well
+    // past the old metadata-only break point (~1500 base64 bytes ≈
+    // ~1100 raw, plus header overhead crosses 2 KiB), and well
+    // under any sane filesystem's PATH_MAX symlink-target limit
+    // (Linux: 4096, macOS: 1024) so the test creates a real link.
+    const longTarget = "deeply/nested/path/" + "x".repeat(600);
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new TextEncoder().encode(SYMLINK_BODY_PREFIX + longTarget);
+      })(),
+      Metadata: { "hq-symlink-target": SYMLINK_MARKER_META_VALUE },
+    };
+
+    const localPath = path.join(tmpRoot, "long-link.md");
+    await downloadFile(makeCtx(), "long-link.md", localPath);
+
+    expect(fs.lstatSync(localPath).isSymbolicLink()).toBe(true);
+    expect(fs.readlinkSync(localPath)).toBe(longTarget);
+  });
+
+  it("falls back to writing a regular file when hq-symlink-target metadata is absent", async () => {
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new Uint8Array([104, 105]); // "hi"
+      })(),
+      Metadata: {},
+    };
+
+    const localPath = path.join(tmpRoot, "regular.md");
+    await downloadFile(makeCtx(), "regular.md", localPath);
+
+    const lstat = fs.lstatSync(localPath);
+    expect(lstat.isSymbolicLink()).toBe(false);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("hi");
+  });
+
+  it("replaces a stale local symlink with a regular file when remote transitions symlink → regular", async () => {
+    // Codex P2 follow-up: pre-fix, the regular-file branch went
+    // straight to writeFileSync. If localPath was a stale symlink from
+    // a prior sync (when this key was a symlink in cloud), writeFileSync
+    // would FOLLOW the link and overwrite the link's target file
+    // contents — leaving the link in place and the new "regular file at
+    // localPath" never materializing. The fix: lstat before write; if
+    // the existing entry is a symlink, unlink it first.
+    const localPath = path.join(tmpRoot, "transitioned.md");
+    const targetFile = path.join(tmpRoot, "stale-target.md");
+    fs.writeFileSync(targetFile, "do-not-clobber-me");
+    fs.symlinkSync(targetFile, localPath);
+
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new Uint8Array([110, 101, 119]); // "new"
+      })(),
+      Metadata: {}, // no symlink metadata → regular file path
+    };
+
+    await downloadFile(makeCtx(), "transitioned.md", localPath);
+
+    // localPath is now a regular file with the new content...
+    expect(fs.lstatSync(localPath).isSymbolicLink()).toBe(false);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("new");
+    // ...and the link's former target was NOT clobbered.
+    expect(fs.readFileSync(targetFile, "utf-8")).toBe("do-not-clobber-me");
+  });
+
+  it("replaces an existing entry when the link is rewritten (target change)", async () => {
+    // Pre-existing regular file at the target location must not block
+    // creation of a symlink — fs.symlinkSync would EEXIST otherwise.
+    // Same applies to a stale symlink whose target has changed: the new
+    // target string must win.
+    const localPath = path.join(tmpRoot, "rewrite.md");
+    fs.writeFileSync(localPath, "stale regular file");
+
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new Uint8Array();
+      })(),
+      Metadata: { "hq-symlink-target": "fresh-target.md" },
+    };
+
+    await downloadFile(makeCtx(), "rewrite.md", localPath);
+
+    expect(fs.lstatSync(localPath).isSymbolicLink()).toBe(true);
+    expect(fs.readlinkSync(localPath)).toBe("fresh-target.md");
+  });
 });