@indigoai-us/hq-cloud 5.16.0 → 5.18.1

package/src/s3.ts

@@ -73,6 +73,102 @@ function buildAuthorMetadata(
   return meta;
 }
 
+/**
+ * S3 user-metadata header that marks an object as a symlink record.
+ * The value is now an OPAQUE MARKER ('1') — the target lives in the
+ * object body. Earlier drafts of this feature stored the target in
+ * metadata (raw, then base64), but S3 user-metadata is HTTP-header-
+ * bound: total ≤ 2 KiB across all user-defined keys + values. A
+ * sufficiently long POSIX target (or one with author-metadata
+ * adding to the total) would exceed the limit and PutObject would
+ * reject the upload outright. Moving the target to the body — which
+ * has no such limit — makes target length bounded only by S3's 5 GB
+ * object size cap. The metadata header still serves as the read-
+ * time discriminator (cheaper than peeking at body bytes via HEAD).
+ *
+ * Backward compat: downloadFile prefers the body (sliced after
+ * SYMLINK_BODY_PREFIX) for the target string. If the body doesn't
+ * carry the prefix (a legacy upload from earlier in this PR's
+ * lifetime), it falls back to base64-decoding the metadata value —
+ * the round-trip-validating decoder returns raw or decoded as
+ * appropriate. Any prior in-flight upload still resolves correctly.
+ */
+export const SYMLINK_TARGET_META_KEY = "hq-symlink-target";
+
+/**
+ * Constant value written to SYMLINK_TARGET_META_KEY. Any non-empty
+ * string would work as a discriminator — '1' is just compact and
+ * conventional for boolean flags in HTTP headers.
+ */
+export const SYMLINK_MARKER_META_VALUE = "1";
+
+/**
+ * Encode a target for the S3 metadata header value. Retained as the
+ * legacy encoder so a downloader can still receive it and round-trip
+ * via decodeSymlinkMetadataValue, but new uploads use
+ * SYMLINK_MARKER_META_VALUE — the target lives in the body now.
+ */
+export function encodeSymlinkMetadataValue(target: string): string {
+  return Buffer.from(target, "utf-8").toString("base64");
+}
+
+/**
+ * Decode a target from the S3 metadata header value. Used as a
+ * legacy fallback when the body doesn't carry SYMLINK_BODY_PREFIX
+ * (i.e. an in-flight upload from earlier in this PR before the
+ * marker-only metadata convention). Round-trip-validates: if the
+ * value isn't valid base64 of UTF-8, returns the raw string.
+ */
+export function decodeSymlinkMetadataValue(value: string): string {
+  try {
+    const decoded = Buffer.from(value, "base64").toString("utf-8");
+    if (Buffer.from(decoded, "utf-8").toString("base64") === value) {
+      return decoded;
+    }
+  } catch {
+    // fall through
+  }
+  return value;
+}
+
+/**
+ * Magic prefix prepended to symlink-record bodies on the wire. Two
+ * properties this gives us:
+ *
+ *   1. ETag distinguishability. S3 ETag = MD5(body). Without a prefix,
+ *      a symlink whose target string equals some regular file's exact
+ *      contents would produce the same ETag, and the LIST-based pull
+ *      planner (which can't see per-object metadata) would classify a
+ *      symlink ↔ regular-file transition as "no change" and never
+ *      replace the local representation. The prefix makes those two
+ *      shapes ETag-distinguishable for the realistic case (collision
+ *      now requires a regular file whose contents *start* with this
+ *      prefix, which is implausible for any non-malicious source).
+ *
+ *   2. Fallback discriminator. If user-metadata is ever lost (S3
+ *      cross-region replication of object data only, manual S3 console
+ *      copy that drops Metadata), the body prefix lets a downloader
+ *      recover the symlink record without needing the metadata header.
+ *      We don't currently rely on this fallback — the metadata header
+ *      is still the primary discriminator on the read path — but the
+ *      prefix keeps the option open and avoids painting us into a
+ *      "metadata is the only signal" corner.
+ *
+ * Format: `hq-symlink:` + target string (UTF-8 bytes). No trailing
+ * newline. The colon separates the marker from the target so a future
+ * extension can encode additional fields if needed.
+ */
+export const SYMLINK_BODY_PREFIX = "hq-symlink:";
+
+/**
+ * Encode/decode the symlink wire body. Kept as exported helpers so the
+ * format is centrally defined and tests can probe both sides without
+ * duplicating the prefix string.
+ */
+export function encodeSymlinkBody(target: string): Buffer {
+  return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
+}
+
 export async function uploadFile(
   ctx: EntityContext,
   localPath: string,
@@ -117,6 +213,72 @@ export async function uploadFile(
   return { etag: response.ETag || "" };
 }
 
+/**
+ * Upload a symlink as a zero-byte object whose user metadata carries the
+ * link's target string. Mirrors uploadFile's signature so callers can pick
+ * the right primitive once they've classified the entry as link vs file.
+ *
+ * The target string is stored verbatim — whatever fs.readlinkSync returned.
+ * Relative targets transfer cleanly across machines; absolute targets are
+ * preserved as-is and may be broken on a destination that doesn't share
+ * the source's $HOME layout. Cross-machine portability of absolute targets
+ * is out of scope for this primitive — the policy decision lives in the
+ * caller (currently: upload anyway, never silently rewrite).
+ */
+export async function uploadSymlink(
+  ctx: EntityContext,
+  target: string,
+  key: string,
+  author?: UploadAuthor,
+): Promise<{ etag: string }> {
+  const client = buildClient(ctx);
+
+  // Same created-at preservation logic as uploadFile so the hq-console
+  // NEW-pill ageing window doesn't reset when a symlink is re-uploaded
+  // unchanged across syncs.
+  let createdAt = new Date().toISOString();
+  if (author) {
+    try {
+      const head = await client.send(
+        new HeadObjectCommand({ Bucket: ctx.bucketName, Key: key }),
+      );
+      const existing = head.Metadata?.["created-at"];
+      if (typeof existing === "string" && existing.length > 0) {
+        createdAt = existing;
+      }
+    } catch {
+      // First upload of this key, or HEAD denied — keep `now`.
+    }
+  }
+
+  const Metadata: Record<string, string> = {
+    // Marker-only: a constant flag value, not the target. The body
+    // is the source of truth for the target (no 2 KiB cap, no
+    // header-encoding considerations). See SYMLINK_TARGET_META_KEY
+    // doc for the full reasoning.
+    [SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
+    ...(author ? buildAuthorMetadata(author, createdAt) : {}),
+  };
+
+  const response = await client.send(
+    new PutObjectCommand({
+      Bucket: ctx.bucketName,
+      Key: key,
+      // Body = SYMLINK_BODY_PREFIX + target (UTF-8). The prefix is what
+      // makes a symlink record's ETag distinguishable from a regular
+      // file whose contents happen to equal the target string — the
+      // LIST-based pull planner can't see per-object metadata, so ETag
+      // is its only drift signal across symlink ↔ regular-file
+      // transitions. See SYMLINK_BODY_PREFIX doc above.
+      Body: encodeSymlinkBody(target),
+      ContentType: "application/octet-stream",
+      Metadata,
+    }),
+  );
+
+  return { etag: response.ETag || "" };
+}
+
 export async function downloadFile(
   ctx: EntityContext,
   key: string,
@@ -140,6 +302,93 @@ export async function downloadFile(
     fs.mkdirSync(dir, { recursive: true });
   }
 
+  // Symlink path: presence of SYMLINK_TARGET_META_KEY (any non-empty
+  // value) is the discriminator. The TARGET is now sourced from the
+  // body — the marker-only metadata convention removes the 2 KiB
+  // header limit so long POSIX targets don't fail PutObject.
+  //
+  // S3 lowercases user-metadata keys on read (and sometimes on
+  // write), so the lookup uses the lowercased form. We don't
+  // normalize Metadata keys ourselves — the AWS SDK already does it.
+  const symlinkMarker = response.Metadata?.[SYMLINK_TARGET_META_KEY];
+  const isSymlinkRecord =
+    typeof symlinkMarker === "string" && symlinkMarker.length > 0;
+  if (isSymlinkRecord) {
+    // Consume the body to extract the target. Symlink record bodies
+    // are bounded by target length (typically <300 bytes for
+    // relative paths, hard-capped by S3's 5 GB object size); the
+    // read is cheap. Drain explicitly so the SDK's HTTP socket is
+    // released back to the connection pool — without this, a sync
+    // over a tree with many symlinks can stall or pool-exhaust.
+    const chunks: Buffer[] = [];
+    const stream = response.Body as AsyncIterable<Uint8Array>;
+    for await (const chunk of stream) {
+      chunks.push(Buffer.from(chunk));
+    }
+    const bodyString = Buffer.concat(chunks).toString("utf-8");
+
+    let symlinkTarget: string;
+    if (bodyString.startsWith(SYMLINK_BODY_PREFIX)) {
+      symlinkTarget = bodyString.slice(SYMLINK_BODY_PREFIX.length);
+    } else {
+      // Backward-compat fallback: a legacy upload from earlier in
+      // this PR's lifetime stored the target in metadata (raw or
+      // base64) rather than the body. decodeSymlinkMetadataValue
+      // round-trip-validates so a raw value passes through and a
+      // base64 value decodes; either way we get the target.
+      symlinkTarget = decodeSymlinkMetadataValue(symlinkMarker);
+    }
+
+    if (symlinkTarget.length === 0) {
+      throw new Error(
+        `Symlink record for ${key} had no target (body: ${bodyString.length} bytes, marker: ${symlinkMarker})`,
+      );
+    }
+
+    // Replace whatever's at localPath. unlink covers regular files; for
+    // a stale symlink this also frees the slot. ENOENT is fine — first
+    // download of this key has nothing to clear. Other errors propagate
+    // because they signal a real problem (permissions, parent missing).
+    try {
+      fs.unlinkSync(localPath);
+    } catch (err: unknown) {
+      if (
+        !err ||
+        typeof err !== "object" ||
+        !("code" in err) ||
+        (err as { code?: string }).code !== "ENOENT"
+      ) {
+        throw err;
+      }
+    }
+    fs.symlinkSync(symlinkTarget, localPath);
+    return;
+  }
+
+  // Symmetric to the symlink branch above: when a key was previously a
+  // symlink and is later replaced in S3 by a regular object, the local
+  // path still holds the stale symlink from the last sync. Without this
+  // unlink, fs.writeFileSync follows the link and overwrites its
+  // target file's contents — leaving the link in place and the new
+  // regular object never materializing at the intended path. lstat
+  // (not statSync) avoids following the link to test what's there.
+  try {
+    const existing = fs.lstatSync(localPath);
+    if (existing.isSymbolicLink()) {
+      fs.unlinkSync(localPath);
+    }
+  } catch (err: unknown) {
+    // ENOENT means nothing's there; let writeFileSync handle creation.
+    if (
+      err &&
+      typeof err === "object" &&
+      "code" in err &&
+      (err as { code?: string }).code !== "ENOENT"
+    ) {
+      throw err;
+    }
+  }
+
   const chunks: Buffer[] = [];
   const stream = response.Body as AsyncIterable<Uint8Array>;
   for await (const chunk of stream) {
@@ -179,6 +428,28 @@ export async function listRemoteFiles(
       // plan. Narrow the guard to "no key" only; surface real 0-byte
       // objects to the planner.
       if (!obj.Key) continue;
+      // Drop S3 directory-marker objects: the canonical shape is `0-byte
+      // key ending in '/'` (S3 console "Create folder", `aws s3 sync` of
+      // empty dirs, sync tools that mirror empty trees). Two downstream
+      // sites blow up on them — pull planner (sync.ts: `hashFile` calls
+      // `readFileSync` on an existing local dir → EISDIR "read") and the
+      // download path (s3.ts: `writeFileSync` on a trailing-slash path
+      // → EISDIR "open" after the parent mkdir creates the leaf as a
+      // directory). Filtering here eliminates both.
+      //
+      // Narrow on Size===0 (not just trailing-slash) so a hypothetical
+      // non-empty object whose key happens to end in '/' is NOT silently
+      // hidden — it stays visible and downloadFile surfaces the same
+      // EISDIR "open" error pointing at the specific key, which is the
+      // signal an operator needs to reconcile the bucket. The vault
+      // service doesn't have a code path that produces such an object,
+      // but ListObjectsV2 returns whatever lives in the bucket; silent
+      // drop would be worse than loud failure for that case.
+      //
+      // Real 0-byte placeholders like `.gitkeep` never end in `/` and
+      // continue to flow through — the 5.13.0 `.gitkeep` regression
+      // remains fixed.
+      if (obj.Key.endsWith("/") && (obj.Size ?? 0) === 0) continue;
 
       files.push({
         key: obj.Key,

package/tsconfig.json

@@ -1,6 +1,17 @@
 {
-  "extends": "../../tsconfig.base.json",
   "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "lib": ["ES2022"],
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
     "outDir": "dist",
     "rootDir": "src"
   },