@indigoai-us/hq-cloud 5.17.0 → 5.18.1

package/src/cli/share.ts

@@ -9,9 +9,17 @@ import * as fs from "fs";
 import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
-import { uploadFile, headRemoteFile, deleteRemoteFile } from "../s3.js";
+import { uploadFile, uploadSymlink, headRemoteFile, deleteRemoteFile } from "../s3.js";
 import type { UploadAuthor } from "../s3.js";
-import { readJournal, writeJournal, hashFile, updateEntry, removeEntry, normalizeEtag } from "../journal.js";
+import {
+  readJournal,
+  writeJournal,
+  hashFile,
+  hashSymlinkTarget,
+  updateEntry,
+  removeEntry,
+  normalizeEtag,
+} from "../journal.js";
 import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy } from "./conflict.js";
@@ -30,11 +38,24 @@ import type { SyncProgressEvent } from "./sync.js";
 type PushPlanItem =
   | {
       action: "upload";
+      kind: "file";
       absolutePath: string;
       relativePath: string;
       localHash: string;
       size: number;
     }
+  | {
+      action: "upload";
+      kind: "symlink";
+      absolutePath: string;
+      relativePath: string;
+      // The link's target string verbatim (whatever readlink returned).
+      // Hashed into localHash so a target rewrite re-uploads even when
+      // skipUnchanged is on; size stays 0 because the wire body is empty.
+      target: string;
+      localHash: string;
+      size: 0;
+    }
   | {
       action: "skip-size-limit";
       absolutePath: string;
@@ -64,13 +85,46 @@ interface PushPlan {
  * Consumers that want a conflict count get it from the `complete` event.
  */
 function computePushPlan(
-  filesToShare: { absolutePath: string; relativePath: string }[],
+  filesToShare: CollectedEntry[],
   journal: SyncJournal,
   skipUnchanged: boolean,
 ): PushPlan {
   const items: PushPlanItem[] = [];
 
-  for (const { absolutePath, relativePath } of filesToShare) {
+  for (const entry of filesToShare) {
+    const { absolutePath, relativePath } = entry;
+
+    // Symlinks bypass the size-limit gate (the wire body is small,
+    // bounded by target length) and hash the target through the
+    // symlink-namespaced hash so a symlink to "real.md" can never
+    // collide with a regular file containing the bytes "real.md" in
+    // the skip-unchanged gate. The target is what we're actually
+    // uploading, so its hash is what should drive change detection —
+    // a target rewrite must re-fire an upload even when the link
+    // itself "looks" identical.
+    if (entry.kind === "symlink") {
+      const localHash = hashSymlinkTarget(entry.target);
+
+      if (skipUnchanged) {
+        const existing = journal.files[relativePath];
+        if (existing && existing.hash === localHash) {
+          items.push({ action: "skip-unchanged", absolutePath, relativePath });
+          continue;
+        }
+      }
+
+      items.push({
+        action: "upload",
+        kind: "symlink",
+        absolutePath,
+        relativePath,
+        target: entry.target,
+        localHash,
+        size: 0,
+      });
+      continue;
+    }
+
     if (!isWithinSizeLimit(absolutePath)) {
       items.push({ action: "skip-size-limit", absolutePath, relativePath });
       continue;
@@ -89,6 +143,7 @@ function computePushPlan(
     const size = fs.statSync(absolutePath).size;
     items.push({
       action: "upload",
+      kind: "file",
       absolutePath,
       relativePath,
       localHash,
@@ -204,11 +259,13 @@ export interface ShareOptions {
    *     mirror a destructive local checkout opts in here explicitly.
    *
    * Independently of this policy, an entry is also dropped from the
-   * plan when `shouldSync(localPath, false) === false` — i.e. the
-   * current ignore filter would have skipped the path on pull. That
-   * symmetry blocks the failure mode where a path was filtered locally
-   * but lived in the vault (and the journal) from an older HQ layout
-   * or a different machine, causing the next push to erase it.
+   * plan when neither the file-shape nor the directory-shape probe of
+   * `shouldSync` accepts the path — i.e. the current ignore filter
+   * would have skipped the path on pull (whether classified as a
+   * regular file or a symlink record / directory). That symmetry
+   * blocks the failure mode where a path was filtered locally but
+   * lived in the vault (and the journal) from an older HQ layout or
+   * a different machine, causing the next push to erase it.
    */
   propagateDeletePolicy?: "owned-only" | "all";
   /**
@@ -453,18 +510,26 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
       }
     }
 
-    // Upload
+    // Upload — symlinks go through uploadSymlink (zero-byte body + target
+    // metadata), regular files through uploadFile (file contents). The
+    // discriminator is item.kind set by computePushPlan; both branches
+    // converge on the same journal/event update path below.
     try {
-      const stat = fs.statSync(absolutePath);
+      const isSymlinkUpload = item.kind === "symlink";
+      const size = isSymlinkUpload ? 0 : fs.statSync(absolutePath).size;
 
-      const { etag } = options.author
-        ? await uploadFile(ctx, absolutePath, relativePath, options.author)
-        : await uploadFile(ctx, absolutePath, relativePath);
+      const { etag } = isSymlinkUpload
+        ? options.author
+          ? await uploadSymlink(ctx, item.target, relativePath, options.author)
+          : await uploadSymlink(ctx, item.target, relativePath)
+        : options.author
+          ? await uploadFile(ctx, absolutePath, relativePath, options.author)
+          : await uploadFile(ctx, absolutePath, relativePath);
 
       // Update journal with optional message; capture the post-upload ETag
       // so the next sync can distinguish "remote moved since we last wrote"
       // from "user edited locally" without conflating the two.
-      updateEntry(journal, relativePath, localHash, stat.size, "up", etag);
+      updateEntry(journal, relativePath, localHash, size, "up", etag);
       if (message) {
         journal.files[relativePath] = {
           ...journal.files[relativePath],
@@ -473,11 +538,11 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
       }
 
       filesUploaded++;
-      bytesUploaded += stat.size;
+      bytesUploaded += size;
       emit({
         type: "progress",
         path: relativePath,
-        bytes: stat.size,
+        bytes: size,
         ...(message ? { message } : {}),
       });
     } catch (err) {
@@ -578,42 +643,104 @@ function resolveActiveCompany(hqRoot: string): string | undefined {
   return undefined;
 }
 
+/**
+ * One entry produced by collectFiles/walkDir. Files describe regular
+ * payloads that get hashed + size-checked + uploaded via uploadFile;
+ * symlinks describe link records whose target string flows through
+ * uploadSymlink as user metadata. Walked-dir traversal NEVER descends
+ * into a symlink — directory symlinks are recorded as link entries and
+ * left at that, so following them never duplicates content into the
+ * wrong vault path (the same topology safety the legacy walker provided
+ * by accident of Dirent.isFile() returning false for links).
+ */
+type CollectedEntry =
+  | { kind: "file"; absolutePath: string; relativePath: string }
+  | { kind: "symlink"; absolutePath: string; relativePath: string; target: string };
+
 /**
  * Collect files from paths (expanding directories recursively).
  *
  * Remote S3 keys are computed relative to `syncRoot` (companies/{slug}/), not
  * `hqRoot`. Files outside `syncRoot` are skipped with a warning — sharing
  * anything outside a company's folder would leak state into the wrong vault.
+ *
+ * Symlink classification uses lstat (not stat), so a top-level path that is
+ * itself a symlink is recorded as a link record rather than dereferenced.
+ * Pre-fix, statSync followed the link and the target's bytes were uploaded
+ * under the link's key — silently flattening the link topology.
  */
 function collectFiles(
   paths: string[],
   hqRoot: string,
   syncRoot: string,
   filter: (p: string, isDir?: boolean) => boolean,
-): { absolutePath: string; relativePath: string }[] {
-  const results: { absolutePath: string; relativePath: string }[] = [];
+): CollectedEntry[] {
+  const results: CollectedEntry[] = [];
 
   for (const p of paths) {
     const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
 
-    if (!fs.existsSync(absolutePath)) {
+    // existsSync follows symlinks: a dangling top-level link will report
+    // not-existing and be skipped here. lstatSync below handles the
+    // valid-link case directly without needing the existsSync gate.
+    let lstat: fs.Stats;
+    try {
+      lstat = fs.lstatSync(absolutePath);
+    } catch {
       console.error(`  Warning: ${p} does not exist, skipping.`);
       continue;
     }
 
+    // Containment check is split by entry kind: regular files and
+    // directories use isWithin (which canonicalizes via realpathSync to
+    // tolerate macOS APFS case-insensitivity), but symlinks use the
+    // link's own pathname rather than the link's resolved target. A
+    // valid use case for this asymmetry: a directory symlink whose
+    // target lives outside the company folder (e.g. companies/{co}/
+    // knowledge → repos/private/knowledge-{co}/) — the LINK itself is
+    // inside the company folder and is exactly what we want to record,
+    // but isWithin's realpath would say the resolved target is outside
+    // and reject the share. Recording symlinks rather than following
+    // them is the whole topology contract this fix establishes; the
+    // containment check needs to honor the same semantic.
+    if (lstat.isSymbolicLink()) {
+      if (!isWithinForLink(syncRoot, absolutePath)) {
+        console.error(`  Warning: ${p} is outside company folder, skipping.`);
+        continue;
+      }
+      const relativePath = path.relative(syncRoot, absolutePath);
+      // Probe the filter with both isDir hints — we don't know whether
+      // the link's target is a file or a directory without
+      // stat-following the link, which we explicitly avoid (it would
+      // re-introduce the dereference behavior this whole change set is
+      // designed to prevent). An `.hqinclude` dir-only pattern like
+      // `companies/*/knowledge/` only matches with isDir=true, so a
+      // single isDir=false probe would silently drop directory
+      // symlinks under allowlist mode (the motivating case for this
+      // whole branch). The filter is pure path lookup with no I/O,
+      // so two calls are free.
+      if (!filter(absolutePath, false) && !filter(absolutePath, true)) continue;
+      results.push({
+        kind: "symlink",
+        absolutePath,
+        relativePath,
+        target: fs.readlinkSync(absolutePath),
+      });
+      continue;
+    }
+
     if (!isWithin(syncRoot, absolutePath)) {
       console.error(`  Warning: ${p} is outside company folder, skipping.`);
       continue;
     }
 
-    const stat = fs.statSync(absolutePath);
-    if (stat.isDirectory()) {
+    if (lstat.isDirectory()) {
       if (!filter(absolutePath, true)) continue;
       results.push(...walkDir(absolutePath, syncRoot, filter));
-    } else if (stat.isFile()) {
+    } else if (lstat.isFile()) {
       const relativePath = path.relative(syncRoot, absolutePath);
       if (filter(absolutePath)) {
-        results.push({ absolutePath, relativePath });
+        results.push({ kind: "file", absolutePath, relativePath });
       }
     }
   }
@@ -625,14 +752,43 @@ function walkDir(
   dir: string,
   syncRoot: string,
   filter: (p: string, isDir?: boolean) => boolean,
-): { absolutePath: string; relativePath: string }[] {
-  const results: { absolutePath: string; relativePath: string }[] = [];
+): CollectedEntry[] {
+  const results: CollectedEntry[] = [];
   if (!fs.existsSync(dir)) return results;
 
   const entries = fs.readdirSync(dir, { withFileTypes: true });
   for (const entry of entries) {
     const absolutePath = path.join(dir, entry.name);
     const isDir = entry.isDirectory();
+
+    // Symlinks need their own filter probe BEFORE the regular gate.
+    // Dirent.isDirectory() returns false for any symlink — even a
+    // directory symlink — so the regular filter call below would use
+    // isDir=false and a dir-only allowlist pattern like
+    // `companies/*/knowledge/` would reject the link before the
+    // record-only branch runs. Probe with both hints; include if
+    // either matches. The filter is pure path lookup with no I/O.
+    if (entry.isSymbolicLink()) {
+      if (!filter(absolutePath, false) && !filter(absolutePath, true)) continue;
+      // Record the link without descending into its target. Following
+      // a directory symlink would re-enter content via a path that
+      // isn't its on-disk home (e.g. companies/{co}/knowledge → repos/
+      // private/knowledge-{co}/), causing per-company knowledge repos
+      // to be uploaded into every vault that links them. Recording
+      // and not following preserves the link topology while avoiding
+      // that duplication. readlinkSync on a Dirent-known link cannot
+      // fail under normal conditions; let the throw propagate if it
+      // somehow does (race with rm, EPERM) — the operator needs to
+      // see it rather than us silently dropping the link again.
+      results.push({
+        kind: "symlink",
+        absolutePath,
+        relativePath: path.relative(syncRoot, absolutePath),
+        target: fs.readlinkSync(absolutePath),
+      });
+      continue;
+    }
+
     // Pass the dir hint so dir-only ignore/include patterns (`foo/`)
     // resolve correctly for the descent decision.
     if (!filter(absolutePath, isDir)) continue;
@@ -641,6 +797,7 @@ function walkDir(
       results.push(...walkDir(absolutePath, syncRoot, filter));
     } else if (entry.isFile()) {
       results.push({
+        kind: "file",
         absolutePath,
         relativePath: path.relative(syncRoot, absolutePath),
       });
@@ -670,6 +827,28 @@ function realpathSafe(p: string): string {
   }
 }
 
+/**
+ * Containment check tailored for symlinks. Canonicalizes the link's
+ * PARENT DIR (which is a real dir, not the link), then compares the
+ * recombined `parentReal/basename(linkPath)` against `parent`. Skipping
+ * the link's own canonicalization means a symlink that points outside
+ * `parent` is still considered "inside" so long as the link file itself
+ * lives inside — which is exactly the topology we want to upload as a
+ * link record without dereferencing.
+ *
+ * Falls back to `parent` / `path.dirname(linkPath)` literally when
+ * realpath throws (e.g. permission denied on a parent), trading a tiny
+ * window of macOS-APFS case-sensitivity drift for the more common case
+ * of "link lives inside, target lives outside."
+ */
+function isWithinForLink(parent: string, linkPath: string): boolean {
+  const parentReal = realpathSafe(parent);
+  const linkParentReal = realpathSafe(path.dirname(linkPath));
+  const candidate = path.join(linkParentReal, path.basename(linkPath));
+  const rel = path.relative(parentReal, candidate);
+  return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
+
 /**
  * Returns true when the remote object appears to have moved since the
  * journal entry's last-recorded sync. Prefers ETag equality; falls back to
@@ -760,11 +939,45 @@ function computeDeletePlan(
     );
     if (!inScope) continue;
     const localPath = path.join(syncRoot, relativeKey);
-    if (fs.existsSync(localPath)) continue;
+    // lstat (not existsSync) so a dangling symlink — a link whose
+    // target has been removed but whose link file is still on disk —
+    // counts as "still present locally" and is NOT delete-propagated.
+    // Pre-fix, existsSync followed the link, returned false, and the
+    // entry was queued for remote DeleteObject in the same sync that
+    // had just uploaded it via uploadSymlink. The link round-tripped
+    // as "upload, then delete" in one cycle. ENOENT means truly
+    // absent → eligible; other lstat errors propagate.
+    let presentLocally = true;
+    try {
+      fs.lstatSync(localPath);
+    } catch (err: unknown) {
+      if (
+        err &&
+        typeof err === "object" &&
+        "code" in err &&
+        (err as { code?: string }).code === "ENOENT"
+      ) {
+        presentLocally = false;
+      } else {
+        throw err;
+      }
+    }
+    if (presentLocally) continue;
     // (3) Symmetric filter guard. `shouldSync` is constructed from the same
     //     hqRoot the pull leg uses, so a key the pull would have skipped
     //     ("ignored") is also one we must not delete-propagate.
-    if (!shouldSync(localPath, false)) continue;
+    //
+    //     Dual-hint probe: by the time we're considering this entry for
+    //     remote deletion, the local file is already gone — we have no
+    //     way to know whether it was a regular file or a symlink record.
+    //     A single isDir=false probe would silently keep the remote
+    //     record alive whenever the only matching .hqinclude allowlist
+    //     pattern is dir-only (e.g. `companies/*/knowledge/`), since
+    //     gitignore's slash semantics reject the slashless probe. The
+    //     same dual-hint pattern in walkDir/collectFiles (push) and
+    //     computePullPlan (pull) applies symmetrically here. Pure path
+    //     lookup, no I/O.
+    if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) continue;
     // (4) Direction guard under "owned-only" policy.
     if (policy === "owned-only" && entry.direction !== "up") continue;
     out.push(relativeKey);