@indigoai-us/hq-cloud 5.16.0 → 5.18.1

package/src/cli/share.ts

@@ -9,9 +9,17 @@ import * as fs from "fs";
 import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
-import { uploadFile, headRemoteFile, deleteRemoteFile } from "../s3.js";
+import { uploadFile, uploadSymlink, headRemoteFile, deleteRemoteFile } from "../s3.js";
 import type { UploadAuthor } from "../s3.js";
-import { readJournal, writeJournal, hashFile, updateEntry, removeEntry, normalizeEtag } from "../journal.js";
+import {
+  readJournal,
+  writeJournal,
+  hashFile,
+  hashSymlinkTarget,
+  updateEntry,
+  removeEntry,
+  normalizeEtag,
+} from "../journal.js";
 import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy } from "./conflict.js";
@@ -30,11 +38,24 @@ import type { SyncProgressEvent } from "./sync.js";
 type PushPlanItem =
   | {
       action: "upload";
+      kind: "file";
       absolutePath: string;
       relativePath: string;
       localHash: string;
       size: number;
     }
+  | {
+      action: "upload";
+      kind: "symlink";
+      absolutePath: string;
+      relativePath: string;
+      // The link's target string verbatim (whatever readlink returned).
+      // Hashed into localHash so a target rewrite re-uploads even when
+      // skipUnchanged is on; size stays 0 because the wire body is empty.
+      target: string;
+      localHash: string;
+      size: 0;
+    }
   | {
       action: "skip-size-limit";
       absolutePath: string;
@@ -64,13 +85,46 @@ interface PushPlan {
  * Consumers that want a conflict count get it from the `complete` event.
  */
 function computePushPlan(
-  filesToShare: { absolutePath: string; relativePath: string }[],
+  filesToShare: CollectedEntry[],
   journal: SyncJournal,
   skipUnchanged: boolean,
 ): PushPlan {
   const items: PushPlanItem[] = [];
 
-  for (const { absolutePath, relativePath } of filesToShare) {
+  for (const entry of filesToShare) {
+    const { absolutePath, relativePath } = entry;
+
+    // Symlinks bypass the size-limit gate (the wire body is small,
+    // bounded by target length) and hash the target through the
+    // symlink-namespaced hash so a symlink to "real.md" can never
+    // collide with a regular file containing the bytes "real.md" in
+    // the skip-unchanged gate. The target is what we're actually
+    // uploading, so its hash is what should drive change detection —
+    // a target rewrite must re-fire an upload even when the link
+    // itself "looks" identical.
+    if (entry.kind === "symlink") {
+      const localHash = hashSymlinkTarget(entry.target);
+
+      if (skipUnchanged) {
+        const existing = journal.files[relativePath];
+        if (existing && existing.hash === localHash) {
+          items.push({ action: "skip-unchanged", absolutePath, relativePath });
+          continue;
+        }
+      }
+
+      items.push({
+        action: "upload",
+        kind: "symlink",
+        absolutePath,
+        relativePath,
+        target: entry.target,
+        localHash,
+        size: 0,
+      });
+      continue;
+    }
+
     if (!isWithinSizeLimit(absolutePath)) {
       items.push({ action: "skip-size-limit", absolutePath, relativePath });
       continue;
@@ -89,6 +143,7 @@ function computePushPlan(
     const size = fs.statSync(absolutePath).size;
     items.push({
       action: "upload",
+      kind: "file",
       absolutePath,
       relativePath,
       localHash,
@@ -185,6 +240,34 @@ export interface ShareOptions {
    * full-tree bidirectional runner opts in.
    */
   propagateDeletes?: boolean;
+  /**
+   * Policy for which journal entries `propagateDeletes` is willing to
+   * convert into remote `DeleteObject` calls. Only consulted when
+   * `propagateDeletes === true`.
+   *
+   *   - `"owned-only"` (default, safer): only entries whose journal
+   *     `direction === "up"` are eligible. That is, only files this
+   *     machine previously uploaded can be remotely deleted on its
+   *     behalf. Entries the journal records as pulled from elsewhere
+   *     (`direction === "down"`) are never delete-propagated — the
+   *     local absence may just be an unpulled state or a filter
+   *     mismatch, both of which previously caused this machine to
+   *     erase other machines' uploads.
+   *   - `"all"`: legacy behaviour — every in-scope journal entry whose
+   *     local file is missing is eligible (regardless of direction). The
+   *     bidirectional runner's first-push and any tool that wants to
+   *     mirror a destructive local checkout opts in here explicitly.
+   *
+   * Independently of this policy, an entry is also dropped from the
+   * plan when neither the file-shape nor the directory-shape probe of
+   * `shouldSync` accepts the path — i.e. the current ignore filter
+   * would have skipped the path on pull (whether classified as a
+   * regular file or a symlink record / directory). That symmetry
+   * blocks the failure mode where a path was filtered locally but
+   * lived in the vault (and the journal) from an older HQ layout or
+   * a different machine, causing the next push to erase it.
+   */
+  propagateDeletePolicy?: "owned-only" | "all";
   /**
    * Identity stamped onto each uploaded object's S3 user metadata
    * (`created-by`, `created-by-sub`, `created-at`). The hq-console vault UI
@@ -236,6 +319,14 @@ export interface ShareResult {
  */
 export async function share(options: ShareOptions): Promise<ShareResult> {
   const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged, propagateDeletes } = options;
+  // Default to the safer "owned-only" policy when delete-propagation is on
+  // but the caller hasn't pinned a policy. Pre-existing callers that passed
+  // `propagateDeletes: true` (the `sync now` push leg, the runner's
+  // bidirectional sync, the `--all` fanout) thereby flip to the safer
+  // semantics automatically. Set `propagateDeletePolicy: "all"` explicitly
+  // to opt back into the legacy any-missing-file-deletes behaviour.
+  const propagateDeletePolicy: "owned-only" | "all" =
+    options.propagateDeletePolicy ?? "owned-only";
   const emit = options.onEvent ?? defaultConsoleLogger;
 
   // Exactly-one-of contract: either we vend (vaultConfig) or the caller did
@@ -316,7 +407,13 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     ? resolveDeleteScopeRoots(paths, hqRoot, syncRoot)
     : [];
   const deletePlan = propagateDeletes === true
-    ? computeDeletePlan(journal, syncRoot, deleteScopeRoots)
+    ? computeDeletePlan(
+        journal,
+        syncRoot,
+        deleteScopeRoots,
+        shouldSync,
+        propagateDeletePolicy,
+      )
     : [];
 
   emit({
@@ -413,18 +510,26 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
       }
     }
 
-    // Upload
+    // Upload — symlinks go through uploadSymlink (zero-byte body + target
+    // metadata), regular files through uploadFile (file contents). The
+    // discriminator is item.kind set by computePushPlan; both branches
+    // converge on the same journal/event update path below.
     try {
-      const stat = fs.statSync(absolutePath);
+      const isSymlinkUpload = item.kind === "symlink";
+      const size = isSymlinkUpload ? 0 : fs.statSync(absolutePath).size;
 
-      const { etag } = options.author
-        ? await uploadFile(ctx, absolutePath, relativePath, options.author)
-        : await uploadFile(ctx, absolutePath, relativePath);
+      const { etag } = isSymlinkUpload
+        ? options.author
+          ? await uploadSymlink(ctx, item.target, relativePath, options.author)
+          : await uploadSymlink(ctx, item.target, relativePath)
+        : options.author
+          ? await uploadFile(ctx, absolutePath, relativePath, options.author)
+          : await uploadFile(ctx, absolutePath, relativePath);
 
       // Update journal with optional message; capture the post-upload ETag
       // so the next sync can distinguish "remote moved since we last wrote"
       // from "user edited locally" without conflating the two.
-      updateEntry(journal, relativePath, localHash, stat.size, "up", etag);
+      updateEntry(journal, relativePath, localHash, size, "up", etag);
       if (message) {
         journal.files[relativePath] = {
           ...journal.files[relativePath],
@@ -433,11 +538,11 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
       }
 
       filesUploaded++;
-      bytesUploaded += stat.size;
+      bytesUploaded += size;
       emit({
         type: "progress",
         path: relativePath,
-        bytes: stat.size,
+        bytes: size,
         ...(message ? { message } : {}),
       });
     } catch (err) {
@@ -538,42 +643,104 @@ function resolveActiveCompany(hqRoot: string): string | undefined {
   return undefined;
 }
 
+/**
+ * One entry produced by collectFiles/walkDir. Files describe regular
+ * payloads that get hashed + size-checked + uploaded via uploadFile;
+ * symlinks describe link records whose target string flows through
+ * uploadSymlink as user metadata. Walked-dir traversal NEVER descends
+ * into a symlink — directory symlinks are recorded as link entries and
+ * left at that, so following them never duplicates content into the
+ * wrong vault path (the same topology safety the legacy walker provided
+ * by accident of Dirent.isFile() returning false for links).
+ */
+type CollectedEntry =
+  | { kind: "file"; absolutePath: string; relativePath: string }
+  | { kind: "symlink"; absolutePath: string; relativePath: string; target: string };
+
 /**
  * Collect files from paths (expanding directories recursively).
  *
  * Remote S3 keys are computed relative to `syncRoot` (companies/{slug}/), not
  * `hqRoot`. Files outside `syncRoot` are skipped with a warning — sharing
  * anything outside a company's folder would leak state into the wrong vault.
+ *
+ * Symlink classification uses lstat (not stat), so a top-level path that is
+ * itself a symlink is recorded as a link record rather than dereferenced.
+ * Pre-fix, statSync followed the link and the target's bytes were uploaded
+ * under the link's key — silently flattening the link topology.
  */
 function collectFiles(
   paths: string[],
   hqRoot: string,
   syncRoot: string,
   filter: (p: string, isDir?: boolean) => boolean,
-): { absolutePath: string; relativePath: string }[] {
-  const results: { absolutePath: string; relativePath: string }[] = [];
+): CollectedEntry[] {
+  const results: CollectedEntry[] = [];
 
   for (const p of paths) {
     const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
 
-    if (!fs.existsSync(absolutePath)) {
+    // existsSync follows symlinks: a dangling top-level link will report
+    // not-existing and be skipped here. lstatSync below handles the
+    // valid-link case directly without needing the existsSync gate.
+    let lstat: fs.Stats;
+    try {
+      lstat = fs.lstatSync(absolutePath);
+    } catch {
       console.error(`  Warning: ${p} does not exist, skipping.`);
       continue;
     }
 
+    // Containment check is split by entry kind: regular files and
+    // directories use isWithin (which canonicalizes via realpathSync to
+    // tolerate macOS APFS case-insensitivity), but symlinks use the
+    // link's own pathname rather than the link's resolved target. A
+    // valid use case for this asymmetry: a directory symlink whose
+    // target lives outside the company folder (e.g. companies/{co}/
+    // knowledge → repos/private/knowledge-{co}/) — the LINK itself is
+    // inside the company folder and is exactly what we want to record,
+    // but isWithin's realpath would say the resolved target is outside
+    // and reject the share. Recording symlinks rather than following
+    // them is the whole topology contract this fix establishes; the
+    // containment check needs to honor the same semantic.
+    if (lstat.isSymbolicLink()) {
+      if (!isWithinForLink(syncRoot, absolutePath)) {
+        console.error(`  Warning: ${p} is outside company folder, skipping.`);
+        continue;
+      }
+      const relativePath = path.relative(syncRoot, absolutePath);
+      // Probe the filter with both isDir hints — we don't know whether
+      // the link's target is a file or a directory without
+      // stat-following the link, which we explicitly avoid (it would
+      // re-introduce the dereference behavior this whole change set is
+      // designed to prevent). An `.hqinclude` dir-only pattern like
+      // `companies/*/knowledge/` only matches with isDir=true, so a
+      // single isDir=false probe would silently drop directory
+      // symlinks under allowlist mode (the motivating case for this
+      // whole branch). The filter is pure path lookup with no I/O,
+      // so two calls are free.
+      if (!filter(absolutePath, false) && !filter(absolutePath, true)) continue;
+      results.push({
+        kind: "symlink",
+        absolutePath,
+        relativePath,
+        target: fs.readlinkSync(absolutePath),
+      });
+      continue;
+    }
+
     if (!isWithin(syncRoot, absolutePath)) {
       console.error(`  Warning: ${p} is outside company folder, skipping.`);
       continue;
     }
 
-    const stat = fs.statSync(absolutePath);
-    if (stat.isDirectory()) {
+    if (lstat.isDirectory()) {
       if (!filter(absolutePath, true)) continue;
       results.push(...walkDir(absolutePath, syncRoot, filter));
-    } else if (stat.isFile()) {
+    } else if (lstat.isFile()) {
       const relativePath = path.relative(syncRoot, absolutePath);
       if (filter(absolutePath)) {
-        results.push({ absolutePath, relativePath });
+        results.push({ kind: "file", absolutePath, relativePath });
       }
     }
   }
@@ -585,14 +752,43 @@ function walkDir(
   dir: string,
   syncRoot: string,
   filter: (p: string, isDir?: boolean) => boolean,
-): { absolutePath: string; relativePath: string }[] {
-  const results: { absolutePath: string; relativePath: string }[] = [];
+): CollectedEntry[] {
+  const results: CollectedEntry[] = [];
   if (!fs.existsSync(dir)) return results;
 
   const entries = fs.readdirSync(dir, { withFileTypes: true });
   for (const entry of entries) {
     const absolutePath = path.join(dir, entry.name);
     const isDir = entry.isDirectory();
+
+    // Symlinks need their own filter probe BEFORE the regular gate.
+    // Dirent.isDirectory() returns false for any symlink — even a
+    // directory symlink — so the regular filter call below would use
+    // isDir=false and a dir-only allowlist pattern like
+    // `companies/*/knowledge/` would reject the link before the
+    // record-only branch runs. Probe with both hints; include if
+    // either matches. The filter is pure path lookup with no I/O.
+    if (entry.isSymbolicLink()) {
+      if (!filter(absolutePath, false) && !filter(absolutePath, true)) continue;
+      // Record the link without descending into its target. Following
+      // a directory symlink would re-enter content via a path that
+      // isn't its on-disk home (e.g. companies/{co}/knowledge → repos/
+      // private/knowledge-{co}/), causing per-company knowledge repos
+      // to be uploaded into every vault that links them. Recording
+      // and not following preserves the link topology while avoiding
+      // that duplication. readlinkSync on a Dirent-known link cannot
+      // fail under normal conditions; let the throw propagate if it
+      // somehow does (race with rm, EPERM) — the operator needs to
+      // see it rather than us silently dropping the link again.
+      results.push({
+        kind: "symlink",
+        absolutePath,
+        relativePath: path.relative(syncRoot, absolutePath),
+        target: fs.readlinkSync(absolutePath),
+      });
+      continue;
+    }
+
     // Pass the dir hint so dir-only ignore/include patterns (`foo/`)
     // resolve correctly for the descent decision.
     if (!filter(absolutePath, isDir)) continue;
@@ -601,6 +797,7 @@ function walkDir(
       results.push(...walkDir(absolutePath, syncRoot, filter));
     } else if (entry.isFile()) {
       results.push({
+        kind: "file",
         absolutePath,
         relativePath: path.relative(syncRoot, absolutePath),
       });
@@ -630,6 +827,28 @@ function realpathSafe(p: string): string {
   }
 }
 
+/**
+ * Containment check tailored for symlinks. Canonicalizes the link's
+ * PARENT DIR (which is a real dir, not the link), then compares the
+ * recombined `parentReal/basename(linkPath)` against `parent`. Skipping
+ * the link's own canonicalization means a symlink that points outside
+ * `parent` is still considered "inside" so long as the link file itself
+ * lives inside — which is exactly the topology we want to upload as a
+ * link record without dereferencing.
+ *
+ * Falls back to `parent` / `path.dirname(linkPath)` literally when
+ * realpath throws (e.g. permission denied on a parent), trading a tiny
+ * window of macOS-APFS case-sensitivity drift for the more common case
+ * of "link lives inside, target lives outside."
+ */
+function isWithinForLink(parent: string, linkPath: string): boolean {
+  const parentReal = realpathSafe(parent);
+  const linkParentReal = realpathSafe(path.dirname(linkPath));
+  const candidate = path.join(linkParentReal, path.basename(linkPath));
+  const rel = path.relative(parentReal, candidate);
+  return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
+
 /**
  * Returns true when the remote object appears to have moved since the
  * journal entry's last-recorded sync. Prefers ETag equality; falls back to
@@ -681,18 +900,37 @@ function resolveDeleteScopeRoots(
 
 /**
  * Walk every journal key in `scopeRoots` whose local file is missing from
- * disk, and return the keys to delete. A key is in-scope when it matches
- * (or sits beneath) one of the resolved prefixes. Empty `scopeRoots` ⇒
- * empty plan (caller didn't opt in).
+ * disk and return the keys eligible for a remote `DeleteObject`. An entry
+ * is in the plan only when ALL of the following hold:
+ *
+ *   1. Its key matches (or sits beneath) one of the `scopeRoots` prefixes.
+ *   2. Its local file is missing from disk.
+ *   3. The current ignore filter (`shouldSync`) accepts the key — so paths
+ *      filtered out by `.hqignore` / `.gitignore` / `DEFAULT_IGNORES` are
+ *      never delete-propagated. This blocks the failure mode where a path
+ *      lives in the vault (and the journal) but the local walk skips it
+ *      because of asymmetric ignore rules; without this guard the push
+ *      leg would erase it.
+ *   4. When `policy === "owned-only"`: the journal entry's `direction`
+ *      is `"up"` (i.e. this machine previously uploaded the file). This
+ *      blocks the failure mode where a behind machine's first `sync now`
+ *      push leg would otherwise erase recent uploads from peers, since
+ *      those entries are recorded as `direction: "down"` (pulled) or
+ *      absent (never seen). Set `policy: "all"` to opt back into the
+ *      legacy any-missing-file-deletes behaviour.
+ *
+ * Empty `scopeRoots` ⇒ empty plan (caller didn't opt in).
  */
 function computeDeletePlan(
   journal: SyncJournal,
   syncRoot: string,
   scopeRoots: string[],
+  shouldSync: (filePath: string, isDir?: boolean) => boolean,
+  policy: "owned-only" | "all",
 ): string[] {
   if (scopeRoots.length === 0) return [];
   const out: string[] = [];
-  for (const relativeKey of Object.keys(journal.files)) {
+  for (const [relativeKey, entry] of Object.entries(journal.files)) {
     const inScope = scopeRoots.some(
       (root) =>
         root === "" ||
@@ -701,9 +939,48 @@ function computeDeletePlan(
     );
     if (!inScope) continue;
     const localPath = path.join(syncRoot, relativeKey);
-    if (!fs.existsSync(localPath)) {
-      out.push(relativeKey);
+    // lstat (not existsSync) so a dangling symlink — a link whose
+    // target has been removed but whose link file is still on disk —
+    // counts as "still present locally" and is NOT delete-propagated.
+    // Pre-fix, existsSync followed the link, returned false, and the
+    // entry was queued for remote DeleteObject in the same sync that
+    // had just uploaded it via uploadSymlink. The link round-tripped
+    // as "upload, then delete" in one cycle. ENOENT means truly
+    // absent → eligible; other lstat errors propagate.
+    let presentLocally = true;
+    try {
+      fs.lstatSync(localPath);
+    } catch (err: unknown) {
+      if (
+        err &&
+        typeof err === "object" &&
+        "code" in err &&
+        (err as { code?: string }).code === "ENOENT"
+      ) {
+        presentLocally = false;
+      } else {
+        throw err;
+      }
     }
+    if (presentLocally) continue;
+    // (3) Symmetric filter guard. `shouldSync` is constructed from the same
+    //     hqRoot the pull leg uses, so a key the pull would have skipped
+    //     ("ignored") is also one we must not delete-propagate.
+    //
+    //     Dual-hint probe: by the time we're considering this entry for
+    //     remote deletion, the local file is already gone — we have no
+    //     way to know whether it was a regular file or a symlink record.
+    //     A single isDir=false probe would silently keep the remote
+    //     record alive whenever the only matching .hqinclude allowlist
+    //     pattern is dir-only (e.g. `companies/*/knowledge/`), since
+    //     gitignore's slash semantics reject the slashless probe. The
+    //     same dual-hint pattern in walkDir/collectFiles (push) and
+    //     computePullPlan (pull) applies symmetrically here. Pure path
+    //     lookup, no I/O.
+    if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) continue;
+    // (4) Direction guard under "owned-only" policy.
+    if (policy === "owned-only" && entry.direction !== "up") continue;
+    out.push(relativeKey);
   }
   return out;
 }