@indigoai-us/hq-cloud 5.10.1 → 5.11.0

package/src/cli/share.test.ts

@@ -21,7 +21,7 @@ vi.mock("../s3.js", () => ({
 }));
 
 import { share } from "./share.js";
-import { headRemoteFile, uploadFile } from "../s3.js";
+import { deleteRemoteFile, headRemoteFile, uploadFile } from "../s3.js";
 import type { EntityContext } from "../types.js";
 
 const mockConfig: VaultServiceConfig = {
@@ -797,4 +797,241 @@ describe("share", () => {
       filesToSkip: 1,
     });
   });
+
+  // ── Delete propagation (propagateDeletes) ──────────────────────────────────
+  //
+  // The bug: when a user deletes a local file, the next pull re-downloads it
+  // from S3 because the remote object is still listable and the pull plan
+  // can't tell "never synced" from "synced then deleted". The fix is to
+  // propagate local deletes to S3 on the push side. The vault buckets have
+  // versioning enabled, so DeleteObject is soft (a delete-marker becomes the
+  // current version; prior object versions remain recoverable).
+
+  it("propagateDeletes: deletes journal-tracked files whose local copy is gone", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    // Only "kept.md" exists locally; "gone.md" was previously synced and then
+    // deleted by the user.
+    fs.writeFileSync(path.join(companyRoot, "kept.md"), "still here");
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "kept.md": {
+            hash: "irrelevant-not-checked-here",
+            size: 10,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            remoteEtag: "kept-etag",
+          },
+          "gone.md": {
+            hash: "irrelevant-not-checked-here",
+            size: 7,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            remoteEtag: "gone-etag",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+    });
+
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledTimes(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "gone.md");
+
+    // Journal entry for the gone file is removed; the kept entry stays.
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["gone.md"]).toBeUndefined();
+    expect(journal.files["kept.md"]).toBeDefined();
+  });
+
+  it("propagateDeletes: emits a `progress` event with deleted:true and bytes from the journal", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "removed.md": {
+            hash: "h",
+            size: 42,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    const events: Array<{ type: string; path?: string; bytes?: number; deleted?: boolean }> = [];
+    await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+      onEvent: (e) => events.push(e as { type: string }),
+    });
+
+    const planEvent = events.find((e) => e.type === "plan") as { filesToDelete?: number } | undefined;
+    expect(planEvent?.filesToDelete).toBe(1);
+
+    const deleteProgress = events.find(
+      (e) => e.type === "progress" && e.deleted === true,
+    );
+    expect(deleteProgress).toMatchObject({
+      type: "progress",
+      path: "removed.md",
+      bytes: 42,
+      deleted: true,
+    });
+  });
+
+  it("propagateDeletes=false (default): missing local files do NOT trigger a remote delete", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "gone.md": {
+            hash: "h",
+            size: 7,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      // propagateDeletes omitted ⇒ defaults to false
+    });
+
+    expect(result.filesDeleted).toBe(0);
+    expect(deleteRemoteFile).not.toHaveBeenCalled();
+    // Journal entry survives so the next opt-in run can still propagate.
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["gone.md"]).toBeDefined();
+  });
+
+  it("propagateDeletes: scope is constrained to the supplied paths — sibling deletes are not swept", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(path.join(companyRoot, "in-scope"), { recursive: true });
+    fs.mkdirSync(path.join(companyRoot, "other"), { recursive: true });
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "in-scope/gone.md": {
+            hash: "h",
+            size: 1,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+          "other/also-gone.md": {
+            hash: "h",
+            size: 1,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [path.join(companyRoot, "in-scope")],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+    });
+
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledTimes(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(
+      expect.anything(),
+      "in-scope/gone.md",
+    );
+
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["in-scope/gone.md"]).toBeUndefined();
+    // Sibling tree's journal entry is untouched — `hq share <subtree>` must
+    // not act on files outside the named scope.
+    expect(journal.files["other/also-gone.md"]).toBeDefined();
+  });
+
+  it("propagateDeletes: a failed DeleteObject leaves the journal entry intact for retry", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "flaky.md": {
+            hash: "h",
+            size: 5,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    vi.mocked(deleteRemoteFile).mockRejectedValueOnce(new Error("S3 down"));
+
+    const events: Array<{ type: string; path?: string; message?: string }> = [];
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+      onEvent: (e) => events.push(e as { type: string }),
+    });
+
+    expect(result.filesDeleted).toBe(0);
+    const errorEvent = events.find((e) => e.type === "error");
+    expect(errorEvent).toMatchObject({ path: "flaky.md", message: expect.stringContaining("S3 down") });
+
+    // Entry survives — next run will retry the delete.
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["flaky.md"]).toBeDefined();
+  });
 });

package/src/cli/share.ts

@@ -9,8 +9,8 @@ import * as fs from "fs";
 import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
-import { uploadFile, headRemoteFile } from "../s3.js";
-import { readJournal, writeJournal, hashFile, updateEntry, normalizeEtag } from "../journal.js";
+import { uploadFile, headRemoteFile, deleteRemoteFile } from "../s3.js";
+import { readJournal, writeJournal, hashFile, updateEntry, removeEntry, normalizeEtag } from "../journal.js";
 import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy } from "./conflict.js";
@@ -166,12 +166,36 @@ export interface ShareOptions {
    * hash matches the last-sync state (e.g. to re-heal a bucket).
    */
   skipUnchanged?: boolean;
+  /**
+   * When true, journal entries whose local file is gone trigger a remote
+   * `DeleteObject`. Only entries whose key falls under one of the supplied
+   * `paths` (after resolution to absolute paths under `companies/{slug}/`)
+   * are considered, so `hq share <file>` can never sweep deletes outside
+   * the named scope.
+   *
+   * Vault buckets have versioning enabled, so the delete is soft: a
+   * delete-marker becomes the current version and prior object versions
+   * remain recoverable indefinitely. The pull-side `listRemoteFiles` skips
+   * objects whose current version is a delete-marker (default
+   * `ListObjectsV2` behavior), so a deletion stops the next pull from
+   * re-downloading the file on this and any other machine.
+   *
+   * Default false to preserve `hq share <file>` semantics — only the
+   * full-tree bidirectional runner opts in.
+   */
+  propagateDeletes?: boolean;
 }
 
 export interface ShareResult {
   filesUploaded: number;
   bytesUploaded: number;
   filesSkipped: number;
+  /**
+   * Number of remote `DeleteObject` calls that succeeded this run. Always 0
+   * when `propagateDeletes` is false. The corresponding journal entries are
+   * removed in the same pass so the next sync sees the key as truly gone.
+   */
+  filesDeleted: number;
   /**
    * Paths (company-relative) that were detected as push conflicts. Mirrors
    * `SyncResult.conflictPaths` so push and pull surface conflicts the same
@@ -185,7 +209,7 @@ export interface ShareResult {
  * Share local file(s) to the entity vault.
  */
 export async function share(options: ShareOptions): Promise<ShareResult> {
-  const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged } = options;
+  const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged, propagateDeletes } = options;
   const emit = options.onEvent ?? defaultConsoleLogger;
 
   // Exactly-one-of contract: either we vend (vaultConfig) or the caller did
@@ -237,6 +261,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
   let filesUploaded = 0;
   let bytesUploaded = 0;
   let filesSkipped = 0;
+  let filesDeleted = 0;
   const conflictPaths: string[] = [];
 
   // Collect all files to share
@@ -249,6 +274,17 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
   // knowable here). The final `complete` event reports authoritative counts.
   const plan = computePushPlan(filesToShare, journal, skipUnchanged === true);
 
+  // Delete-propagation plan: walk journal entries whose key falls under the
+  // requested scope; any whose local file is gone is a candidate for a
+  // remote DeleteObject. Only computed when the caller opts in — `hq share
+  // <file>` must never sweep deletes outside the explicit path list.
+  const deleteScopeRoots = propagateDeletes === true
+    ? resolveDeleteScopeRoots(paths, hqRoot, syncRoot)
+    : [];
+  const deletePlan = propagateDeletes === true
+    ? computeDeletePlan(journal, syncRoot, deleteScopeRoots)
+    : [];
+
   emit({
     type: "plan",
     // share() is push-only; pull counts are sourced from sync()'s plan event.
@@ -260,6 +296,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     // Push conflicts require a remote HEAD; we don't yet do that in Stage 1,
     // so this stays 0. V1.5 (single LIST) will let us classify them up-front.
     filesToConflict: 0,
+    filesToDelete: deletePlan.length,
   });
 
   // Stage 2: execute. Skip items pre-classified as no-ops, then for each
@@ -329,6 +366,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
             filesUploaded,
             bytesUploaded,
             filesSkipped,
+            filesDeleted,
             conflictPaths,
             aborted: true,
           };
@@ -375,6 +413,36 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     }
   }
 
+  // Stage 3: propagate deletes. Each call writes a delete-marker (versioning
+  // is enabled on the bucket) and removes the corresponding journal entry so
+  // the next sync sees the key as truly gone on this machine. A failed
+  // DeleteObject leaves both the journal entry and the remote object intact
+  // — the next run will retry.
+  for (const relativePath of deletePlan) {
+    if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
+      ctx = await refreshEntityContext(companyRef, vaultConfig);
+    }
+    try {
+      const entry = journal.files[relativePath];
+      const size = entry?.size ?? 0;
+      await deleteRemoteFile(ctx, relativePath);
+      removeEntry(journal, relativePath);
+      filesDeleted++;
+      emit({
+        type: "progress",
+        path: relativePath,
+        bytes: size,
+        deleted: true,
+      });
+    } catch (err) {
+      emit({
+        type: "error",
+        path: relativePath,
+        message: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
   // See cli/sync.ts: stamp lastSync on completion so a no-op share still
   // ticks the "Last sync" indicator.
   journal.lastSync = new Date().toISOString();
@@ -384,6 +452,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     filesUploaded,
     bytesUploaded,
     filesSkipped,
+    filesDeleted,
     conflictPaths,
     aborted: false,
   };
@@ -401,7 +470,9 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
       );
     }
   } else if (event.type === "progress") {
-    if (event.message) {
+    if (event.deleted) {
+      console.log(`  ✗ ${event.path} (deleted)`);
+    } else if (event.message) {
       console.log(`  ✓ ${event.path} — "${event.message}"`);
     } else {
       console.log(`  ✓ ${event.path}`);
@@ -536,3 +607,63 @@ function hasRemoteChanged(
   const syncedAt = new Date(entry.syncedAt).getTime();
   return remote.lastModified.getTime() > syncedAt;
 }
+
+/**
+ * Resolve each user-supplied share path to a directory under `syncRoot`,
+ * returning the company-relative prefix that constrains delete propagation.
+ * Files (non-directories) and paths outside the company root are dropped —
+ * a delete sweep keyed off a single file or a sibling tree would surprise
+ * users who expected deletes to mirror the targeted scope.
+ *
+ * Returns `[""]` (whole-tree) when any input path resolves to `syncRoot`
+ * itself; this is the bidirectional-runner case.
+ */
+function resolveDeleteScopeRoots(
+  paths: string[],
+  hqRoot: string,
+  syncRoot: string,
+): string[] {
+  const prefixes = new Set<string>();
+  for (const p of paths) {
+    const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
+    if (!fs.existsSync(absolutePath)) continue;
+    if (!isWithin(syncRoot, absolutePath)) continue;
+    const stat = fs.statSync(absolutePath);
+    if (!stat.isDirectory()) continue;
+    const rel = path.relative(syncRoot, absolutePath);
+    if (rel === "" || rel === ".") {
+      return [""];
+    }
+    prefixes.add(rel.split(path.sep).join("/"));
+  }
+  return Array.from(prefixes);
+}
+
+/**
+ * Walk every journal key in `scopeRoots` whose local file is missing from
+ * disk, and return the keys to delete. A key is in-scope when it matches
+ * (or sits beneath) one of the resolved prefixes. Empty `scopeRoots` ⇒
+ * empty plan (caller didn't opt in).
+ */
+function computeDeletePlan(
+  journal: SyncJournal,
+  syncRoot: string,
+  scopeRoots: string[],
+): string[] {
+  if (scopeRoots.length === 0) return [];
+  const out: string[] = [];
+  for (const relativeKey of Object.keys(journal.files)) {
+    const inScope = scopeRoots.some(
+      (root) =>
+        root === "" ||
+        relativeKey === root ||
+        relativeKey.startsWith(`${root}/`),
+    );
+    if (!inScope) continue;
+    const localPath = path.join(syncRoot, relativeKey);
+    if (!fs.existsSync(localPath)) {
+      out.push(relativeKey);
+    }
+  }
+  return out;
+}

package/src/cli/sync.ts

@@ -56,8 +56,24 @@ export type SyncProgressEvent =
        * conflict detection requires a remote HEAD that runs in Stage 2.
        */
       filesToConflict: number;
+      /**
+       * Remote keys this run intends to delete from S3 (push-only with
+       * `propagateDeletes`; 0 from sync). A key is scheduled for deletion when
+       * its journal entry exists, the local file is gone, and the key falls
+       * within the share scope. The bucket has versioning enabled so the
+       * delete is soft (a delete-marker is written; prior versions remain
+       * recoverable).
+       */
+      filesToDelete: number;
+    }
+  | {
+      type: "progress";
+      path: string;
+      bytes: number;
+      message?: string;
+      /** True when this event reports a remote DeleteObject (no upload). */
+      deleted?: boolean;
     }
-  | { type: "progress"; path: string; bytes: number; message?: string }
   | { type: "error"; path: string; message: string }
   | {
       type: "conflict";
@@ -173,6 +189,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     bytesToUpload: 0,
     filesToSkip: plan.filesToSkip,
     filesToConflict: plan.filesToConflict,
+    filesToDelete: 0,
   });
 
   // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor