@indigoai-us/hq-cloud 6.10.1 → 6.11.0

package/src/cli/sync.ts

@@ -539,6 +539,94 @@ async function reportNewFilesToNotify(
   }
 }
 
+/** Timeout for the best-effort FILE_TOMBSTONE fetch (GET /v1/files/tombstones). */
+const FETCH_TOMBSTONES_TIMEOUT_MS = 5000;
+
+/**
+ * A FILE_TOMBSTONE as the pull planner needs it: the deleted key + when it was
+ * deleted. The `deletedAt` timestamp is the decisive precedence signal — a
+ * remote object newer than it is a genuine re-create (sync it), an object at or
+ * older than it is a stale resurrection of a deleted key (suppress it).
+ */
+interface CompanyTombstone {
+  deletedAt: string;
+}
+
+/**
+ * Fetch the company's FILE_TOMBSTONE rows from hq-pro (GET /v1/files/tombstones)
+ * and return them as a POSIX-keyed map the pull planner consults to avoid
+ * resurrecting an intentionally-deleted object (delete-resync). The endpoint is
+ * ACL-filtered server-side, so the map only ever contains keys this caller can
+ * read — exactly the keys that can appear in the (STS-scoped) remote LIST.
+ *
+ * Best-effort and bounded by a 5s timeout: a tombstone read that fails, times
+ * out, or returns non-2xx degrades to an EMPTY map — i.e. to the pre-fix
+ * behavior (no suppression). That is the safe failure direction: a missed
+ * tombstone re-pulls a deleted file (a known, recoverable annoyance), whereas a
+ * spurious tombstone would HIDE a file the user wants. The failure is logged
+ * (never silently swallowed) so a persistently-degraded read is visible.
+ */
+async function fetchCompanyTombstones(
+  vaultConfig: VaultServiceConfig,
+  companyUid: string,
+): Promise<Map<string, CompanyTombstone>> {
+  const out = new Map<string, CompanyTombstone>();
+  try {
+    const token =
+      typeof vaultConfig.authToken === "function"
+        ? await vaultConfig.authToken()
+        : vaultConfig.authToken;
+    const base = vaultConfig.apiUrl.replace(/\/+$/, "");
+    const url = `${base}/v1/files/tombstones?company=${encodeURIComponent(
+      companyUid,
+    )}`;
+    const controller = new AbortController();
+    const timer = setTimeout(
+      () => controller.abort(),
+      FETCH_TOMBSTONES_TIMEOUT_MS,
+    );
+    try {
+      const res = await fetch(url, {
+        method: "GET",
+        headers: { Authorization: `Bearer ${token}` },
+        signal: controller.signal,
+      });
+      if (!res.ok) {
+        // Non-2xx is non-fatal: log and degrade to no-suppression. A 404 means
+        // the endpoint is not deployed yet (hq-pro release lag) — the pull
+        // proceeds with the legacy behavior until it lands.
+        console.error(
+          `[hq-sync] tombstone fetch returned ${res.status} (degrading to no-suppression)`,
+        );
+        return out;
+      }
+      const body = (await res.json()) as {
+        tombstones?: Array<{ key?: string; deletedAt?: string }>;
+      };
+      for (const t of body.tombstones ?? []) {
+        if (typeof t.key === "string" && typeof t.deletedAt === "string") {
+          out.set(toPosixKey(t.key), { deletedAt: t.deletedAt });
+        }
+      }
+    } finally {
+      clearTimeout(timer);
+    }
+  } catch (err) {
+    // Best-effort: a failed tombstone read must never break the pull. Log
+    // (policy: never silently swallow) and degrade to no-suppression.
+    try {
+      console.error(
+        `[hq-sync] tombstone fetch failed (non-fatal, degrading to no-suppression): ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    } catch {
+      // swallow — logging must never break sync
+    }
+  }
+  return out;
+}
+
 /**
  * Sync (pull) all allowed files from the entity vault.
  */
@@ -614,6 +702,14 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
   // List all remote files (IAM session policy filters at the AWS layer)
   const remoteFiles = await listRemoteFiles(ctx);
 
+  // Fetch the company's FILE_TOMBSTONE records so the planner can suppress
+  // resurrection of an intentionally-deleted object (delete-resync). Done in
+  // parallel intent with the LIST above conceptually, but kept serial here for
+  // a clean read of `ctx`; best-effort — a failed read degrades to an empty map
+  // (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
+  // companyUid the tombstone rows are keyed under.
+  const tombstones = await fetchCompanyTombstones(vaultConfig, ctx.uid);
+
   // Stage 1: classify every remote file against the journal + local disk.
   // Hashing happens here (not in the transfer loop) so the plan event below
   // carries an accurate denominator before any progress events fire.
@@ -626,6 +722,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     options.includeLocalCompanies === true,
     options.teamSyncedSlugs ?? null,
     currentPrefixSet,
+    tombstones,
   );
 
   emit({
@@ -637,7 +734,10 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     bytesToUpload: 0,
     filesToSkip: plan.filesToSkip,
     filesToConflict: plan.filesToConflict,
-    filesToDelete: 0,
+    // Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
+    // deletes known at plan time; the journal-vs-LIST tombstones are
+    // HEAD-verified later and surfaced via the final filesTombstoned count.
+    filesToDelete: plan.filesToTombstoneDelete,
   });
 
   // ── Scope-shrink cleanup (US-005) ─────────────────────────────────────────
@@ -835,6 +935,58 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
       continue;
     }
 
+    if (item.action === "tombstone-delete") {
+      // Authoritative FILE_TOMBSTONE delete (delete-resync): the remote object
+      // is present but a tombstone marks the key intentionally deleted and it is
+      // not a newer re-create. Delete any local copy and drop the journal entry
+      // so it stays gone — the mirror of the journal-vs-LIST tombstone executor
+      // below, but WITHOUT the HEAD-verify (the remote object is present by
+      // definition; the FILE_TOMBSTONE is the deletion authority). The planner
+      // already routed any divergent local copy to `conflict`, so a local file
+      // reaching here matches the deleted baseline and is safe to remove.
+      const tombstoneKey = item.remoteFile.key;
+      // Same Windows-backslash landmine guard as the journal-tombstone executor:
+      // a malformed key must never reach fs.unlinkSync (path.join collapses the
+      // backslashes onto a REAL POSIX file). Drop the poisoned journal entry
+      // without touching disk.
+      if (isMalformedVaultKey(tombstoneKey)) {
+        removeEntry(journal, tombstoneKey);
+        continue;
+      }
+      try {
+        const lstat = fs.lstatSync(item.localPath);
+        if (lstat.isSymbolicLink() || lstat.isFile()) {
+          fs.unlinkSync(item.localPath);
+        }
+        // A directory at the key: don't recursively rm-rf the operator's dir;
+        // just drop the journal entry (safe-by-default, same as the other path).
+      } catch (err: unknown) {
+        const code =
+          err && typeof err === "object" && "code" in err
+            ? (err as { code?: string }).code
+            : undefined;
+        // ENOENT → local already absent (the common case: a fresh machine that
+        // never held the file, or a prior pull already removed it) → drop the
+        // journal entry and converge. Other errors (EACCES/EPERM/…) leave the
+        // file in place; surface and KEEP the journal entry so the next sync
+        // retries rather than forgetting the delete.
+        if (code !== "ENOENT") {
+          emit({
+            type: "error",
+            path: tombstoneKey,
+            message: `tombstone-suppress unlink failed: ${
+              err instanceof Error ? err.message : String(err)
+            }`,
+          });
+          continue;
+        }
+      }
+      removeEntry(journal, tombstoneKey);
+      filesTombstoned++;
+      emit({ type: "progress", path: tombstoneKey, bytes: 0 });
+      continue;
+    }
+
     if (item.action === "download") {
       downloadItems.push(item);
       continue;
@@ -1481,6 +1633,37 @@ function hasRemoteChanged(
   return remote.lastModified.getTime() > syncedAt;
 }
 
+/**
+ * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
+ * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
+ * object must still sync (the tombstone is not permanent suppression). Returns
+ * true to ALLOW the download (re-create), false to honor the tombstone.
+ *
+ * Decisive signal: the remote object's `lastModified` strictly newer than the
+ * tombstone's `deletedAt`. A delete-marker + tombstone are written together at
+ * delete time, so any object that post-dates the tombstone is a new write at
+ * that key. An object at-or-before the tombstone is the deleted version (a
+ * stale re-push or an un-propagated delete-marker) → honor the tombstone.
+ *
+ * Fail-OPEN (treat as re-create, allow download) when the comparison can't be
+ * made — a malformed `deletedAt`, or no remote `lastModified`. Hiding a file the
+ * user can see is worse than re-pulling one they deleted; the latter is
+ * recoverable, the former looks like data loss.
+ */
+function isRemoteRecreateAfterTombstone(
+  remote: { lastModified?: Date },
+  tombstone: CompanyTombstone,
+): boolean {
+  const deletedAtMs = Date.parse(tombstone.deletedAt);
+  if (Number.isNaN(deletedAtMs)) return true; // malformed tombstone → don't suppress
+  const remoteMs =
+    remote.lastModified instanceof Date
+      ? remote.lastModified.getTime()
+      : NaN;
+  if (Number.isNaN(remoteMs)) return true; // no remote timestamp → don't suppress
+  return remoteMs > deletedAtMs;
+}
+
 /**
  * Stage-1 classification for a single remote object. Each remote file falls
  * into exactly one bucket; the executor in `sync()` switches on `action` to
@@ -1501,6 +1684,14 @@ type PullPlanItem =
   // the remote LIST (and accessible per STS) but deliberately not downloaded
   // because the membership's sync scope doesn't cover them.
   | { action: "skip-out-of-scope"; remoteFile: RemoteFile; localPath: string }
+  // Remote key present in the LIST but carrying a FILE_TOMBSTONE that marks it
+  // intentionally deleted (and the remote object is NOT a newer re-create). The
+  // executor deletes any local copy and drops the journal entry — the
+  // authoritative-delete arm of delete-resync. Unlike `PullPlan.tombstones`
+  // (keys ABSENT from the LIST, HEAD-verified before delete), these are
+  // suppressed purely on the FILE_TOMBSTONE authority and skip HEAD-verify (the
+  // remote object is present by definition).
+  | { action: "tombstone-delete"; remoteFile: RemoteFile; localPath: string }
   | {
       action: "conflict";
       remoteFile: RemoteFile;
@@ -1544,6 +1735,12 @@ interface PullPlan {
    * Carried on the plan so the executor can iterate without re-walking.
    */
   tombstones: string[];
+  /**
+   * Count of `tombstone-delete` items — remote keys present in the LIST but
+   * suppressed by a FILE_TOMBSTONE (delete-resync). Surfaced on the plan event's
+   * `filesToDelete` axis; the per-item executor applies the local delete.
+   */
+  filesToTombstoneDelete: number;
 }
 
 /**
@@ -1569,6 +1766,13 @@ function computePullPlan(
   // `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
   // the scope filter below becomes a no-op and legacy behavior is preserved.
   prefixSet: string[],
+  // FILE_TOMBSTONE records (POSIX-keyed) for the company — the durable
+  // "this key was intentionally deleted" signal the planner consults before
+  // re-downloading a key, so a deleted folder does not resync back in
+  // (delete-resync). An empty map (the default / degraded-fetch case)
+  // preserves the legacy behavior bit-for-bit. (Named `fileTombstones` to avoid
+  // shadowing the local `tombstones` string[] — the journal-vs-LIST delete set.)
+  fileTombstones: ReadonlyMap<string, CompanyTombstone> = new Map(),
 ): PullPlan {
   const items: PullPlanItem[] = [];
 
@@ -1651,6 +1855,22 @@ function computePullPlan(
     }
 
     const journalEntry = getEntry(journal, remoteFile.key);
+
+    // ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
+    // A remote object present in the LIST may be an intentionally-deleted key
+    // (a peer re-pushed it, or its delete-marker hasn't propagated to this
+    // caller's view). `tombstoneSuppresses` is true when a FILE_TOMBSTONE marks
+    // this key deleted AND the remote object is NOT a newer re-create — i.e. the
+    // object is the stale/deleted version and must NOT be downloaded. A genuine
+    // re-create (object newer than the tombstone) leaves this false so the
+    // normal download/merge path runs (the tombstone is not permanent
+    // suppression). An empty tombstone map (the default / degraded-fetch case)
+    // makes this always false — legacy behavior preserved bit-for-bit.
+    const tombstone = fileTombstones.get(toPosixKey(remoteFile.key));
+    const tombstoneSuppresses =
+      tombstone !== undefined &&
+      !isRemoteRecreateAfterTombstone(remoteFile, tombstone);
+
     // lstat (not existsSync/statSync) handles three cases the legacy
     // checks got wrong for symlinks:
     //   1. A valid symlink at localPath: existsSync returns true and the
@@ -1750,6 +1970,32 @@ function computePullPlan(
       const remoteChanged =
         !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
 
+      // ── Authoritative delete reaches a machine that still holds the file ──
+      // A FILE_TOMBSTONE marks this key deleted and the remote object is not a
+      // newer re-create, but this machine still has a local copy. Honor the
+      // delete — EXCEPT never destroy unsynced local work: if the local copy
+      // diverges from the synced baseline (a post-tombstone edit, or an
+      // untracked local file with no journal entry), surface a CONFLICT instead
+      // of silently deleting (the safety guard the brief requires + the 3-way
+      // merge invariant). A local copy that still matches the deleted baseline
+      // is the stale version → delete it + drop the journal entry so it stays
+      // gone on this machine.
+      if (tombstoneSuppresses) {
+        if (localChanged || !journalEntry) {
+          items.push({
+            action: "conflict",
+            remoteFile,
+            localPath,
+            localHash,
+            localMtime: localLstat!.mtime,
+            localSize: isLocalSymlink ? 0 : localLstat!.size,
+          });
+        } else {
+          items.push({ action: "tombstone-delete", remoteFile, localPath });
+        }
+        continue;
+      }
+
       // Mirror the original 3-way merge from the inline loop. Tested by
       // `does NOT flag a pull conflict when only local changed since last
       // sync` and `detects conflicts with local changes…`.
@@ -1827,6 +2073,19 @@ function computePullPlan(
       // through to download.
     }
 
+    // Suppress the re-download of an intentionally-deleted key (delete-resync).
+    // Reaches here only for the `!localExists` case — the `localExists` branch
+    // above already handled (and `continue`d) any suppressing tombstone, so a
+    // tombstone that survives to here means "remote present, local absent, not a
+    // re-create": the classic resurrection ("remote present → I'm behind →
+    // download") the planner must NOT do. Route to `tombstone-delete` so the
+    // executor drops any stale journal entry and the key stays gone, instead of
+    // pulling the deleted object back in.
+    if (tombstoneSuppresses) {
+      items.push({ action: "tombstone-delete", remoteFile, localPath });
+      continue;
+    }
+
     items.push({ action: "download", remoteFile, localPath, isNew: !localExists });
   }
 
@@ -1836,6 +2095,7 @@ function computePullPlan(
   let filesToConflict = 0;
   let filesExcludedByPolicy = 0;
   let filesOutOfScope = 0;
+  let filesToTombstoneDelete = 0;
   const newFiles: Array<{ path: string; bytes: number }> = [];
   for (const item of items) {
     if (item.action === "download") {
@@ -1846,6 +2106,10 @@ function computePullPlan(
       }
     } else if (item.action === "conflict") {
       filesToConflict++;
+    } else if (item.action === "tombstone-delete") {
+      // Authoritative FILE_TOMBSTONE delete — its own axis so it never inflates
+      // the "unchanged" tally. Surfaced via the plan event's filesToDelete.
+      filesToTombstoneDelete++;
     } else if (item.action === "skip-excluded-policy") {
       filesExcludedByPolicy++;
       // Excluded-policy items don't roll into filesToSkip — they're a
@@ -1972,6 +2236,7 @@ function computePullPlan(
     newFilesCount: newFiles.length,
     filesExcludedByPolicy,
     filesOutOfScope,
+    filesToTombstoneDelete,
     tombstones,
   };
 }

package/vitest.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig } from "vitest/config";
+
+// Vitest defaults to a 5_000ms per-test timeout and 10_000ms hook timeout.
+// That is comfortable for the unit suite, but the `rescue-*.test.ts`
+// integration suites build a real git repository in `beforeAll` and then shell
+// out to `git` (and the rescue CLI, which itself spawns several `git`
+// subprocesses) inside each `it`. Measured cold these tests run ~4–5s each;
+// under full file-parallelism they contend for CPU and disk and routinely
+// cross the 5s default, producing flaky "Test timed out in 5000ms" failures
+// that have nothing to do with the code under test.
+//
+// Raise the global ceilings to give those git-bound tests headroom. The values
+// are still tight enough to catch a genuinely hung test (an unresolved promise
+// or a deadlocked subprocess) — they just stop punishing legitimately-slow
+// integration work. Keep this central rather than sprinkling per-test timeouts
+// so future git-heavy suites inherit the headroom automatically.
+export default defineConfig({
+  test: {
+    testTimeout: 20_000,
+    hookTimeout: 30_000,
+  },
+});