@indigoai-us/hq-cloud 6.14.18 → 6.14.19

package/src/bin/sync-runner-watch-loop.ts

@@ -1,5 +1,6 @@
 import * as os from "os";
 import * as path from "path";
+import * as fs from "fs";
 import { inventoryManagedSnapshots } from "../backup-prune.js";
 import { VaultClient } from "../index.js";
 import { pickCanonicalPersonEntity } from "../vault-client.js";
@@ -16,8 +17,17 @@ import { getOrCreateMachineId } from "../lib/machine-id.js";
 import {
   TreeWatcher,
   systemClock,
+  type LocalDeleteSnapshot,
+  type TreeChange,
   type TreeChangeBatch,
 } from "../watcher.js";
+import {
+  clearLocalDeleteIntent,
+  markLocalDeleteIntent,
+  PERSONAL_VAULT_JOURNAL_SLUG,
+  readJournal,
+  writeJournal,
+} from "../journal.js";
 import {
   NoopPushReceiver,
   type PushReceiver,
@@ -37,7 +47,12 @@ import {
   routeKey,
   type WatchRoute,
 } from "./sync-runner-watch-routes.js";
-import type { Direction, RunnerLoopDeps, WatcherSurface } from "./sync-runner.js";
+import type {
+  Direction,
+  RunnerLoopDeps,
+  RunnerPassOutcome,
+  WatcherSurface,
+} from "./sync-runner.js";
 
 const FULL_RECONCILE_EVERY_TICKS = 10;
 
@@ -88,7 +103,9 @@ export interface ParsedLoopArgs {
 }
 
 export interface WatchLoopRuntime {
-  runPassWithOperationLockAlreadyHeld: (passArgv: string[]) => Promise<number>;
+  runPassWithOperationLockAlreadyHeld: (
+    passArgv: string[],
+  ) => Promise<number | RunnerPassOutcome>;
   authRequiredPassExit: number;
   defaultGetIdTokenClaims: () => ({ email?: string } & TelemetryClaims) | null;
   defaultGetAccessToken: () => Promise<string>;
@@ -96,6 +113,31 @@ export interface WatchLoopRuntime {
   region: string;
 }
 
+type JournalFiles = ReturnType<typeof readJournal>["files"];
+
+/**
+ * Select journal entries covered by one watcher delete. Exact file unlinks
+ * deliberately avoid enumeration; only directory unlinks expand descendants.
+ */
+export function journalEntriesForWatcherDelete(
+  files: JournalFiles,
+  key: string,
+  kind: "unlink" | "unlinkDir",
+): Array<[string, JournalFiles[string]]> {
+  if (kind === "unlink") {
+    const entry = files[key];
+    return entry ? [[key, entry]] : [];
+  }
+  return Object.entries(files).filter(
+    ([candidate]) =>
+      key === "" || candidate === key || candidate.startsWith(`${key}/`),
+  );
+}
+
+function passExitCode(outcome: number | RunnerPassOutcome): number {
+  return typeof outcome === "number" ? outcome : outcome.exitCode;
+}
+
 export async function runOneShotWithOperationLock(
   argv: string[],
   parsed: ParsedLoopArgs,
@@ -105,9 +147,11 @@ export async function runOneShotWithOperationLock(
   const runOnce =
     deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
   try {
-    return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
-      timeoutSec: parsed.lockTimeoutSec,
-    });
+    return passExitCode(
+      await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
+        timeoutSec: parsed.lockTimeoutSec,
+      }),
+    );
   } catch (err) {
     if (err instanceof OperationLockedError) {
       process.stderr.write(err.message + "\n");
@@ -154,8 +198,10 @@ export async function runWatchLoop(
     if (a === "--poll-remote-ms") return false;
     if (a === "--event-push") return false;
     if (a === "--scope-path") return false;
+    if (a === "--delete-scope-path") return false;
     if (i > 0 && argv[i - 1] === "--poll-remote-ms") return false;
     if (i > 0 && argv[i - 1] === "--scope-path") return false;
+    if (i > 0 && argv[i - 1] === "--delete-scope-path") return false;
     return true;
   });
   // Authoritative direction from parseArgs — never re-derive from argv (that
@@ -183,13 +229,18 @@ export async function runWatchLoop(
     }
   }
 
-  const runPassWithLock = async (passArgvForRun: string[]): Promise<number> => {
+  type PassPreparation = () => void;
+  const runPassWithLock = async (
+    passArgvForRun: string[],
+    prepare?: PassPreparation,
+  ): Promise<number | RunnerPassOutcome> => {
     try {
       const handle = acquireOperationLock(hqRoot, "sync", {
         timeoutSec: 0,
         onWaitStart: () => undefined,
       });
       try {
+        prepare?.();
         return await runPass(passArgvForRun);
       } finally {
         handle.release();
@@ -206,7 +257,10 @@ export async function runWatchLoop(
       return await withOperationLock(
         hqRoot,
         "sync",
-        () => runPass(passArgvForRun),
+        () => {
+          prepare?.();
+          return runPass(passArgvForRun);
+        },
         { timeoutSec: parsed.lockTimeoutSec },
       );
     } catch (err) {
@@ -219,10 +273,11 @@ export async function runWatchLoop(
   };
 
   let stopped = false;
-  let activePass: Promise<number> | null = null;
+  let activePass: Promise<number | RunnerPassOutcome> | null = null;
   type QueuedPass = {
     argv: string[];
-    resolve: (code: number) => void;
+    prepare?: PassPreparation;
+    resolve: (outcome: number | RunnerPassOutcome) => void;
     reject: (err: unknown) => void;
   };
   const pendingPasses: QueuedPass[] = [];
@@ -239,13 +294,16 @@ export async function runWatchLoop(
     }
     const next = pendingPasses.shift();
     if (!next) return;
-    const current = startGuardedPass(next.argv);
+    const current = startGuardedPass(next.argv, next.prepare);
     void current.then(next.resolve, next.reject);
   };
-  const startGuardedPass = (passArgvForRun: string[]): Promise<number> => {
+  const startGuardedPass = (
+    passArgvForRun: string[],
+    prepare?: PassPreparation,
+  ): Promise<number | RunnerPassOutcome> => {
     const current = stopped
       ? Promise.resolve(0)
-      : runPassWithLock(passArgvForRun);
+      : runPassWithLock(passArgvForRun, prepare);
     activePass = current;
     void current
       .finally(() => {
@@ -257,13 +315,16 @@ export async function runWatchLoop(
       .catch(() => undefined);
     return current;
   };
-  const runGuarded = (passArgvForRun: string[]): Promise<number> => {
+  const runGuarded = (
+    passArgvForRun: string[],
+    prepare?: PassPreparation,
+  ): Promise<number | RunnerPassOutcome> => {
     if (activePass === null && pendingPasses.length === 0) {
-      return startGuardedPass(passArgvForRun);
+      return startGuardedPass(passArgvForRun, prepare);
     }
 
-    return new Promise<number>((resolve, reject) => {
-      pendingPasses.push({ argv: passArgvForRun, resolve, reject });
+    return new Promise<number | RunnerPassOutcome>((resolve, reject) => {
+      pendingPasses.push({ argv: passArgvForRun, prepare, resolve, reject });
       drainQueuedPasses();
     });
   };
@@ -271,6 +332,7 @@ export async function runWatchLoop(
   let watcher: WatcherSurface | null = null;
   let detachSignal: (() => void) | null = null;
   const pendingWatcherPaths = new Map<string, string>();
+  const pendingWatcherChanges = new Map<string, TreeChange>();
   let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
   let pendingWatcherBareChange = false;
   let pendingWatcherOverflowed = false;
@@ -290,6 +352,8 @@ export async function runWatchLoop(
           : null;
       for (const [absolutePath, relativePath] of batch.paths.entries()) {
         pendingWatcherPaths.set(absolutePath, relativePath);
+        const change = batch.changes?.get(absolutePath);
+        if (change) pendingWatcherChanges.set(absolutePath, change);
       }
       if (batch.overflowed) {
         pendingWatcherOverflowed = true;
@@ -300,7 +364,9 @@ export async function runWatchLoop(
     }
     if (changedRelPath) {
       pendingWatcherOriginalBatch = null;
-      pendingWatcherPaths.set(path.join(hqRoot, changedRelPath), changedRelPath);
+      const absolutePath = path.join(hqRoot, changedRelPath);
+      pendingWatcherPaths.set(absolutePath, changedRelPath);
+      pendingWatcherChanges.delete(absolutePath);
       return;
     }
     pendingWatcherOriginalBatch = null;
@@ -316,6 +382,9 @@ export async function runWatchLoop(
         : pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
           ? {
               paths: new Map(pendingWatcherPaths),
+              ...(pendingWatcherChanges.size > 0
+                ? { changes: new Map(pendingWatcherChanges) }
+                : {}),
               ...(pendingWatcherOverflowed
                 ? {
                     overflowed: true,
@@ -327,6 +396,7 @@ export async function runWatchLoop(
           : null;
     const bare = pendingWatcherBareChange;
     pendingWatcherPaths.clear();
+    pendingWatcherChanges.clear();
     pendingWatcherOriginalBatch = null;
     pendingWatcherBareChange = false;
     pendingWatcherOverflowed = false;
@@ -341,15 +411,178 @@ export async function runWatchLoop(
     }
   };
 
+  const journalCoordinates = (
+    relativePath: string,
+  ): { slug: string; key: string } | null => {
+    const route = routeChangeToTarget(relativePath);
+    if (!route) return null;
+    if (route.kind === "personal") {
+      return { slug: PERSONAL_VAULT_JOURNAL_SLUG, key: relativePath };
+    }
+    const prefix = `companies/${route.slug}`;
+    return {
+      slug: route.slug,
+      key:
+        relativePath === prefix
+          ? ""
+          : relativePath.slice(prefix.length + 1),
+    };
+  };
+
+  const captureLocalDeleteSnapshots = (
+    relativePath: string,
+    kind: "unlink" | "unlinkDir",
+  ): LocalDeleteSnapshot[] => {
+    const coordinates = journalCoordinates(relativePath);
+    const route = routeChangeToTarget(relativePath);
+    if (!coordinates || !route) return [];
+    const journal = readJournal(coordinates.slug);
+    const snapshots: LocalDeleteSnapshot[] = [];
+    const captureEntry = (
+      key: string,
+      entry: (typeof journal.files)[string] | undefined,
+    ): void => {
+      if (!entry?.remoteEtag || !entry.kind || entry.removedAt !== undefined) {
+        return;
+      }
+      const snapshot: LocalDeleteSnapshot = {
+        journalSlug: coordinates.slug,
+        journalPath: key,
+        deleteScopeRoot: coordinates.key,
+        absolutePath:
+          route.kind === "company"
+            ? path.join(hqRoot, "companies", route.slug, key)
+            : path.join(hqRoot, key),
+        remoteEtag: entry.remoteEtag,
+        localHash: entry.hash,
+        localKind: entry.kind,
+      };
+      if (
+        markLocalDeleteIntent(
+          journal,
+          key,
+          snapshot.localHash,
+          snapshot.localKind,
+          snapshot.remoteEtag,
+        )
+      ) {
+        snapshots.push(snapshot);
+      }
+    };
+
+    // Chokidar emits one unlink per file during rm -rf. Avoid walking the
+    // whole shard for every exact-key event; directory unlinks alone need
+    // descendant expansion.
+    for (const [key, entry] of journalEntriesForWatcherDelete(
+      journal.files,
+      coordinates.key,
+      kind,
+    )) {
+      captureEntry(key, entry);
+    }
+
+    // Persist at observation time, before the debounced batch can queue
+    // behind an in-flight pull. The pull planner treats this current,
+    // version-bound intent as authoritative and will not resurrect the path.
+    if (snapshots.length > 0) {
+      writeJournal(coordinates.slug, journal);
+    }
+    return snapshots;
+  };
+
+  const prepareWatcherChanges = (changes: Map<string, TreeChange>): void => {
+    const journals = new Map<string, ReturnType<typeof readJournal>>();
+    const dirty = new Set<string>();
+    const journalFor = (slug: string) => {
+      const loaded = journals.get(slug);
+      if (loaded) return loaded;
+      const journal = readJournal(slug);
+      journals.set(slug, journal);
+      return journal;
+    };
+
+    for (const [absolutePath, change] of changes.entries()) {
+      if (change.kind === "unlink" || change.kind === "unlinkDir") {
+        for (const snapshot of change.deleteSnapshots ?? []) {
+          let absent = false;
+          try {
+            fs.lstatSync(snapshot.absolutePath);
+          } catch (err: unknown) {
+            const code =
+              err && typeof err === "object" && "code" in err
+                ? (err as { code?: string }).code
+                : undefined;
+            if (code !== "ENOENT") continue;
+            absent = true;
+          }
+          if (!absent) {
+            if (
+              clearLocalDeleteIntent(
+                journalFor(snapshot.journalSlug),
+                snapshot.journalPath,
+              )
+            ) {
+              dirty.add(snapshot.journalSlug);
+            }
+            continue;
+          }
+          if (
+            markLocalDeleteIntent(
+              journalFor(snapshot.journalSlug),
+              snapshot.journalPath,
+              snapshot.localHash,
+              snapshot.localKind,
+              snapshot.remoteEtag,
+            )
+          ) {
+            dirty.add(snapshot.journalSlug);
+          }
+        }
+        continue;
+      }
+
+      const relativePath = path
+        .relative(hqRoot, absolutePath)
+        .split(path.sep)
+        .join("/");
+      const coordinates = journalCoordinates(relativePath);
+      if (
+        coordinates &&
+        clearLocalDeleteIntent(
+          journalFor(coordinates.slug),
+          coordinates.key,
+        )
+      ) {
+        dirty.add(coordinates.slug);
+      }
+    }
+
+    for (const slug of dirty) {
+      writeJournal(slug, journals.get(slug)!);
+    }
+  };
+
   const groupBatchByRoute = (
     batch: TreeChangeBatch,
   ): Map<
     string,
-    { route: WatchRoute; paths: Map<string, string>; relPaths: string[] }
+    {
+      route: WatchRoute;
+      paths: Map<string, string>;
+      changes: Map<string, TreeChange>;
+      relPaths: string[];
+      deleteScopeRoots: string[];
+    }
   > => {
     const grouped = new Map<
       string,
-      { route: WatchRoute; paths: Map<string, string>; relPaths: string[] }
+      {
+        route: WatchRoute;
+        paths: Map<string, string>;
+        changes: Map<string, TreeChange>;
+        relPaths: string[];
+        deleteScopeRoots: string[];
+      }
     >();
     for (const [absolutePath, relativePath] of batch.paths.entries()) {
       const route = routeChangeToTarget(relativePath);
@@ -357,9 +590,28 @@ export async function runWatchLoop(
       const key = routeKey(route);
       const existing =
         grouped.get(key) ??
-        { route, paths: new Map<string, string>(), relPaths: [] };
+        {
+          route,
+          paths: new Map<string, string>(),
+          changes: new Map<string, TreeChange>(),
+          relPaths: [],
+          deleteScopeRoots: [],
+        };
       existing.paths.set(absolutePath, relativePath);
+      const change = batch.changes?.get(absolutePath);
+      if (change) existing.changes.set(absolutePath, change);
       existing.relPaths = [...new Set([...existing.paths.values()])];
+      existing.deleteScopeRoots = [
+        ...new Set(
+          [...existing.changes.values()].flatMap((item) =>
+            (item.deleteSnapshots ?? []).flatMap((snapshot) =>
+              snapshot.deleteScopeRoot === undefined
+                ? []
+                : [snapshot.deleteScopeRoot],
+            ),
+          ),
+        ),
+      ];
       grouped.set(key, existing);
     }
     return grouped;
@@ -381,11 +633,38 @@ export async function runWatchLoop(
           group.route,
           group.relPaths,
           passArgv,
+          group.deleteScopeRoots,
         );
         try {
-          const result = await runGuarded(scopedArgv);
-          if (result === 0) {
-            publishBatchIfAvailable(group.paths);
+          const result = await runGuarded(
+            scopedArgv,
+            group.changes.size > 0
+              ? () => prepareWatcherChanges(group.changes)
+              : undefined,
+          );
+          if (passExitCode(result) === 0) {
+            const refusedPaths = new Set(
+              typeof result === "number"
+                ? []
+                : (result.result?.pushPathResults ?? [])
+                    .filter((pathResult) => pathResult.status === "refused")
+                    .map((pathResult) => pathResult.relativePath),
+            );
+            publishBatchIfAvailable(
+              new Map(
+                [...group.paths.entries()].filter(
+                  ([absolutePath, relativePath]) => {
+                    if (refusedPaths.has(relativePath)) return false;
+                    if (group.changes.get(absolutePath)?.kind !== "unlinkDir") {
+                      return true;
+                    }
+                    return ![...refusedPaths].some((refusedPath) =>
+                      refusedPath.startsWith(`${relativePath}/`),
+                    );
+                  },
+                ),
+              ),
+            );
           } else {
             restorePendingWatcherPaths(group.paths);
           }
@@ -399,7 +678,9 @@ export async function runWatchLoop(
     }
 
     if (!drainRunsPull) return 0;
-    return runGuarded(buildScopedDrainPullArgv(passArgv));
+    return passExitCode(
+      await runGuarded(buildScopedDrainPullArgv(passArgv)),
+    );
   };
 
   let receiver: PushReceiver | null = null;
@@ -416,8 +697,14 @@ export async function runWatchLoop(
           debounceMs: opts.debounceMs,
           clock: opts.clock,
           personalMode: !companiesMode,
+          captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
         }));
-    watcher = createWatcher({ hqRoot, debounceMs, clock });
+    watcher = createWatcher({
+      hqRoot,
+      debounceMs,
+      clock,
+      captureLocalDeleteSnapshots,
+    });
 
     watcher.onChange((changedRelPath, batch) => {
       if (stopped) return;
@@ -431,8 +718,10 @@ export async function runWatchLoop(
       if (!route) return;
       const targetedArgv = buildTargetedPullArgv(route, passArgv);
       const result = await runGuarded(targetedArgv);
-      if (result !== 0) {
-        throw new Error(`targeted pull failed with exit code ${result}`);
+      if (passExitCode(result) !== 0) {
+        throw new Error(
+          `targeted pull failed with exit code ${passExitCode(result)}`,
+        );
       }
     };
     const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
@@ -537,12 +826,13 @@ export async function runWatchLoop(
       const result = shouldRunFull
         ? await runGuarded(passArgv)
         : await runScopedDrain(pending.batch);
-      if (result === runtime.authRequiredPassExit) {
+      const exitCode = passExitCode(result);
+      if (exitCode === runtime.authRequiredPassExit) {
         // The pass already emitted auth-error. Stop the unattended loop but
         // keep the process exit successful so desktop/manual consumers treat
         // this as a reauth state, not a crash.
         return 0;
-      } else if (result === TRANSIENT_NETWORK_EXIT) {
+      } else if (exitCode === TRANSIENT_NETWORK_EXIT) {
         // The machine was briefly offline (the discovery fetch failed after
         // retries). That is NOT a crash — stay alive and retry on the next
         // poll. Returning it here would exit the watcher and the menubar would
@@ -551,7 +841,7 @@ export async function runWatchLoop(
         process.stderr.write(
           "hq-sync-runner: watch pass skipped — transient network failure; retrying next poll\n",
         );
-      } else if (result === PARTIAL_SYNC_EXIT) {
+      } else if (exitCode === PARTIAL_SYNC_EXIT) {
         // The pass completed but some per-file transfers threw. Also NOT a
         // crash: the per-file errors were already emitted as ndjson `error`
         // events (menubar breadcrumbs + journal), and exiting here just gets
@@ -562,8 +852,8 @@ export async function runWatchLoop(
         process.stderr.write(
           "hq-sync-runner: watch pass completed with per-file errors; staying alive, retrying next poll\n",
         );
-      } else if (result !== 0) {
-        return result;
+      } else if (exitCode !== 0) {
+        return exitCode;
       }
       if (Date.now() >= nextBackupPruneAt) {
         // Advance before invoking: a failed/unavailable backstop cannot become

package/src/bin/sync-runner-watch-routes.ts

@@ -43,8 +43,13 @@ export function buildScopedPushArgv(
   route: WatchRoute,
   relPaths: readonly string[],
   baseArgv: string[],
+  deleteScopeRoots: readonly string[] = [],
 ): string[] {
   const scopeArgs = scopedPathArgs(route, relPaths);
+  const deleteScopeArgs = deleteScopeRoots.flatMap((root) => [
+    "--delete-scope-path",
+    root,
+  ]);
   const carried = carriedFlags(baseArgv);
   if (route.kind === "company") {
     return [
@@ -53,10 +58,18 @@ export function buildScopedPushArgv(
       "--direction",
       "push",
       ...scopeArgs,
+      ...deleteScopeArgs,
       ...carried,
     ];
   }
-  return ["--personal", "--direction", "push", ...scopeArgs, ...carried];
+  return [
+    "--personal",
+    "--direction",
+    "push",
+    ...scopeArgs,
+    ...deleteScopeArgs,
+    ...carried,
+  ];
 }
 
 export function buildFullFanoutPushArgv(baseArgv: string[]): string[] {