@indigoai-us/hq-cloud 6.11.11 → 6.11.12

package/src/bin/sync-runner.ts

@@ -107,6 +107,7 @@ import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
 import { reindexAfterSync } from "../qmd-reindex.js";
 import { pruneConflictIndex } from "../lib/conflict-index.js";
 import {
+  acquireOperationLock,
   withOperationLock,
   OperationLockedError,
   OPERATION_LOCKED_EXIT,
@@ -486,6 +487,8 @@ export interface RunnerDeps {
   createVaultClient?: (config: VaultServiceConfig) => VaultClientSurface;
   /** Sync function. Defaults to `cli/sync.sync`. */
   sync?: (options: SyncOptions) => Promise<SyncResult>;
+  /** Internal: set when runRunner is invoked under the per-root operation lock. */
+  operationLockAlreadyHeld?: boolean;
   /** Share function (push phase). Defaults to `cli/share.share`. */
   share?: (options: ShareOptions) => Promise<ShareResult>;
   /**
@@ -1169,6 +1172,7 @@ export async function runRunner(
     // rows land on the right company regardless of which phase emitted them.
     // Also updates `state` for `progress` events so the rollup has accurate
     // partial counts even if the sync function throws before returning.
+    let companyHadTransferError = false;
     const tagAndEmit = (event: SyncProgressEvent): void => {
       if (event.type === "plan") {
         emit({
@@ -1212,6 +1216,11 @@ export async function runRunner(
           resolution: event.resolution,
         });
       } else if (event.type === "error") {
+        companyHadTransferError = true;
+        errors.push({
+          company: companyLabel,
+          message: event.path ? `${event.path}: ${event.message}` : event.message,
+        });
         emit({
           type: "error",
           company: companyLabel,
@@ -1440,6 +1449,9 @@ export async function runRunner(
                 teamSyncedSlugs,
               }
             : {}),
+          ...(deps.operationLockAlreadyHeld
+            ? { operationLockAlreadyHeld: true }
+            : {}),
           onEvent: tagAndEmit,
         });
       }
@@ -1474,7 +1486,11 @@ export async function runRunner(
       state.bytesDownloaded = pullResult.bytesDownloaded;
       state.filesUploaded = pushResult.filesUploaded;
       state.bytesUploaded = pushResult.bytesUploaded;
-      state.status = aborted ? "aborted" : "complete";
+      state.status = companyHadTransferError
+        ? "errored"
+        : aborted
+          ? "aborted"
+          : "complete";
 
       emit({
         type: "complete",
@@ -1882,6 +1898,26 @@ export function buildTargetedPushArgv(
   return ["--companies", "--direction", "push", ...carried];
 }
 
+type WatchRoute = NonNullable<ReturnType<typeof routeChangeToTarget>>;
+
+function routeKey(route: WatchRoute): string {
+  return route.kind === "company" ? `company:${route.slug}` : "personal";
+}
+
+function routesForBatch(batch: TreeChangeBatch): Map<string, WatchRoute> {
+  const routes = new Map<string, WatchRoute>();
+  for (const relPath of batch.paths.values()) {
+    const route = routeChangeToTarget(relPath);
+    if (!route) continue;
+    routes.set(routeKey(route), route);
+  }
+  return routes;
+}
+
+function buildFullFanoutPushArgv(baseArgv: string[]): string[] {
+  return ["--companies", "--direction", "push", ...carriedFlags(baseArgv)];
+}
+
 /**
  * Build the argv for a targeted PULL pass from a routed change (US-009 — the
  * receiver's pull-on-event path). Mirrors {@link buildTargetedPushArgv} but
@@ -1908,14 +1944,12 @@ export async function runRunnerWithLoop(
   argv: string[],
   deps: RunnerLoopDeps = {},
 ): Promise<number> {
+  const parsed = parseArgs(argv);
   if (!argv.includes("--watch")) {
     // One-shot cloud sync — take the per-root operation lock so it is mutually
-    // exclusive with rescue/reindex. The `--watch` path below is the push
-    // watcher and is intentionally EXEMPT (it neither takes nor is blocked by
-    // the lock; its in-process targeted passes call `runRunner` directly, not
-    // through here). If args don't parse, fall through to `runRunner` so it
-    // surfaces the parse error rather than us masking it with a lock failure.
-    const parsed = parseArgs(argv);
+    // exclusive with rescue/reindex. If args don't parse, fall through to
+    // `runRunner` so it surfaces the parse error rather than us masking it
+    // with a lock failure.
     if ("error" in parsed) return runRunner(argv);
     // The actual sync pass — same seam the watch loop uses (deps.runPass),
     // so a test can assert "waits for a short-lived holder, THEN proceeds to
@@ -1924,7 +1958,10 @@ export async function runRunnerWithLoop(
     // (feedback_28a1833f): instant-sync one-shots used to exit 17 and die on
     // a lock conflict with the ~1-min reindex hook; they now WAIT (default)
     // and proceed once the short holder releases.
-    const runOnce = deps.runPass ?? ((passArgv: string[]) => runRunner(passArgv));
+    const runOnce =
+      deps.runPass ??
+      ((passArgv: string[]) =>
+        runRunner(passArgv, { operationLockAlreadyHeld: true }));
     try {
       return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
         timeoutSec: parsed.lockTimeoutSec,
@@ -1944,10 +1981,14 @@ export async function runRunnerWithLoop(
       throw err;
     }
   }
+  if ("error" in parsed) return runRunner(argv);
   const sleep =
     deps.sleep ??
     ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
-  const runPass = deps.runPass ?? ((passArgv: string[]) => runRunner(passArgv));
+  const runPass =
+    deps.runPass ??
+    ((passArgv: string[]) =>
+      runRunner(passArgv, { operationLockAlreadyHeld: true }));
   const pollIdx = argv.indexOf("--poll-remote-ms");
   const pollMs =
     pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
@@ -1961,9 +2002,7 @@ export async function runRunnerWithLoop(
   // watch filter correctly in companies mode. personalMode is only for a
   // personal-vault-as-root run, where companies/ et al. genuinely aren't synced.
   const companiesMode = argv.includes("--companies");
-  const hqIdx = argv.indexOf("--hq-root");
-  const hqRoot =
-    hqIdx >= 0 && argv[hqIdx + 1] ? argv[hqIdx + 1] : DEFAULT_HQ_ROOT;
+  const hqRoot = parsed.hqRoot;
 
   // Strip the loop-only flags before delegating: the parser inside runRunner
   // accepts --watch/--poll-remote-ms/--event-push, but we don't want a per-
@@ -1976,31 +2015,181 @@ export async function runRunnerWithLoop(
     return true;
   });
 
-  // ---- shared in-flight guard ------------------------------------------
-  // The poll loop AND watcher-triggered targeted pushes funnel through this
-  // mutex so a watcher push never overlaps an in-flight pass (PRD AC). A
-  // trigger that arrives while a pass runs is collapsed by WatchPushDriver's
-  // own pending-while-pushing logic, then re-armed after the pass settles.
-  let inFlight = false;
-  let stopped = false;
-  const runGuarded = async (
-    pass: () => Promise<number>,
-  ): Promise<number | "skipped"> => {
-    if (inFlight) return "skipped";
-    inFlight = true;
+  if (parsed.lockTimeoutSec === 0) {
     try {
-      return await pass();
-    } finally {
-      inFlight = false;
+      const handle = acquireOperationLock(hqRoot, "sync", {
+        timeoutSec: 0,
+        onWaitStart: () => undefined,
+      });
+      handle.release();
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
+      throw err;
+    }
+  }
+
+  const runPassWithLock = async (passArgvForRun: string[]): Promise<number> => {
+    try {
+      const handle = acquireOperationLock(hqRoot, "sync", {
+        timeoutSec: 0,
+        onWaitStart: () => undefined,
+      });
+      try {
+        return await runPass(passArgvForRun);
+      } finally {
+        handle.release();
+      }
+    } catch (err) {
+      if (!(err instanceof OperationLockedError)) throw err;
+      if (parsed.lockTimeoutSec === 0) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
     }
+
+    try {
+      return await withOperationLock(
+        hqRoot,
+        "sync",
+        () => runPass(passArgvForRun),
+        { timeoutSec: parsed.lockTimeoutSec },
+      );
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        process.stderr.write(err.message + "\n");
+        return OPERATION_LOCKED_EXIT;
+      }
+      throw err;
+    }
+  };
+
+  // ---- shared pass queue -----------------------------------------------
+  // The poll loop, pull-on-event receiver, and watcher-triggered pushes all
+  // funnel through this queue so local/remote triggers never overlap, and a
+  // trigger arriving during an active pass runs immediately after it instead
+  // of being dropped.
+  let stopped = false;
+  let activePass: Promise<number> | null = null;
+  type QueuedPass = {
+    argv: string[];
+    resolve: (code: number) => void;
+    reject: (err: unknown) => void;
+  };
+  const pendingPasses: QueuedPass[] = [];
+  const resolveStoppedQueue = (): void => {
+    while (pendingPasses.length > 0) {
+      pendingPasses.shift()?.resolve(0);
+    }
+  };
+  const drainQueuedPasses = (): void => {
+    if (activePass !== null) return;
+    if (stopped) {
+      resolveStoppedQueue();
+      return;
+    }
+    const next = pendingPasses.shift();
+    if (!next) return;
+    const current = startGuardedPass(next.argv);
+    void current.then(next.resolve, next.reject);
+  };
+  const startGuardedPass = (passArgvForRun: string[]): Promise<number> => {
+    const current = stopped
+      ? Promise.resolve(0)
+      : runPassWithLock(passArgvForRun);
+    activePass = current;
+    void current
+      .finally(() => {
+        if (activePass === current) {
+          activePass = null;
+          drainQueuedPasses();
+        }
+      })
+      .catch(() => undefined);
+    return current;
+  };
+  const runGuarded = (passArgvForRun: string[]): Promise<number> => {
+    if (activePass === null && pendingPasses.length === 0) {
+      return startGuardedPass(passArgvForRun);
+    }
+
+    return new Promise<number>((resolve, reject) => {
+      pendingPasses.push({ argv: passArgvForRun, resolve, reject });
+      drainQueuedPasses();
+    });
   };
 
   // ---- event-push wiring (Phase 1) -------------------------------------
   let watcher: WatcherSurface | null = null;
   let driver: WatchPushDriver | null = null;
   let detachSignal: (() => void) | null = null;
-  let lastChangedRel: string | null = null;
-  let lastBatch: TreeChangeBatch | null = null;
+  const pendingWatcherPaths = new Map<string, string>();
+  let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
+  let pendingWatcherBareChange = false;
+  let pendingWatcherOverflowed = false;
+  let pendingWatcherDroppedPaths = 0;
+  let pendingWatcherDroppedBytes = 0;
+  const addPendingWatcherChange = (
+    changedRelPath?: string,
+    batch?: TreeChangeBatch,
+  ): void => {
+    if (batch) {
+      pendingWatcherOriginalBatch =
+        pendingWatcherPaths.size === 0 &&
+        !pendingWatcherBareChange &&
+        !pendingWatcherOverflowed &&
+        !batch.overflowed
+          ? batch
+          : null;
+      for (const [absolutePath, relativePath] of batch.paths.entries()) {
+        pendingWatcherPaths.set(absolutePath, relativePath);
+      }
+      if (batch.overflowed) {
+        pendingWatcherOverflowed = true;
+        pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
+        pendingWatcherDroppedBytes += batch.droppedBytes ?? 0;
+      }
+      return;
+    }
+    if (changedRelPath) {
+      pendingWatcherOriginalBatch = null;
+      pendingWatcherPaths.set(path.join(hqRoot, changedRelPath), changedRelPath);
+      return;
+    }
+    pendingWatcherOriginalBatch = null;
+    pendingWatcherBareChange = true;
+  };
+  const takePendingWatcherChange = (): {
+    rel: string | null;
+    batch: TreeChangeBatch | null;
+  } => {
+    const batch =
+      pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
+        ? pendingWatcherOriginalBatch
+        : pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
+        ? {
+            paths: new Map(pendingWatcherPaths),
+            ...(pendingWatcherOverflowed
+              ? {
+                  overflowed: true,
+                  droppedPaths: pendingWatcherDroppedPaths,
+                  droppedBytes: pendingWatcherDroppedBytes,
+                }
+              : {}),
+          }
+        : null;
+    const rel = [...pendingWatcherPaths.values()][0] ?? null;
+    const fallbackRel = rel ?? (pendingWatcherBareChange ? null : null);
+    pendingWatcherPaths.clear();
+    pendingWatcherOriginalBatch = null;
+    pendingWatcherBareChange = false;
+    pendingWatcherOverflowed = false;
+    pendingWatcherDroppedPaths = 0;
+    pendingWatcherDroppedBytes = 0;
+    return { rel: fallbackRel, batch };
+  };
   // ---- pull-on-event receiver (Phase 2, US-009) ------------------------
   // Started after the watcher, disposed before the watcher (mirror of the
   // PushTransport ordering). Dormant by default: the default factory returns
@@ -2037,24 +2226,36 @@ export async function runRunnerWithLoop(
       clock,
       push: async () => {
         if (stopped) return;
-        const rel = lastChangedRel;
-        // Snapshot the settled batch BEFORE the await: a change landing
-        // mid-pass overwrites lastBatch for the NEXT pass, and this pass
-        // must only announce what it actually pushed.
-        const batchForPublish = lastBatch;
-        lastBatch = null;
-        const route = rel
-          ? routeChangeToTarget(rel)
-          : { kind: "personal" as const };
-        if (!route) return;
-        const targetedArgv = buildTargetedPushArgv(route, passArgv);
-        const result = await runGuarded(() => runPass(targetedArgv));
+        // Snapshot accumulated watcher work BEFORE the await. Changes landing
+        // mid-pass are accumulated for the NEXT pass instead of overwriting
+        // this pass's publish target.
+        const { rel, batch: batchForPublish } = takePendingWatcherChange();
+        const batchRoutes = batchForPublish
+          ? routesForBatch(batchForPublish)
+          : new Map<string, WatchRoute>();
+        let targetedArgv: string[];
+        if (batchForPublish?.overflowed || batchRoutes.size > 1) {
+          targetedArgv = buildFullFanoutPushArgv(passArgv);
+        } else {
+          const route =
+            batchRoutes.size === 1
+              ? [...batchRoutes.values()][0]
+              : rel
+                ? routeChangeToTarget(rel)
+                : { kind: "personal" as const };
+          if (!route) return;
+          targetedArgv = buildTargetedPushArgv(route, passArgv);
+        }
+        const result = await runGuarded(targetedArgv);
         // Phase 3 (US-017): publish PushEvents only AFTER the targeted push
         // pass succeeded — an event must never announce bytes that are not
-        // in S3 yet. A skipped pass (guard held) or a failed pass publishes
-        // nothing; the cadence poll covers the miss. Fall back to a
-        // single-path batch when the watcher emitted a bare path signal.
-        if (result === 0 && eventSync) {
+        // in S3 yet. A failed pass publishes nothing; queued passes run after
+        // the active pass instead of dropping the watcher-triggered push. Fall
+        // back to a single-path batch when the watcher emitted a bare path
+        // signal. Overflowed batches deliberately publish nothing because the
+        // exact path set was dropped; the full fanout push plus cadence poll is
+        // the resync signal.
+        if (result === 0 && !stopped && eventSync && !batchForPublish?.overflowed) {
           const batch: TreeChangeBatch | null =
             batchForPublish ??
             (rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
@@ -2070,8 +2271,7 @@ export async function runRunnerWithLoop(
     // the bare-signal TreeWatcher leaves it null → personal-vault route.
     watcher.onChange((changedRelPath, batch) => {
       if (stopped) return;
-      lastChangedRel = changedRelPath ?? null;
-      lastBatch = batch ?? null;
+      addPendingWatcherChange(changedRelPath, batch);
       driver?.notifyChange();
     });
     watcher.start();
@@ -2088,7 +2288,10 @@ export async function runRunnerWithLoop(
       const route = routeChangeToTarget(ctx.event.relativePath);
       if (!route) return;
       const targetedArgv = buildTargetedPullArgv(route, passArgv);
-      await runGuarded(() => runPass(targetedArgv));
+      const result = await runGuarded(targetedArgv);
+      if (result !== 0) {
+        throw new Error(`targeted pull failed with exit code ${result}`);
+      }
     };
     const createReceiver =
       deps.createReceiver ?? (() => new NoopPushReceiver());
@@ -2167,6 +2370,7 @@ export async function runRunnerWithLoop(
   const shutdown = (): void => {
     if (stopped) return;
     stopped = true;
+    resolveStoppedQueue();
     // Dispose the receiver FIRST (mirror of the PushTransport ordering:
     // inbound subscription torn down before the watcher) so no new
     // pull-on-event fires mid-teardown. dispose() is async (it drains the
@@ -2213,10 +2417,8 @@ export async function runRunnerWithLoop(
 
   try {
     while (!stopped) {
-      const result = await runGuarded(() => runPass(passArgv));
-      // A poll pass that was skipped because a watcher push held the guard is
-      // benign — the next iteration retries after the poll interval.
-      if (typeof result === "number" && result !== 0) {
+      const result = await runGuarded(passArgv);
+      if (result !== 0) {
         return result;
       }
       // Sleep the poll interval, but wake early on shutdown so SIGTERM stops

package/src/cli/rescue-classify-ordering.test.ts

@@ -74,6 +74,41 @@ const lexists = (p: string) => {
   }
 };
 
+function hasRecoveryManifestFor(root: string, rel: string): boolean {
+  let found = false;
+  const interesting = /(recover|recovery|rollback|transaction|manifest|resume|rescue)/i;
+  const walk = (dir: string) => {
+    if (found) return;
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const ent of entries) {
+      if (found) return;
+      const abs = path.join(dir, ent.name);
+      if (ent.isDirectory()) {
+        walk(abs);
+        continue;
+      }
+      if (!ent.isFile()) continue;
+      const relPath = abs.slice(root.length + 1);
+      if (!interesting.test(relPath)) continue;
+      let body = "";
+      try {
+        body = fs.readFileSync(abs, "utf-8");
+      } catch {
+        body = "";
+      }
+      found = body.includes(rel) && interesting.test(`${relPath}\n${body}`);
+    }
+  };
+  walk(root);
+  return found;
+}
+
 describe.skipIf(!gitAvailable)("rescue classify-before-delete + dir-symlink handling", () => {
   let workDir: string;
   let upstream: string;
@@ -108,6 +143,8 @@ describe.skipIf(!gitAvailable)("rescue classify-before-delete + dir-symlink hand
     git(workDir, "init", "-b", "main", "upstream");
     fs.writeFileSync(path.join(upstream, "core/a.md"), "v1\n");
     fs.writeFileSync(path.join(upstream, "core/b.md"), "beta\n");
+    fs.mkdirSync(path.join(upstream, "core/blocker"), { recursive: true });
+    fs.writeFileSync(path.join(upstream, "core/blocker/second.md"), "upstream\n");
     git(upstream, "add", "-A");
     git(upstream, "commit", "-m", "floor");
     floorSha = git(upstream, "rev-parse", "HEAD");
@@ -267,6 +304,30 @@ exec ${JSON.stringify(realGit)} "$@"
     expect(fs.existsSync(path.join(hqRoot, "core/zzz.md"))).toBe(true);
   });
 
+  it("F12: rolls back or records recovery when an apply action fails after earlier destructive actions", () => {
+    const hqRoot = makeHqRoot();
+    fs.mkdirSync(path.join(hqRoot, "core/blocker"), { recursive: true });
+    fs.writeFileSync(path.join(hqRoot, "core/blocker/second.md"), "local edit\n");
+    fs.writeFileSync(path.join(hqRoot, "personal/blocker"), "not a directory\n");
+
+    const firstDeletedByApply = path.join(hqRoot, "core/a.md");
+    const firstOriginal = fs.readFileSync(firstDeletedByApply, "utf-8");
+    const r = runRescueCapture(liveArgs(hqRoot), baseEnv);
+    const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
+
+    const applyFailed = r.threw !== undefined || (r.status !== undefined && r.status !== 0);
+    expect(applyFailed, out).toBe(true);
+    expect(fs.existsSync(path.join(hqRoot, "core/blocker/second.md")), out).toBe(true);
+
+    const firstActionRolledBack =
+      fs.existsSync(firstDeletedByApply) &&
+      fs.readFileSync(firstDeletedByApply, "utf-8") === firstOriginal;
+    expect(
+      firstActionRolledBack || hasRecoveryManifestFor(hqRoot, "core/a.md"),
+      `${out}\ncore/a.md was deleted without rollback or a recovery/transaction manifest`,
+    ).toBe(true);
+  });
+
   it("dry-run classifies the dir-symlink identically to the live run and changes nothing", () => {
     const hqRoot = makeHqRoot();
     const target = path.join(hqRoot, "personal/knowledge/ad-creative-engine");