@indigoai-us/hq-cloud 6.14.34 → 6.14.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cloud",
-  "version": "6.14.34",
+  "version": "6.14.36",
   "description": "HQ by Indigo cloud sync engine — bidirectional S3 sync for mobile access",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/bin/sync-runner-watch-loop.ts

@@ -468,6 +468,46 @@ export async function runWatchLoop(
     }
   };
 
+  /**
+   * Journal roots the bulk-intent quarantine has refused for the current
+   * delete episode, keyed `slug\0journalKey`.
+   *
+   * Chokidar (the Linux backend) reports every removed child through `unlink`
+   * BEFORE the parent `unlinkDir`, so by the time the directory expansion is
+   * refused each descendant has already persisted its own version-bound intent
+   * and handed a snapshot to the pending batch. Declining the expansion alone
+   * leaves those intents standing and `prepareWatcherChanges` re-stamps them
+   * from the snapshots, so a large `rm -rf` still propagates in full under
+   * `HQ_SYNC_DELETE_POLICY=all` — the path this producer-side layer exists to
+   * cover, because it bypasses the engine breaker entirely. The trip therefore
+   * revokes the descendants' intents as well, and this set keeps the refusal
+   * in force for the rest of the episode (the native macOS/Windows backends
+   * make no child-before-parent guarantee, so descendants can also arrive
+   * after the directory event).
+   */
+  const quarantinedDeleteRoots = new Set<string>();
+  const QUARANTINE_KEY_SEPARATOR = "\u0000";
+  const quarantineRootKey = (slug: string, journalKey: string): string =>
+    `${slug}${QUARANTINE_KEY_SEPARATOR}${journalKey}`;
+  const quarantineRootFor = (
+    slug: string,
+    journalKey: string,
+  ): string | null => {
+    for (const root of quarantinedDeleteRoots) {
+      const separator = root.indexOf(QUARANTINE_KEY_SEPARATOR);
+      if (root.slice(0, separator) !== slug) continue;
+      const rootKey = root.slice(separator + 1);
+      if (
+        rootKey === "" ||
+        journalKey === rootKey ||
+        journalKey.startsWith(`${rootKey}/`)
+      ) {
+        return root;
+      }
+    }
+    return null;
+  };
+
   const journalCoordinates = (
     relativePath: string,
   ): { slug: string; key: string } | null => {
@@ -536,6 +576,29 @@ export async function runWatchLoop(
       kind,
     );
 
+    /**
+     * Revoke every delete intent under this event, including ones a descendant
+     * `unlink` already persisted before the parent `unlinkDir` arrived. An
+     * intent-less disappearance is the safe state: the engine refuses it
+     * (`missing-delete-intent`) under every delete policy, nothing is deleted
+     * locally or remotely, and the next pull restores the subtree.
+     */
+    const revokeCoveredDeleteIntents = (): number => {
+      let revoked = 0;
+      for (const [key] of covered) {
+        if (clearLocalDeleteIntent(journal, key)) revoked += 1;
+      }
+      if (revoked > 0) writeJournal(coordinates.slug, journal);
+      return revoked;
+    };
+
+    // A refused directory removal covers its descendants for the rest of the
+    // episode, whichever order the backend reports them in.
+    if (quarantineRootFor(coordinates.slug, coordinates.key)) {
+      revokeCoveredDeleteIntents();
+      return [];
+    }
+
     // Bulk-intent quarantine: one directory event must not be able to mint
     // intents across a large fraction of the shard. Leaving them intent-less
     // is the safe state — the engine refuses intent-less disappearances and
@@ -552,12 +615,18 @@ export async function runWatchLoop(
         Object.keys(journal.files).length,
       )
     ) {
+      quarantinedDeleteRoots.add(
+        quarantineRootKey(coordinates.slug, coordinates.key),
+      );
+      const revoked = revokeCoveredDeleteIntents();
       process.stderr.write(
         `hq-sync-runner: refusing to record delete intent for ${stampable.length} ` +
           `of ${Object.keys(journal.files).length} journal entries under ` +
           `"${coordinates.key || "<root>"}" — a single directory removal that large ` +
           `is treated as a possible mirror loss. Nothing was deleted. ` +
-          `Set HQ_SYNC_DELETE_BULK_OVERRIDE=1 to propagate it deliberately.\n`,
+          `Revoked ${revoked} intent(s) already recorded by descendant unlink ` +
+          `events. Set HQ_SYNC_DELETE_BULK_OVERRIDE=1 to propagate it ` +
+          `deliberately.\n`,
       );
       return [];
     }
@@ -589,6 +658,21 @@ export async function runWatchLoop(
     for (const [absolutePath, change] of changes.entries()) {
       if (change.kind === "unlink" || change.kind === "unlinkDir") {
         for (const snapshot of change.deleteSnapshots ?? []) {
+          // Snapshots captured by descendant unlinks before the directory
+          // expansion was refused are part of the same quarantined episode.
+          // Re-stamping them here would hand the engine exactly the bulk
+          // delete the quarantine declined.
+          if (quarantineRootFor(snapshot.journalSlug, snapshot.journalPath)) {
+            if (
+              clearLocalDeleteIntent(
+                journalFor(snapshot.journalSlug),
+                snapshot.journalPath,
+              )
+            ) {
+              dirty.add(snapshot.journalSlug);
+            }
+            continue;
+          }
           let absent = false;
           try {
             fs.lstatSync(snapshot.absolutePath);
@@ -922,6 +1006,15 @@ export async function runWatchLoop(
     while (!stopped) {
       pollTick++;
       const pending = takePendingWatcherChange();
+      // The quarantine is scoped to the delete episode it refused, and only
+      // the roots already refused when this batch was taken belong to it.
+      // Snapshot them here rather than clearing the whole set after the pass:
+      // the pass is awaited, so a bulk removal observed while it runs adds its
+      // root afterwards and its snapshots ride the NEXT batch. Clearing
+      // unconditionally would drop that root before `prepareWatcherChanges`
+      // ever sees the batch, re-stamping the revoked descendant intents and
+      // handing the engine exactly the bulk delete the quarantine refused.
+      const drainedQuarantineRoots = new Set(quarantinedDeleteRoots);
       const shouldRunFull =
         pollTick === 1 ||
         pending.bare ||
@@ -930,6 +1023,12 @@ export async function runWatchLoop(
       const result = shouldRunFull
         ? await runGuarded(passArgv)
         : await runScopedDrain(pending.batch);
+      // This batch's episode has now been prepared and pushed, so a later
+      // removal under the same path is judged fresh — a root that never
+      // expired would silently refuse ordinary deletes from here on.
+      for (const root of drainedQuarantineRoots) {
+        quarantinedDeleteRoots.delete(root);
+      }
       const exitCode = passExitCode(result);
       if (exitCode === runtime.authRequiredPassExit) {
         // The pass already emitted auth-error. Stop the unattended loop but

package/src/bin/sync-runner.test.ts

@@ -37,6 +37,7 @@ import type {
 import {
   FakeClock,
   type LocalDeleteSnapshot,
+  type TreeChange,
   type TreeChangeBatch,
 } from "../watcher.js";
 import {
@@ -7547,7 +7548,11 @@ describe("US-002 — manifest reconciliation wiring", () => {
 describe("watcher delete intents without pre-seeded authorization", () => {
   function startDeleteHarness(
     keys: string[],
-    options: { gateInitialPull?: boolean } = {},
+    options: {
+      gateInitialPull?: boolean;
+      onPass?: (argv: string[]) => Promise<void> | void;
+      direction?: "both" | "push";
+    } = {},
   ) {
     const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-watch-delete-"));
     const companyRoot = path.join(hqRoot, "companies", "indigo");
@@ -7590,6 +7595,9 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     });
     let initialPullGated = options.gateInitialPull === true;
     const runPass = vi.fn(async (argv: string[]) => {
+      // Lets a test hold a pass mid-flight and act while the loop is awaiting
+      // it, which is where producer-side state that outlives a pass is decided.
+      await options.onPass?.(argv);
       const direction = argv[argv.indexOf("--direction") + 1];
       if (direction === "push") {
         const journal = readJournal("indigo");
@@ -7664,7 +7672,8 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     });
     const loop = runRunnerWithLoop(
       [
-        "--companies", "--watch", "--event-push", "--direction", "both",
+        "--companies", "--watch", "--event-push",
+        "--direction", options.direction ?? "both",
         "--hq-root", hqRoot, "--poll-remote-ms", "60000",
       ],
       {
@@ -8038,6 +8047,229 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     await stopHarness(h);
   });
 
+  // Chokidar (the Linux backend) reports every removed child through `unlink`
+  // BEFORE the parent `unlinkDir`. Each descendant therefore persists its own
+  // version-bound intent and hands a snapshot to the pending batch before the
+  // directory expansion is ever seen, so a quarantine that only declines the
+  // expansion leaves a whole `rm -rf` intent-backed — and
+  // `HQ_SYNC_DELETE_POLICY=all`, which bypasses the engine breaker entirely,
+  // would still delete the subtree without HQ_SYNC_DELETE_BULK_OVERRIDE.
+  it("unlink-before-unlinkDir: the trip revokes intents descendants already recorded", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    const intentCount = () =>
+      Object.values(readJournal("indigo").files).filter(
+        (entry) => entry.localDeleteIntent,
+      ).length;
+
+    const childChanges = new Map<string, TreeChange>();
+    for (const key of keys) {
+      const absolutePath = path.join(h.companyRoot, key);
+      fs.unlinkSync(absolutePath);
+      const snapshots = h.capture()(`companies/indigo/${key}`, "unlink");
+      expect(snapshots).toHaveLength(1);
+      childChanges.set(absolutePath, {
+        kind: "unlink",
+        deleteSnapshots: snapshots,
+      });
+    }
+    expect(intentCount()).toBe(12);
+
+    fs.rmSync(subtree, { recursive: true, force: true });
+    const dirSnapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    expect(dirSnapshots).toHaveLength(0);
+    expect(intentCount()).toBe(0);
+
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map<string, string>([
+        ...keys.map(
+          (key) =>
+            [
+              path.join(h.companyRoot, key),
+              `companies/indigo/${key}`,
+            ] as [string, string],
+        ),
+        [subtree, "companies/indigo/knowledge/moved"],
+      ]),
+      changes: new Map<string, TreeChange>([
+        ...childChanges,
+        [subtree, { kind: "unlinkDir", deleteSnapshots: dirSnapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+
+    // The batch still carries the descendants' snapshots; preparing it must
+    // not re-stamp the intents the trip revoked.
+    expect(intentCount()).toBe(0);
+    expect(h.remote.size).toBe(12);
+    expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
+    await stopHarness(h);
+  });
+
+  // The native macOS/Windows backends make no child-before-parent guarantee,
+  // so the refusal has to hold for descendants that arrive after it too.
+  it("a descendant unlink arriving after a refused unlinkDir records no intent", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    fs.rmSync(subtree, { recursive: true, force: true });
+
+    expect(
+      h.capture()("companies/indigo/knowledge/moved", "unlinkDir"),
+    ).toHaveLength(0);
+    expect(h.capture()(`companies/indigo/${keys[0]}`, "unlink")).toHaveLength(0);
+    expect(
+      readJournal("indigo").files[keys[0]]?.localDeleteIntent,
+    ).toBeUndefined();
+    await stopHarness(h);
+  });
+
+  // The quarantine must not reach past the episode it refused: an unexpired
+  // root would silently swallow every later delete under that path.
+  it("an ordinary delete under a previously refused root still propagates", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    fs.rmSync(subtree, { recursive: true, force: true });
+    const dirSnapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
+      changes: new Map([
+        [subtree, { kind: "unlinkDir", deleteSnapshots: dirSnapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(h.remote.size).toBe(12);
+
+    // Next episode: one ordinary file removal under the same directory.
+    const lonePath = path.join(h.companyRoot, keys[0]);
+    fs.mkdirSync(path.dirname(lonePath), { recursive: true });
+    fs.writeFileSync(lonePath, `synced:${keys[0]}`);
+    fs.unlinkSync(lonePath);
+    const snapshots = h.capture()(`companies/indigo/${keys[0]}`, "unlink");
+    expect(snapshots).toHaveLength(1);
+    h.watcher.emit(`companies/indigo/${keys[0]}`, {
+      paths: new Map([[lonePath, `companies/indigo/${keys[0]}`]]),
+      changes: new Map([
+        [lonePath, { kind: "unlink", deleteSnapshots: snapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(h.remote.has(keys[0])).toBe(false);
+    expect(h.remote.size).toBe(11);
+    await stopHarness(h);
+  });
+
+  // A pass is awaited, so a bulk removal observed while one is in flight
+  // registers its quarantine root only after that pass's batch was taken — its
+  // own snapshots ride the NEXT batch. Expiring the whole set once the pass
+  // returns would drop the root before `prepareWatcherChanges` ever sees that
+  // batch, re-stamping the revoked descendant intents and handing the engine
+  // the bulk delete the quarantine refused (remotely deleting the subtree under
+  // HQ_SYNC_DELETE_POLICY=all).
+  it("a bulk removal observed during an in-flight pass keeps its quarantine", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    let releaseGatedPush = () => {};
+    const gatedPush = new Promise<void>((resolve) => {
+      releaseGatedPush = resolve;
+    });
+    let gateArmed = false;
+    let gateReached = false;
+    // Push-only: a pull would restore the quarantined subtree (the documented
+    // recovery for an intent-less disappearance) before the batch carrying the
+    // episode is prepared, masking whether the refusal itself survived.
+    const h = startDeleteHarness([...keys, "knowledge/other.md"], {
+      direction: "push",
+      onPass: async (_argv) => {
+        if (!gateArmed) return;
+        gateArmed = false;
+        gateReached = true;
+        await gatedPush;
+      },
+    });
+    await settle();
+    const intentCount = () =>
+      Object.values(readJournal("indigo").files).filter(
+        (entry) => entry.localDeleteIntent,
+      ).length;
+
+    // An unrelated edit drains as a scoped push, which then blocks on the gate
+    // with its batch already taken from the pending buffer.
+    gateArmed = true;
+    const otherPath = path.join(h.companyRoot, "knowledge/other.md");
+    fs.writeFileSync(otherPath, "edited");
+    h.watcher.emit("companies/indigo/knowledge/other.md", {
+      paths: new Map([[otherPath, "companies/indigo/knowledge/other.md"]]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(gateReached).toBe(true);
+
+    // The whole subtree disappears while that pass is still in flight.
+    const childChanges = new Map<string, TreeChange>();
+    for (const key of keys) {
+      const absolutePath = path.join(h.companyRoot, key);
+      fs.unlinkSync(absolutePath);
+      childChanges.set(absolutePath, {
+        kind: "unlink",
+        deleteSnapshots: h.capture()(`companies/indigo/${key}`, "unlink"),
+      });
+    }
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    fs.rmSync(subtree, { recursive: true, force: true });
+    const dirSnapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    expect(dirSnapshots).toHaveLength(0);
+    expect(intentCount()).toBe(0);
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map<string, string>([
+        ...keys.map(
+          (key) =>
+            [
+              path.join(h.companyRoot, key),
+              `companies/indigo/${key}`,
+            ] as [string, string],
+        ),
+        [subtree, "companies/indigo/knowledge/moved"],
+      ]),
+      changes: new Map<string, TreeChange>([
+        ...childChanges,
+        [subtree, { kind: "unlinkDir", deleteSnapshots: dirSnapshots }],
+      ]),
+    } as TreeChangeBatch);
+
+    // Completing the overlapped pass must not expire the refusal recorded
+    // during it: the next drain prepares the batch that carries the episode.
+    releaseGatedPush();
+    await settle();
+    h.tick();
+    await settle();
+
+    expect(intentCount()).toBe(0);
+    for (const key of keys) expect(h.remote.has(key)).toBe(true);
+    await stopHarness(h);
+  });
+
   it("HQ_SYNC_DELETE_BULK_OVERRIDE=1 lets a whole-shard unlinkDir propagate", async () => {
     process.env.HQ_SYNC_DELETE_BULK_OVERRIDE = "1";
     try {