@indigoai-us/hq-cloud 6.14.33 → 6.14.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cloud",
-  "version": "6.14.33",
+  "version": "6.14.35",
   "description": "HQ by Indigo cloud sync engine — bidirectional S3 sync for mobile access",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/bin/sync-runner-watch-loop.ts

@@ -136,6 +136,46 @@ export function journalEntriesForWatcherDelete(
   );
 }
 
+/**
+ * Bulk-intent quarantine thresholds. Deliberately the same shape and values as
+ * the engine's `BULK_ASYMMETRY_*` constants (`src/cli/share.ts`) so operators
+ * learn one rule and one knob: a disappearance covering >= 10 journal entries
+ * AND >= 10% of the shard is treated as suspect rather than deliberate.
+ */
+const WATCHER_BULK_INTENT_MIN_ABS = 10;
+const WATCHER_BULK_INTENT_RATIO = 0.1;
+
+/**
+ * Should a single `unlinkDir` expansion be quarantined instead of stamping a
+ * version-bound delete intent on every descendant journal entry?
+ *
+ * One filesystem event can cover an arbitrary subtree, so the producer is
+ * capable of minting thousands of individually-valid intents from a checkout
+ * swap, an unmount, a partial restore or an `rm -rf` — the 2026-07-30 mass
+ * deletion did exactly that. Quarantined entries are simply left intent-less,
+ * which the engine already refuses (`missing-delete-intent`); nothing is
+ * deleted locally or remotely and the next pull restores the tree.
+ *
+ * Pure so the threshold is unit-testable without a watcher or a journal.
+ */
+export function shouldQuarantineWatcherDeleteIntents(
+  expandedEntries: number,
+  journalEntries: number,
+): boolean {
+  if (expandedEntries < WATCHER_BULK_INTENT_MIN_ABS) return false;
+  if (journalEntries <= 0) return false;
+  return expandedEntries / journalEntries >= WATCHER_BULK_INTENT_RATIO;
+}
+
+/**
+ * Same operator escape hatch as the engine breaker — one knob for both layers.
+ * Set when a genuinely intentional large folder removal must propagate.
+ */
+function isBulkIntentOverride(): boolean {
+  const v = (process.env.HQ_SYNC_DELETE_BULK_OVERRIDE ?? "").toLowerCase();
+  return v === "1" || v === "true" || v === "yes";
+}
+
 function passExitCode(outcome: number | RunnerPassOutcome): number {
   return typeof outcome === "number" ? outcome : outcome.exitCode;
 }
@@ -428,6 +468,46 @@ export async function runWatchLoop(
     }
   };
 
+  /**
+   * Journal roots the bulk-intent quarantine has refused for the current
+   * delete episode, keyed `slug\0journalKey`.
+   *
+   * Chokidar (the Linux backend) reports every removed child through `unlink`
+   * BEFORE the parent `unlinkDir`, so by the time the directory expansion is
+   * refused each descendant has already persisted its own version-bound intent
+   * and handed a snapshot to the pending batch. Declining the expansion alone
+   * leaves those intents standing and `prepareWatcherChanges` re-stamps them
+   * from the snapshots, so a large `rm -rf` still propagates in full under
+   * `HQ_SYNC_DELETE_POLICY=all` — the path this producer-side layer exists to
+   * cover, because it bypasses the engine breaker entirely. The trip therefore
+   * revokes the descendants' intents as well, and this set keeps the refusal
+   * in force for the rest of the episode (the native macOS/Windows backends
+   * make no child-before-parent guarantee, so descendants can also arrive
+   * after the directory event).
+   */
+  const quarantinedDeleteRoots = new Set<string>();
+  const QUARANTINE_KEY_SEPARATOR = "\u0000";
+  const quarantineRootKey = (slug: string, journalKey: string): string =>
+    `${slug}${QUARANTINE_KEY_SEPARATOR}${journalKey}`;
+  const quarantineRootFor = (
+    slug: string,
+    journalKey: string,
+  ): string | null => {
+    for (const root of quarantinedDeleteRoots) {
+      const separator = root.indexOf(QUARANTINE_KEY_SEPARATOR);
+      if (root.slice(0, separator) !== slug) continue;
+      const rootKey = root.slice(separator + 1);
+      if (
+        rootKey === "" ||
+        journalKey === rootKey ||
+        journalKey.startsWith(`${rootKey}/`)
+      ) {
+        return root;
+      }
+    }
+    return null;
+  };
+
   const journalCoordinates = (
     relativePath: string,
   ): { slug: string; key: string } | null => {
@@ -490,11 +570,68 @@ export async function runWatchLoop(
     // Chokidar emits one unlink per file during rm -rf. Avoid walking the
     // whole shard for every exact-key event; directory unlinks alone need
     // descendant expansion.
-    for (const [key, entry] of journalEntriesForWatcherDelete(
+    const covered = journalEntriesForWatcherDelete(
       journal.files,
       coordinates.key,
       kind,
-    )) {
+    );
+
+    /**
+     * Revoke every delete intent under this event, including ones a descendant
+     * `unlink` already persisted before the parent `unlinkDir` arrived. An
+     * intent-less disappearance is the safe state: the engine refuses it
+     * (`missing-delete-intent`) under every delete policy, nothing is deleted
+     * locally or remotely, and the next pull restores the subtree.
+     */
+    const revokeCoveredDeleteIntents = (): number => {
+      let revoked = 0;
+      for (const [key] of covered) {
+        if (clearLocalDeleteIntent(journal, key)) revoked += 1;
+      }
+      if (revoked > 0) writeJournal(coordinates.slug, journal);
+      return revoked;
+    };
+
+    // A refused directory removal covers its descendants for the rest of the
+    // episode, whichever order the backend reports them in.
+    if (quarantineRootFor(coordinates.slug, coordinates.key)) {
+      revokeCoveredDeleteIntents();
+      return [];
+    }
+
+    // Bulk-intent quarantine: one directory event must not be able to mint
+    // intents across a large fraction of the shard. Leaving them intent-less
+    // is the safe state — the engine refuses intent-less disappearances and
+    // the next pull restores the subtree.
+    const stampable = covered.filter(
+      ([, entry]) =>
+        entry?.remoteEtag && entry.kind && entry.removedAt === undefined,
+    );
+    if (
+      kind === "unlinkDir" &&
+      !isBulkIntentOverride() &&
+      shouldQuarantineWatcherDeleteIntents(
+        stampable.length,
+        Object.keys(journal.files).length,
+      )
+    ) {
+      quarantinedDeleteRoots.add(
+        quarantineRootKey(coordinates.slug, coordinates.key),
+      );
+      const revoked = revokeCoveredDeleteIntents();
+      process.stderr.write(
+        `hq-sync-runner: refusing to record delete intent for ${stampable.length} ` +
+          `of ${Object.keys(journal.files).length} journal entries under ` +
+          `"${coordinates.key || "<root>"}" — a single directory removal that large ` +
+          `is treated as a possible mirror loss. Nothing was deleted. ` +
+          `Revoked ${revoked} intent(s) already recorded by descendant unlink ` +
+          `events. Set HQ_SYNC_DELETE_BULK_OVERRIDE=1 to propagate it ` +
+          `deliberately.\n`,
+      );
+      return [];
+    }
+
+    for (const [key, entry] of stampable) {
       captureEntry(key, entry);
     }
 
@@ -521,6 +658,21 @@ export async function runWatchLoop(
     for (const [absolutePath, change] of changes.entries()) {
       if (change.kind === "unlink" || change.kind === "unlinkDir") {
         for (const snapshot of change.deleteSnapshots ?? []) {
+          // Snapshots captured by descendant unlinks before the directory
+          // expansion was refused are part of the same quarantined episode.
+          // Re-stamping them here would hand the engine exactly the bulk
+          // delete the quarantine declined.
+          if (quarantineRootFor(snapshot.journalSlug, snapshot.journalPath)) {
+            if (
+              clearLocalDeleteIntent(
+                journalFor(snapshot.journalSlug),
+                snapshot.journalPath,
+              )
+            ) {
+              dirty.add(snapshot.journalSlug);
+            }
+            continue;
+          }
           let absent = false;
           try {
             fs.lstatSync(snapshot.absolutePath);
@@ -862,6 +1014,11 @@ export async function runWatchLoop(
       const result = shouldRunFull
         ? await runGuarded(passArgv)
         : await runScopedDrain(pending.batch);
+      // The quarantine is scoped to the delete episode it refused. The batch
+      // carrying that episode has now been prepared and pushed, so a later
+      // removal under the same path is judged fresh — a root that never
+      // expired would silently refuse ordinary deletes from here on.
+      quarantinedDeleteRoots.clear();
       const exitCode = passExitCode(result);
       if (exitCode === runtime.authRequiredPassExit) {
         // The pass already emitted auth-error. Stop the unattended loop but

package/src/bin/sync-runner.test.ts

@@ -37,11 +37,13 @@ import type {
 import {
   FakeClock,
   type LocalDeleteSnapshot,
+  type TreeChange,
   type TreeChangeBatch,
 } from "../watcher.js";
 import {
   adaptivePollMs,
   journalEntriesForWatcherDelete,
+  shouldQuarantineWatcherDeleteIntents,
 } from "./sync-runner-watch-loop.js";
 import {
   PERSONAL_VAULT_JOURNAL_SLUG,
@@ -7800,6 +7802,32 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     ).toEqual([["knowledge/exact.md", exactEntry]]);
   });
 
+  // Bulk-intent quarantine. A single `unlinkDir` expands to every descendant
+  // journal entry, so without a ceiling one filesystem event — a checkout
+  // swap, an unmount, a partial restore — mints thousands of individually
+  // valid delete intents. Same thresholds as the engine's bulk-asymmetry
+  // breaker so there is one rule and one override knob across both layers.
+  it.each([
+    // [expanded, journalEntries, quarantined, why]
+    [11, 100, true, "11% of the shard, above both thresholds"],
+    [10, 100, true, "exactly at MIN_ABS and RATIO"],
+    [9, 100, false, "below MIN_ABS — ordinary cleanup must propagate"],
+    [9, 9, false, "100% ratio but below MIN_ABS floor"],
+    [10, 1000, false, "at MIN_ABS but only 1% of the shard"],
+    [50, 50, true, "the whole shard vanished at once"],
+    [0, 0, false, "empty journal never quarantines"],
+  ])(
+    "bulk-intent quarantine for %i of %i journal entries → %s (%s)",
+    (expanded, journalEntries, quarantined) => {
+      expect(
+        shouldQuarantineWatcherDeleteIntents(
+          expanded as number,
+          journalEntries as number,
+        ),
+      ).toBe(quarantined);
+    },
+  );
+
   it("keeps a deleted top-level personal file in the scoped push delete roots", async () => {
     const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-personal-delete-"));
     const key = "personal-note.md";
@@ -7974,17 +8002,219 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     await stopHarness(h);
   });
 
-  it("unlinkDir records every tracked descendant across a 12-file breaker-sized subtree", async () => {
+  // Replaces #227's "unlinkDir records every tracked descendant across a
+  // 12-file breaker-sized subtree", which asserted that one directory event
+  // may mint an intent for the entire shard and wipe it remotely. That is the
+  // shape of the 2026-07-30 incident (1,108 files in one push), and the test
+  // name conceded it was breaker-sized. Quarantining is now the contract; the
+  // three tests below cover the refusal, the operator escape hatch, and the
+  // ordinary folder removal that must keep working.
+  it("unlinkDir covering the whole shard is quarantined: no intents, nothing deleted", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    const snapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    expect(snapshots).toHaveLength(0);
+    expect(
+      Object.values(readJournal("indigo").files).filter(
+        (entry) => entry.localDeleteIntent,
+      ),
+    ).toHaveLength(0);
+    fs.rmSync(subtree, { recursive: true, force: true });
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
+      changes: new Map([
+        [subtree, { kind: "unlinkDir", deleteSnapshots: snapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(h.remote.size).toBe(12);
+    expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
+    await stopHarness(h);
+  });
+
+  // Chokidar (the Linux backend) reports every removed child through `unlink`
+  // BEFORE the parent `unlinkDir`. Each descendant therefore persists its own
+  // version-bound intent and hands a snapshot to the pending batch before the
+  // directory expansion is ever seen, so a quarantine that only declines the
+  // expansion leaves a whole `rm -rf` intent-backed — and
+  // `HQ_SYNC_DELETE_POLICY=all`, which bypasses the engine breaker entirely,
+  // would still delete the subtree without HQ_SYNC_DELETE_BULK_OVERRIDE.
+  it("unlink-before-unlinkDir: the trip revokes intents descendants already recorded", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    const intentCount = () =>
+      Object.values(readJournal("indigo").files).filter(
+        (entry) => entry.localDeleteIntent,
+      ).length;
+
+    const childChanges = new Map<string, TreeChange>();
+    for (const key of keys) {
+      const absolutePath = path.join(h.companyRoot, key);
+      fs.unlinkSync(absolutePath);
+      const snapshots = h.capture()(`companies/indigo/${key}`, "unlink");
+      expect(snapshots).toHaveLength(1);
+      childChanges.set(absolutePath, {
+        kind: "unlink",
+        deleteSnapshots: snapshots,
+      });
+    }
+    expect(intentCount()).toBe(12);
+
+    fs.rmSync(subtree, { recursive: true, force: true });
+    const dirSnapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    expect(dirSnapshots).toHaveLength(0);
+    expect(intentCount()).toBe(0);
+
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map<string, string>([
+        ...keys.map(
+          (key) =>
+            [
+              path.join(h.companyRoot, key),
+              `companies/indigo/${key}`,
+            ] as [string, string],
+        ),
+        [subtree, "companies/indigo/knowledge/moved"],
+      ]),
+      changes: new Map<string, TreeChange>([
+        ...childChanges,
+        [subtree, { kind: "unlinkDir", deleteSnapshots: dirSnapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+
+    // The batch still carries the descendants' snapshots; preparing it must
+    // not re-stamp the intents the trip revoked.
+    expect(intentCount()).toBe(0);
+    expect(h.remote.size).toBe(12);
+    expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
+    await stopHarness(h);
+  });
+
+  // The native macOS/Windows backends make no child-before-parent guarantee,
+  // so the refusal has to hold for descendants that arrive after it too.
+  it("a descendant unlink arriving after a refused unlinkDir records no intent", async () => {
+    const keys = Array.from(
+      { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const h = startDeleteHarness(keys);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
+    fs.rmSync(subtree, { recursive: true, force: true });
+
+    expect(
+      h.capture()("companies/indigo/knowledge/moved", "unlinkDir"),
+    ).toHaveLength(0);
+    expect(h.capture()(`companies/indigo/${keys[0]}`, "unlink")).toHaveLength(0);
+    expect(
+      readJournal("indigo").files[keys[0]]?.localDeleteIntent,
+    ).toBeUndefined();
+    await stopHarness(h);
+  });
+
+  // The quarantine must not reach past the episode it refused: an unexpired
+  // root would silently swallow every later delete under that path.
+  it("an ordinary delete under a previously refused root still propagates", async () => {
     const keys = Array.from(
       { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
     );
     const h = startDeleteHarness(keys);
     await settle();
     const subtree = path.join(h.companyRoot, "knowledge/moved");
+    fs.rmSync(subtree, { recursive: true, force: true });
+    const dirSnapshots = h.capture()(
+      "companies/indigo/knowledge/moved", "unlinkDir",
+    );
+    h.watcher.emit("companies/indigo/knowledge/moved", {
+      paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
+      changes: new Map([
+        [subtree, { kind: "unlinkDir", deleteSnapshots: dirSnapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(h.remote.size).toBe(12);
+
+    // Next episode: one ordinary file removal under the same directory.
+    const lonePath = path.join(h.companyRoot, keys[0]);
+    fs.mkdirSync(path.dirname(lonePath), { recursive: true });
+    fs.writeFileSync(lonePath, `synced:${keys[0]}`);
+    fs.unlinkSync(lonePath);
+    const snapshots = h.capture()(`companies/indigo/${keys[0]}`, "unlink");
+    expect(snapshots).toHaveLength(1);
+    h.watcher.emit(`companies/indigo/${keys[0]}`, {
+      paths: new Map([[lonePath, `companies/indigo/${keys[0]}`]]),
+      changes: new Map([
+        [lonePath, { kind: "unlink", deleteSnapshots: snapshots }],
+      ]),
+    } as TreeChangeBatch);
+    h.tick();
+    await settle();
+    expect(h.remote.has(keys[0])).toBe(false);
+    expect(h.remote.size).toBe(11);
+    await stopHarness(h);
+  });
+
+  it("HQ_SYNC_DELETE_BULK_OVERRIDE=1 lets a whole-shard unlinkDir propagate", async () => {
+    process.env.HQ_SYNC_DELETE_BULK_OVERRIDE = "1";
+    try {
+      const keys = Array.from(
+        { length: 12 }, (_, i) => `knowledge/moved/f-${i}.md`,
+      );
+      const h = startDeleteHarness(keys);
+      await settle();
+      const subtree = path.join(h.companyRoot, "knowledge/moved");
+      const snapshots = h.capture()(
+        "companies/indigo/knowledge/moved", "unlinkDir",
+      );
+      expect(snapshots).toHaveLength(12);
+      fs.rmSync(subtree, { recursive: true, force: true });
+      h.watcher.emit("companies/indigo/knowledge/moved", {
+        paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
+        changes: new Map([
+          [subtree, { kind: "unlinkDir", deleteSnapshots: snapshots }],
+        ]),
+      } as TreeChangeBatch);
+      h.tick();
+      await settle();
+      expect(h.remote.size).toBe(0);
+      expect(Object.keys(readJournal("indigo").files)).toHaveLength(0);
+      await stopHarness(h);
+    } finally {
+      delete process.env.HQ_SYNC_DELETE_BULK_OVERRIDE;
+    }
+  });
+
+  // The quarantine must not touch ordinary folder cleanup — that is the
+  // delete-resurrection bug #227 was written to fix, and it stays fixed.
+  it("an ordinary folder removal (10 of 120 entries) still records intents and propagates", async () => {
+    const removed = Array.from(
+      { length: 10 }, (_, i) => `knowledge/moved/f-${i}.md`,
+    );
+    const kept = Array.from(
+      { length: 110 }, (_, i) => `knowledge/keep/k-${i}.md`,
+    );
+    const h = startDeleteHarness([...removed, ...kept]);
+    await settle();
+    const subtree = path.join(h.companyRoot, "knowledge/moved");
     const snapshots = h.capture()(
       "companies/indigo/knowledge/moved", "unlinkDir",
     );
-    expect(snapshots).toHaveLength(12);
+    expect(snapshots).toHaveLength(10);
     fs.rmSync(subtree, { recursive: true, force: true });
     h.watcher.emit("companies/indigo/knowledge/moved", {
       paths: new Map([[subtree, "companies/indigo/knowledge/moved"]]),
@@ -7994,8 +8224,8 @@ describe("watcher delete intents without pre-seeded authorization", () => {
     } as TreeChangeBatch);
     h.tick();
     await settle();
-    expect(h.remote.size).toBe(0);
-    expect(Object.keys(readJournal("indigo").files)).toHaveLength(0);
+    for (const key of removed) expect(h.remote.has(key)).toBe(false);
+    expect(h.remote.size).toBe(110);
     await stopHarness(h);
   });