@indigoai-us/hq-cloud 6.14.18 → 6.14.20

package/src/journal.ts

@@ -607,9 +607,15 @@ export function markLocalDeleteIntent(
   relativePath: string,
   localHash: string,
   localKind: "file" | "symlink",
+  expectedRemoteEtag: string,
 ): boolean {
   const entry = journal.files[relativePath];
-  if (!entry?.remoteEtag || entry.hash !== localHash || entry.kind !== localKind) {
+  if (
+    !entry?.remoteEtag ||
+    entry.remoteEtag !== normalizeEtag(expectedRemoteEtag) ||
+    entry.hash !== localHash ||
+    entry.kind !== localKind
+  ) {
     return false;
   }
   entry.localDeleteIntent = {
@@ -621,6 +627,17 @@ export function markLocalDeleteIntent(
   return true;
 }
 
+/** Invalidate a prior delete authorization after a watcher-observed recreate. */
+export function clearLocalDeleteIntent(
+  journal: SyncJournal,
+  relativePath: string,
+): boolean {
+  const entry = journal.files[relativePath];
+  if (!entry?.localDeleteIntent) return false;
+  delete entry.localDeleteIntent;
+  return true;
+}
+
 /** True if the entry is a tombstone (set by `tombstoneEntry`). */
 export function isTombstone(entry: JournalEntry | undefined): boolean {
   return !!entry && typeof entry.removedAt === "string";

package/src/personal-vault.ts

@@ -106,6 +106,29 @@ export interface PersonalVaultOptions {
  */
 export const PERSONAL_VAULT_MANIFEST_KEY = "companies/manifest.yaml";
 
+/**
+ * Return true for a path where HQ materializes generated links into the
+ * release-owned `core/` surface. The caller must additionally establish that
+ * the entry is a symlink: regular files in these namespaces remain canonical
+ * core content and must continue to sync.
+ *
+ * The generated links point at canonical content in `personal/` or
+ * `core/packages/`. Reindex / package wiring may remove and recreate them, so
+ * treating the links themselves as personal-vault data creates a pull →
+ * reindex → pull oscillation.
+ */
+export function isGeneratedCoreMirrorKey(key: string): boolean {
+  const parts = key.split("\\").join("/").replace(/^\.\//, "").split("/");
+  return (
+    parts.length >= 3 &&
+    parts[0] === "core" &&
+    (parts[1] === "knowledge" ||
+      parts[1] === "policies" ||
+      parts[1] === "workers" ||
+      parts[1] === "settings")
+  );
+}
+
 /**
  * Release-owned scaffold and reindex-generated wrappers must never be
  * delete-propagated from the personal vault. They are recreated locally by a

package/src/qmd-reindex.test.ts

@@ -5,22 +5,44 @@
  * fake `exec` that records the calls and returns canned results — no real qmd
  * is invoked. We assert on the *sequence of qmd commands* the function would
  * run, which is the contract that keeps a teammate's index fresh after sync.
+ *
+ * The corruption-safety tests (lock serialization, timeout-kill handling, and
+ * SQLITE_CORRUPT quarantine) pin the fix for feedback_b9a369ff — the post-sync
+ * reindex racing an interactive qmd writer and corrupting the vector store.
  */
 
 import { describe, it, expect } from "vitest";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
-import { reindexAfterSync, type QmdExec } from "./qmd-reindex.js";
+import {
+  reindexAfterSync,
+  looksCorrupt,
+  resolveExecTimeoutMs,
+  DEFAULT_QMD_EXEC_TIMEOUT_MS,
+  type QmdExec,
+  type QmdExecResult,
+  type AcquireReindexLock,
+  type QuarantineCorruptIndex,
+} from "./qmd-reindex.js";
+
+/** A lock acquirer that always succeeds — keeps command-sequence tests pure. */
+const noLock: AcquireReindexLock = () => ({ release() {} });
 
 /** Build a fake exec that returns status 0 for everything and logs calls. */
-function fakeExec(opts?: { listStdout?: string; failOn?: string }): {
+function fakeExec(opts?: {
+  listStdout?: string;
+  failOn?: string;
+  resultFor?: (args: string[]) => Partial<QmdExecResult> | undefined;
+}): {
   exec: QmdExec;
   calls: string[][];
 } {
   const calls: string[][] = [];
   const exec: QmdExec = (args) => {
     calls.push(args);
+    const override = opts?.resultFor?.(args);
+    if (override) return { status: 0, stdout: "", ...override };
     if (opts?.failOn && args[0] === opts.failOn) return { status: 1, stdout: "" };
     if (args[0] === "collection" && args[1] === "list") {
       return { status: 0, stdout: opts?.listStdout ?? "" };
@@ -35,6 +57,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec();
     const r = reindexAfterSync("/tmp/not-hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => false, // core/core.yaml absent
     });
     expect(r.qmdAvailable).toBe(false);
@@ -45,6 +68,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ failOn: "collection" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true, // core.yaml present
     });
     expect(r.qmdAvailable).toBe(false);
@@ -57,6 +81,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ listStdout: "Collections (1):\n\nHQ (qmd://HQ/)\n" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true,
       readCompanies: () => ["acme"],
       hasIndexableMarkdown: () => true,
@@ -79,6 +104,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ listStdout: "acme (qmd://acme/)\n" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true,
       readCompanies: () => ["acme"],
       hasIndexableMarkdown: () => true,
@@ -92,6 +118,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ listStdout: "" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true,
       readCompanies: () => ["empty"],
       hasIndexableMarkdown: () => false,
@@ -104,6 +131,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ listStdout: "" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true,
       changedPaths: ["companies/acme/docs/readme.txt"],
     });
@@ -120,6 +148,7 @@ describe("reindexAfterSync", () => {
 
       const first = reindexAfterSync("/hq", {
         exec,
+        acquireReindexLock: noLock,
         existsSync: () => true,
         changedPaths: ["companies/acme/knowledge/note.md"],
         debounceMs: 60_000,
@@ -132,6 +161,7 @@ describe("reindexAfterSync", () => {
 
       const second = reindexAfterSync("/hq", {
         exec,
+        acquireReindexLock: noLock,
         existsSync: () => true,
         changedPaths: [],
         debounceMs: 60_000,
@@ -154,6 +184,7 @@ describe("reindexAfterSync", () => {
     const { exec, calls } = fakeExec({ listStdout: "" });
     const r = reindexAfterSync("/hq", {
       exec,
+      acquireReindexLock: noLock,
       existsSync: () => true,
       readCompanies: () => [],
       embed: true,
@@ -167,7 +198,7 @@ describe("reindexAfterSync", () => {
       throw new Error("boom");
     };
     expect(() =>
-      reindexAfterSync("/hq", { exec, existsSync: () => true }),
+      reindexAfterSync("/hq", { exec, acquireReindexLock: noLock, existsSync: () => true }),
     ).not.toThrow();
   });
 
@@ -183,11 +214,300 @@ describe("reindexAfterSync", () => {
       fs.writeFileSync(path.join(kdir, "note.md"), "# real\n");
 
       const { exec, calls } = fakeExec({ listStdout: "" });
-      const r = reindexAfterSync(hq, { exec }); // real fs walk for md detection
+      const r = reindexAfterSync(hq, { exec, acquireReindexLock: noLock }); // real fs walk
       expect(r.collectionsAdded).toEqual(["acme"]);
       expect(calls.some((c) => c[0] === "collection" && c[1] === "add")).toBe(true);
     } finally {
       fs.rmSync(dir, { recursive: true, force: true });
     }
   });
+
+  // ── Corruption safety: lock serialization (feedback_b9a369ff) ──────────────
+
+  it("skips the whole cycle without touching qmd when another writer holds the lock", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-lockbusy-"));
+    try {
+      const statePath = path.join(dir, "qmd-state.json");
+      const { exec, calls } = fakeExec({ listStdout: "" });
+      const r = reindexAfterSync("/hq", {
+        exec,
+        existsSync: () => true,
+        statePath,
+        // Simulate a concurrent qmd writer already holding the lock.
+        acquireReindexLock: () => null,
+      });
+      // The runner never runs a single qmd command while another writer holds
+      // the lock — this is what prevents the concurrent-writer corruption.
+      expect(calls).toEqual([]);
+      expect(r.lockBusy).toBe(true);
+      expect(r.pendingDirty).toBe(true);
+      expect(r.updated).toBe(false);
+      // The tree is marked dirty so the next sync retries once the lock frees.
+      const persisted = JSON.parse(fs.readFileSync(statePath, "utf8"));
+      expect(persisted.pendingDirty).toBe(true);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("always releases the lock, even when a qmd command throws", () => {
+    let released = false;
+    const exec: QmdExec = () => {
+      throw new Error("boom");
+    };
+    reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      acquireReindexLock: () => ({
+        release() {
+          released = true;
+        },
+      }),
+    });
+    expect(released).toBe(true);
+  });
+
+  // ── Corruption safety: timeout-kill handling ───────────────────────────────
+
+  it("treats a timed-out update as not-done, marks dirty, and does NOT embed", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-timeout-"));
+    try {
+      const statePath = path.join(dir, "qmd-state.json");
+      const { exec, calls } = fakeExec({
+        listStdout: "",
+        resultFor: (args) =>
+          args[0] === "update" ? { status: null, timedOut: true } : undefined,
+      });
+      const r = reindexAfterSync("/hq", {
+        exec,
+        acquireReindexLock: noLock,
+        existsSync: () => true,
+        readCompanies: () => [],
+        embed: true, // would normally run embed after update
+        statePath,
+      });
+      expect(r.timedOut).toBe(true);
+      expect(r.updated).toBe(false);
+      expect(r.embedded).toBe(false);
+      expect(r.pendingDirty).toBe(true);
+      // Crucially: no `embed` after a timed-out update — never write further.
+      expect(calls).toEqual([["collection", "list"], ["update"]]);
+      const persisted = JSON.parse(fs.readFileSync(statePath, "utf8"));
+      expect(persisted.pendingDirty).toBe(true);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("treats a timed-out embed as not-embedded and marks dirty", () => {
+    const { exec } = fakeExec({
+      listStdout: "",
+      resultFor: (args) =>
+        args[0] === "embed" ? { status: null, timedOut: true } : undefined,
+    });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      acquireReindexLock: noLock,
+      existsSync: () => true,
+      readCompanies: () => [],
+      embed: true,
+    });
+    expect(r.updated).toBe(true);
+    expect(r.embedded).toBe(false);
+    expect(r.timedOut).toBe(true);
+    expect(r.pendingDirty).toBe(true);
+  });
+
+  // ── Corruption safety: SQLITE_CORRUPT detection + quarantine ────────────────
+
+  it("quarantines the DB and stops writing when update reports SQLITE_CORRUPT", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-corrupt-"));
+    try {
+      const statePath = path.join(dir, "qmd-state.json");
+      const quarantined: Array<{ qmdDir: string; suffix: string }> = [];
+      const quarantine: QuarantineCorruptIndex = (qmdDir, suffix) => {
+        quarantined.push({ qmdDir, suffix });
+        return path.join(qmdDir, `index.sqlite.corrupt-${suffix}`);
+      };
+      const { exec, calls } = fakeExec({
+        listStdout: "",
+        resultFor: (args) =>
+          args[0] === "update"
+            ? { status: 1, stderr: "SqliteError: database disk image is malformed (SQLITE_CORRUPT)" }
+            : undefined,
+      });
+      const r = reindexAfterSync("/hq", {
+        exec,
+        acquireReindexLock: noLock,
+        existsSync: () => true,
+        readCompanies: () => [],
+        embed: true, // must NOT run after corruption
+        statePath,
+        quarantineCorruptIndex: quarantine,
+      });
+      expect(r.corruptionQuarantined).toBe(true);
+      expect(r.embedded).toBe(false);
+      expect(r.pendingDirty).toBe(true);
+      expect(calls).toEqual([["collection", "list"], ["update"]]); // no embed
+      expect(quarantined).toHaveLength(1);
+      expect(quarantined[0].qmdDir).toBe(path.join("/hq", ".qmd"));
+      const persisted = JSON.parse(fs.readFileSync(statePath, "utf8"));
+      expect(persisted.pendingDirty).toBe(true);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("quarantines on a corrupt collection-list probe before any write", () => {
+    const quarantined: string[] = [];
+    const { exec, calls } = fakeExec({
+      resultFor: (args) =>
+        args[0] === "collection" && args[1] === "list"
+          ? { status: 0, stdout: "database disk image is malformed" }
+          : undefined,
+    });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      acquireReindexLock: noLock,
+      existsSync: () => true,
+      quarantineCorruptIndex: (qmdDir) => {
+        quarantined.push(qmdDir);
+        return null;
+      },
+    });
+    expect(r.corruptionQuarantined).toBe(true);
+    expect(r.updated).toBe(false);
+    // Detected on the probe — never runs update/embed onto the corrupt store.
+    expect(calls).toEqual([["collection", "list"]]);
+    expect(quarantined).toEqual([path.join("/hq", ".qmd")]);
+  });
+
+  it("quarantines when embed reports SQLITE_CORRUPT", () => {
+    const quarantined: string[] = [];
+    const { exec, calls } = fakeExec({
+      listStdout: "",
+      resultFor: (args) =>
+        args[0] === "embed"
+          ? { status: 1, stderr: "code SQLITE_CORRUPT at store.js:1546" }
+          : undefined,
+    });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      acquireReindexLock: noLock,
+      existsSync: () => true,
+      readCompanies: () => [],
+      embed: true,
+      quarantineCorruptIndex: (qmdDir) => {
+        quarantined.push(qmdDir);
+        return null;
+      },
+    });
+    expect(r.corruptionQuarantined).toBe(true);
+    expect(r.embedded).toBe(false);
+    expect(calls).toEqual([["collection", "list"], ["update"], ["embed"]]);
+    expect(quarantined).toEqual([path.join("/hq", ".qmd")]);
+  });
+});
+
+// ── The default reindex lock (real filesystem) ───────────────────────────────
+
+describe("default reindex lock", () => {
+  it("holds the lock for the duration of the cycle and releases it after", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-lock-"));
+    try {
+      const hq = path.join(dir, "hq");
+      const kdir = path.join(hq, "companies", "acme", "knowledge");
+      fs.mkdirSync(kdir, { recursive: true });
+      fs.mkdirSync(path.join(hq, "core"), { recursive: true });
+      fs.writeFileSync(path.join(hq, "core", "core.yaml"), "v: 1\n");
+      fs.writeFileSync(path.join(kdir, "note.md"), "# real\n");
+
+      // While the cycle runs (mid-exec), a naive O_EXCL re-acquire of the same
+      // lock file must fail with EEXIST — proving the lock is genuinely held.
+      const lockPath = path.join(hq, ".qmd", ".reindex.lock");
+      let reAcquireBlockedWhileHeld = false;
+      const { exec } = fakeExec({ listStdout: "" });
+      const observingExec: QmdExec = (args) => {
+        if (args[0] === "update") {
+          try {
+            fs.closeSync(fs.openSync(lockPath, "wx"));
+            fs.unlinkSync(lockPath); // acquired — NOT what we want
+          } catch (err) {
+            reAcquireBlockedWhileHeld =
+              (err as NodeJS.ErrnoException).code === "EEXIST";
+          }
+        }
+        return exec(args);
+      };
+
+      const r = reindexAfterSync(hq, { exec: observingExec });
+      expect(r.updated).toBe(true);
+      expect(reAcquireBlockedWhileHeld).toBe(true);
+      // Released after the cycle — the lock file is gone.
+      expect(fs.existsSync(lockPath)).toBe(false);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("reclaims a lock held by a dead PID (stale takeover)", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-lock-stale-"));
+    try {
+      const hq = path.join(dir, "hq");
+      const kdir = path.join(hq, "companies", "acme", "knowledge");
+      fs.mkdirSync(kdir, { recursive: true });
+      fs.mkdirSync(path.join(hq, "core"), { recursive: true });
+      fs.writeFileSync(path.join(hq, "core", "core.yaml"), "v: 1\n");
+      fs.writeFileSync(path.join(kdir, "note.md"), "# real\n");
+
+      // Plant a lock owned by a PID that is certainly not alive.
+      const lockPath = path.join(hq, ".qmd", ".reindex.lock");
+      fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+      fs.writeFileSync(
+        lockPath,
+        JSON.stringify({ pid: 2147483646, startedAt: "2000-01-01T00:00:00.000Z", command: "dead" }),
+      );
+
+      const { exec, calls } = fakeExec({ listStdout: "" });
+      const r = reindexAfterSync(hq, { exec });
+      // The dead holder was reclaimed, so the reindex ran to completion.
+      expect(r.updated).toBe(true);
+      expect(r.lockBusy).toBe(false);
+      expect(calls.some((c) => c[0] === "update")).toBe(true);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});
+
+// ── Pure helpers ─────────────────────────────────────────────────────────────
+
+describe("resolveExecTimeoutMs", () => {
+  it("defaults to a generous bound, not the old 120s that killed writes", () => {
+    expect(resolveExecTimeoutMs({})).toBe(DEFAULT_QMD_EXEC_TIMEOUT_MS);
+    expect(DEFAULT_QMD_EXEC_TIMEOUT_MS).toBeGreaterThan(120_000);
+  });
+
+  it("honors a valid HQ_QMD_EXEC_TIMEOUT_MS override", () => {
+    expect(resolveExecTimeoutMs({ HQ_QMD_EXEC_TIMEOUT_MS: "300000" })).toBe(300_000);
+  });
+
+  it("ignores an invalid override", () => {
+    expect(resolveExecTimeoutMs({ HQ_QMD_EXEC_TIMEOUT_MS: "nope" })).toBe(DEFAULT_QMD_EXEC_TIMEOUT_MS);
+    expect(resolveExecTimeoutMs({ HQ_QMD_EXEC_TIMEOUT_MS: "-5" })).toBe(DEFAULT_QMD_EXEC_TIMEOUT_MS);
+    expect(resolveExecTimeoutMs({ HQ_QMD_EXEC_TIMEOUT_MS: "0" })).toBe(DEFAULT_QMD_EXEC_TIMEOUT_MS);
+  });
+});
+
+describe("looksCorrupt", () => {
+  it("matches the SQLITE_CORRUPT signature on stdout or stderr", () => {
+    expect(looksCorrupt({ stderr: "SqliteError: ... code SQLITE_CORRUPT" })).toBe(true);
+    expect(looksCorrupt({ stdout: "database disk image is malformed" })).toBe(true);
+    expect(looksCorrupt({ stdout: "Database disk image IS MALFORMED" })).toBe(true);
+  });
+
+  it("does not match clean output", () => {
+    expect(looksCorrupt({ stdout: "Updated 688 documents", stderr: "" })).toBe(false);
+    expect(looksCorrupt({})).toBe(false);
+  });
 });