@indigoai-us/hq-cloud 5.39.0 → 5.41.0

package/src/qmd-reindex.test.ts

@@ -0,0 +1,143 @@
+/**
+ * Unit tests for reindexAfterSync.
+ *
+ * The function shells out to the global `qmd` binary, so every test injects a
+ * fake `exec` that records the calls and returns canned results — no real qmd
+ * is invoked. We assert on the *sequence of qmd commands* the function would
+ * run, which is the contract that keeps a teammate's index fresh after sync.
+ */
+
+import { describe, it, expect } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { reindexAfterSync, type QmdExec } from "./qmd-reindex.js";
+
+/** Build a fake exec that returns status 0 for everything and logs calls. */
+function fakeExec(opts?: { listStdout?: string; failOn?: string }): {
+  exec: QmdExec;
+  calls: string[][];
+} {
+  const calls: string[][] = [];
+  const exec: QmdExec = (args) => {
+    calls.push(args);
+    if (opts?.failOn && args[0] === opts.failOn) return { status: 1, stdout: "" };
+    if (args[0] === "collection" && args[1] === "list") {
+      return { status: 0, stdout: opts?.listStdout ?? "" };
+    }
+    return { status: 0, stdout: "" };
+  };
+  return { exec, calls };
+}
+
+describe("reindexAfterSync", () => {
+  it("no-ops when the path is not an HQ root", () => {
+    const { exec, calls } = fakeExec();
+    const r = reindexAfterSync("/tmp/not-hq", {
+      exec,
+      existsSync: () => false, // core/core.yaml absent
+    });
+    expect(r.qmdAvailable).toBe(false);
+    expect(calls).toHaveLength(0);
+  });
+
+  it("no-ops when qmd is unavailable (collection list errors)", () => {
+    const { exec, calls } = fakeExec({ failOn: "collection" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true, // core.yaml present
+    });
+    expect(r.qmdAvailable).toBe(false);
+    expect(r.updated).toBe(false);
+    // Only the availability probe ran, then bailed.
+    expect(calls).toEqual([["collection", "list"]]);
+  });
+
+  it("registers a missing company collection then runs an incremental update", () => {
+    const { exec, calls } = fakeExec({ listStdout: "Collections (1):\n\nHQ (qmd://HQ/)\n" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      readCompanies: () => ["acme"],
+      hasIndexableMarkdown: () => true,
+    });
+    expect(r.qmdAvailable).toBe(true);
+    expect(r.collectionsAdded).toEqual(["acme"]);
+    expect(r.updated).toBe(true);
+    expect(r.embedded).toBe(false);
+
+    // Expected command sequence: probe, add, context, update.
+    expect(calls).toEqual([
+      ["collection", "list"],
+      ["collection", "add", path.join("/hq", "companies", "acme", "knowledge"), "--name", "acme", "--mask", "**/*.md"],
+      ["context", "add", "qmd://acme", "Knowledge base for acme."],
+      ["update"],
+    ]);
+  });
+
+  it("skips collections that already exist", () => {
+    const { exec, calls } = fakeExec({ listStdout: "acme (qmd://acme/)\n" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      readCompanies: () => ["acme"],
+      hasIndexableMarkdown: () => true,
+    });
+    expect(r.collectionsAdded).toEqual([]);
+    // No `collection add` — straight to update.
+    expect(calls).toEqual([["collection", "list"], ["update"]]);
+  });
+
+  it("skips company dirs with no indexable markdown", () => {
+    const { exec, calls } = fakeExec({ listStdout: "" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      readCompanies: () => ["empty"],
+      hasIndexableMarkdown: () => false,
+    });
+    expect(r.collectionsAdded).toEqual([]);
+    expect(calls).toEqual([["collection", "list"], ["update"]]);
+  });
+
+  it("runs embed only when embed:true", () => {
+    const { exec, calls } = fakeExec({ listStdout: "" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      readCompanies: () => [],
+      embed: true,
+    });
+    expect(r.embedded).toBe(true);
+    expect(calls).toEqual([["collection", "list"], ["update"], ["embed"]]);
+  });
+
+  it("never throws even if exec misbehaves", () => {
+    const exec: QmdExec = () => {
+      throw new Error("boom");
+    };
+    expect(() =>
+      reindexAfterSync("/hq", { exec, existsSync: () => true }),
+    ).not.toThrow();
+  });
+
+  it("hasIndexableMarkdown finds a real .md and ignores INDEX.md", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-test-"));
+    try {
+      const hq = path.join(dir, "hq");
+      const kdir = path.join(hq, "companies", "acme", "knowledge");
+      fs.mkdirSync(kdir, { recursive: true });
+      fs.mkdirSync(path.join(hq, "core"), { recursive: true });
+      fs.writeFileSync(path.join(hq, "core", "core.yaml"), "hqVersion: 1\n");
+      fs.writeFileSync(path.join(kdir, "INDEX.md"), "# index\n");
+      fs.writeFileSync(path.join(kdir, "note.md"), "# real\n");
+
+      const { exec, calls } = fakeExec({ listStdout: "" });
+      const r = reindexAfterSync(hq, { exec }); // real fs walk for md detection
+      expect(r.collectionsAdded).toEqual(["acme"]);
+      expect(calls.some((c) => c[0] === "collection" && c[1] === "add")).toBe(true);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/qmd-reindex.ts

@@ -0,0 +1,151 @@
+/**
+ * Post-sync qmd reindex.
+ *
+ * Why this lives in the runner (not just an HQ-core script): the qmd search
+ * index is a per-machine local store, and nothing re-indexed it after a sync
+ * pulled new files in — so teammates saw divergent search results depending
+ * on who ran `qmd update` most recently, and newly-synced knowledge folders
+ * weren't searchable until someone manually registered them as a collection.
+ *
+ * The runner ships via `npx @indigoai-us/hq-cloud@latest` (both the AppBar
+ * menubar and the `/hq-sync` CLI pull it at runtime), so putting the fix here
+ * reaches every teammate on their next sync WITHOUT requiring them to update
+ * their HQ core. It is therefore intentionally self-contained: it shells out
+ * to the globally-installed `qmd` binary directly and does NOT depend on any
+ * script inside the synced HQ tree (which may be stale).
+ *
+ * What it does, best-effort and idempotent:
+ *   1. Auto-registers any `companies/<slug>/knowledge` dir that isn't yet a
+ *      qmd collection (kills the manual "map" step).
+ *   2. Runs an incremental lexical `qmd update` (fast — qmd skips unchanged
+ *      files by mtime).
+ *   3. Rebuilds embeddings only when `embed: true` (slow on a multi-GB
+ *      index; meant for an idle pass, not every sync).
+ *
+ * The index itself is never synced — it is large, binary, and embeds absolute
+ * local paths. Only its *freshness* is automated here.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { spawnSync } from "child_process";
+
+/** Injectable command runner — real `spawnSync` in prod, a fake in tests. */
+export interface QmdExec {
+  (args: string[]): { status: number | null; stdout: string };
+}
+
+const defaultExec: QmdExec = (args) => {
+  const res = spawnSync("qmd", args, {
+    encoding: "utf8",
+    // qmd update on a large index can take a while; bound it so a wedged
+    // index never hangs the runner forever.
+    timeout: 120_000,
+  });
+  return { status: res.status, stdout: res.stdout ?? "" };
+};
+
+export interface ReindexOptions {
+  /** Rebuild embeddings too (slow). Default false — lexical-only. */
+  embed?: boolean;
+  /** Command runner override for tests. */
+  exec?: QmdExec;
+  /** `existsSync` override for tests. */
+  existsSync?: (p: string) => boolean;
+  /** `readdirSync` override for tests (returns subdir names of companies/). */
+  readCompanies?: (companiesDir: string) => string[];
+  /** Returns true if the knowledge dir has at least one indexable .md file. */
+  hasIndexableMarkdown?: (knowledgeDir: string) => boolean;
+}
+
+/**
+ * Reindex qmd for an HQ tree after a sync. Never throws — all failures are
+ * swallowed so a reindex problem can never mask or fail the sync result.
+ *
+ * @returns a small summary for logging/telemetry (collectionsAdded, updated).
+ */
+export function reindexAfterSync(
+  hqRoot: string,
+  opts: ReindexOptions = {},
+): { qmdAvailable: boolean; collectionsAdded: string[]; updated: boolean; embedded: boolean } {
+  const exec = opts.exec ?? defaultExec;
+  const existsSync = opts.existsSync ?? fs.existsSync;
+  const result = { qmdAvailable: false, collectionsAdded: [] as string[], updated: false, embedded: false };
+
+  try {
+    // Guard: only operate on a real HQ tree.
+    if (!existsSync(path.join(hqRoot, "core", "core.yaml"))) return result;
+
+    // Guard: qmd must be installed. `qmd collection list` doubles as the
+    // availability probe AND the source for which collections already exist.
+    const list = exec(["collection", "list"]);
+    if (list.status !== 0) return result; // qmd absent or errored — no-op
+    result.qmdAvailable = true;
+    const existingCollections = list.stdout;
+
+    // 1. Auto-register missing company knowledge collections.
+    const companiesDir = path.join(hqRoot, "companies");
+    const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
+    for (const slug of slugs) {
+      const knowledgeDir = path.join(companiesDir, slug, "knowledge");
+      if (!existsSync(knowledgeDir)) continue;
+      const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
+      if (!hasMd) continue;
+      // Already registered? qmd collection URIs look like `qmd://<slug>/`.
+      if (existingCollections.includes(`qmd://${slug}/`)) continue;
+
+      const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
+      if (add.status === 0) {
+        exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
+        result.collectionsAdded.push(slug);
+      }
+    }
+
+    // 2. Incremental lexical reindex.
+    const update = exec(["update"]);
+    result.updated = update.status === 0;
+
+    // 3. Embeddings only on explicit request.
+    if (opts.embed) {
+      const embed = exec(["embed"]);
+      result.embedded = embed.status === 0;
+    }
+  } catch {
+    // Reindex is best-effort; the sync result is authoritative. Swallow.
+  }
+
+  return result;
+}
+
+function defaultReadCompanies(companiesDir: string): string[] {
+  try {
+    return fs
+      .readdirSync(companiesDir, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && !e.name.startsWith("_") && !e.name.startsWith("."))
+      .map((e) => e.name);
+  } catch {
+    return [];
+  }
+}
+
+function defaultHasIndexableMarkdown(knowledgeDir: string): boolean {
+  // Shallow check is enough to decide "is this collection worth registering":
+  // walk a bounded depth looking for any .md that isn't INDEX.md.
+  const stack: Array<{ dir: string; depth: number }> = [{ dir: knowledgeDir, depth: 0 }];
+  while (stack.length) {
+    const { dir, depth } = stack.pop()!;
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const e of entries) {
+      if (e.isFile() && e.name.endsWith(".md") && e.name !== "INDEX.md") return true;
+      if (e.isDirectory() && depth < 4 && !e.name.startsWith(".")) {
+        stack.push({ dir: path.join(dir, e.name), depth: depth + 1 });
+      }
+    }
+  }
+  return false;
+}