@indigoai-us/hq-cloud 6.13.2 → 6.13.4

package/src/qmd-reindex.ts

@@ -45,9 +45,24 @@ const defaultExec: QmdExec = (args) => {
   return { status: res.status, stdout: res.stdout ?? "" };
 };
 
+const DEFAULT_CHANGED_PATH_DEBOUNCE_MS = 60_000;
+
 export interface ReindexOptions {
   /** Rebuild embeddings too (slow). Default false — lexical-only. */
   embed?: boolean;
+  /**
+   * HQ-relative paths changed by sync. When omitted, preserves the historical
+   * behavior and runs QMD for any sync caller that invokes this function.
+   */
+  changedPaths?: string[];
+  /** Force a collection registration refresh even when content is not dirty. */
+  forceCollectionRefresh?: boolean;
+  /** Delay dirty updates by this many ms while keeping a persistent marker. */
+  debounceMs?: number;
+  /** Clock override for deterministic debounce tests. */
+  nowMs?: number;
+  /** State file override for tests. */
+  statePath?: string;
   /** Command runner override for tests. */
   exec?: QmdExec;
   /** `existsSync` override for tests. */
@@ -74,15 +89,42 @@ export interface ReindexOptions {
 export function reindexAfterSync(
   hqRoot: string,
   opts: ReindexOptions = {},
-): { qmdAvailable: boolean; collectionsAdded: string[]; updated: boolean; embedded: boolean } {
+): {
+  qmdAvailable: boolean;
+  collectionsAdded: string[];
+  updated: boolean;
+  embedded: boolean;
+  pendingDirty: boolean;
+} {
   const exec = opts.exec ?? defaultExec;
   const existsSync = opts.existsSync ?? fs.existsSync;
-  const result = { qmdAvailable: false, collectionsAdded: [] as string[], updated: false, embedded: false };
+  const result = {
+    qmdAvailable: false,
+    collectionsAdded: [] as string[],
+    updated: false,
+    embedded: false,
+    pendingDirty: false,
+  };
 
   try {
     // Guard: only operate on a real HQ tree.
     if (!existsSync(path.join(hqRoot, "core", "core.yaml"))) return result;
 
+    const statePath = opts.statePath ?? path.join(hqRoot, ".hq", "qmd-reindex-state.json");
+    const state = readState(statePath);
+    const changedPaths = opts.changedPaths;
+    const dirtyFromChanges =
+      changedPaths === undefined ||
+      changedPaths.some(isKnowledgeMarkdownPath);
+    const registrationMayBeStale =
+      opts.forceCollectionRefresh === true ||
+      changedPaths === undefined ||
+      dirtyFromChanges;
+    const pendingDirty = state.pendingDirty === true;
+    if (!dirtyFromChanges && !registrationMayBeStale && !pendingDirty) {
+      return result;
+    }
+
     // Guard: qmd must be installed. `qmd collection list` doubles as the
     // availability probe AND the source for which collections already exist.
     const list = exec(["collection", "list"]);
@@ -91,26 +133,62 @@ export function reindexAfterSync(
     const existingCollections = list.stdout;
 
     // 1. Auto-register missing company knowledge collections.
-    const companiesDir = path.join(hqRoot, "companies");
-    const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
-    for (const slug of slugs) {
-      const knowledgeDir = path.join(companiesDir, slug, "knowledge");
-      if (!existsSync(knowledgeDir)) continue;
-      const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
-      if (!hasMd) continue;
-      // Already registered? qmd collection URIs look like `qmd://<slug>/`.
-      if (existingCollections.includes(`qmd://${slug}/`)) continue;
-
-      const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
-      if (add.status === 0) {
-        exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
-        result.collectionsAdded.push(slug);
+    if (registrationMayBeStale) {
+      const companiesDir = path.join(hqRoot, "companies");
+      const slugs = (opts.readCompanies ?? defaultReadCompanies)(companiesDir);
+      for (const slug of slugs) {
+        const knowledgeDir = path.join(companiesDir, slug, "knowledge");
+        if (!existsSync(knowledgeDir)) continue;
+        const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
+        if (!hasMd) continue;
+        // Already registered? qmd collection URIs look like `qmd://<slug>/`.
+        if (existingCollections.includes(`qmd://${slug}/`)) continue;
+
+        const add = exec(["collection", "add", knowledgeDir, "--name", slug, "--mask", "**/*.md"]);
+        if (add.status === 0) {
+          exec(["context", "add", `qmd://${slug}`, `Knowledge base for ${slug}.`]);
+          result.collectionsAdded.push(slug);
+        }
       }
     }
 
     // 2. Incremental lexical reindex.
-    const update = exec(["update"]);
-    result.updated = update.status === 0;
+    const shouldUpdate = dirtyFromChanges || pendingDirty;
+    if (shouldUpdate) {
+      const nowMs = opts.nowMs ?? Date.now();
+      const debounceMs =
+        opts.debounceMs ??
+        (changedPaths === undefined ? 0 : DEFAULT_CHANGED_PATH_DEBOUNCE_MS);
+      const pendingSinceMs =
+        pendingDirty && typeof state.pendingSinceMs === "number"
+          ? state.pendingSinceMs
+          : nowMs;
+      if (debounceMs > 0 && nowMs - pendingSinceMs < debounceMs) {
+        writeState(statePath, {
+          ...state,
+          pendingDirty: true,
+          pendingSinceMs,
+        });
+        result.pendingDirty = true;
+        return result;
+      }
+
+      const update = exec(["update"]);
+      result.updated = update.status === 0;
+      if (result.updated) {
+        writeState(statePath, {
+          pendingDirty: false,
+          lastSuccessMs: nowMs,
+        });
+      } else {
+        writeState(statePath, {
+          ...state,
+          pendingDirty: true,
+          pendingSinceMs,
+        });
+        result.pendingDirty = true;
+      }
+    }
 
     // 3. Embeddings only on explicit request.
     if (opts.embed) {
@@ -134,6 +212,34 @@ export function reindexAfterSync(
   return result;
 }
 
+interface QmdReindexState {
+  pendingDirty?: boolean;
+  pendingSinceMs?: number;
+  lastSuccessMs?: number;
+}
+
+function readState(statePath: string): QmdReindexState {
+  try {
+    return JSON.parse(fs.readFileSync(statePath, "utf8")) as QmdReindexState;
+  } catch {
+    return {};
+  }
+}
+
+function writeState(statePath: string, state: QmdReindexState): void {
+  try {
+    fs.mkdirSync(path.dirname(statePath), { recursive: true });
+    fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + "\n");
+  } catch {
+    // State is an optimization/guarantee aid; QMD failures must not fail sync.
+  }
+}
+
+function isKnowledgeMarkdownPath(relPath: string): boolean {
+  const normalized = relPath.split(path.sep).join("/");
+  return /^companies\/[^/]+\/knowledge\/.+\.md$/i.test(normalized);
+}
+
 function defaultReadCompanies(companiesDir: string): string[] {
   try {
     return fs