karajan-code 4.70.4 → 5.0.1

package/src/commands/rag.js

@@ -12,6 +12,8 @@ import { fileIndexState } from "../rag/coverage.js";
 import { query } from "../rag/retriever.js";
 import { pathNamedHits, withPathNamedHits } from "../rag/path-hits.js";
 import { installPostMergeHook, maybeAutoUpdate } from "../rag/auto-update.js";
+import { noteDeadline } from "../rag/bootstrap-index.js";
+import { claimLease, indexLeasePath, releaseLease } from "../rag/pid-lease.js";
 import { indexLibrary, LIBRARY_PROJECT } from "../rag/library.js";
 import { compareToBaseline, loadGoldenQueries, runEval } from "../rag/eval.js";
 import { getKarajanHome } from "../utils/paths.js";
@@ -47,8 +49,35 @@ async function indexFiles({ files, projectDir, db, embedder, slug, logger, json,
   return totals;
 }
 
-export async function ragIndexCommand({ config, logger, flags = {} }) {
+// HU-3092: the background indexer is given the time it may stop at; unset, non-numeric or <= 0 means none.
+const deadlineFromEnv = () => { const n = Number(process.env.KJ_RAG_DEADLINE); return Number.isFinite(n) && n > 0 ? n : null; };
+
+// The partial index keeps its chunks but not its commit: the marker says why it stopped, the warning how to finish.
+function reportDeadline({ totals, projectDir, logger, json }) {
+  noteDeadline(projectDir);
+  logger.warn?.(`[rag] KJ_RAG_DEADLINE reached after ${totals.files} file(s): the index is partial and its commit is not recorded; run kj rag index --with-sources by hand`);
+  if (json) process.stdout.write(`${JSON.stringify(totals)}\n`);
+  return totals;
+}
+
+// HU-3116: the cooperative stop only looks between files; a hung embed call is ended this long after the deadline.
+// It stays inside DEADLINE_MARGIN_MS (60 s), so the child is gone before its lease reads as expired.
+export const BACKSTOP_GRACE_MS = 30 * 1000;
+
+export async function ragIndexCommand({ config, logger, flags = {}, backstop = {} }) {
+  const { setTimer = setTimeout, clearTimer = clearTimeout, exit = process.exit } = backstop;
   const projectDir = config?.projectDir || process.cwd();
+  const deadline = deadlineFromEnv();
+  // Past already (a queued start, a slow machine): nothing is opened, claimed or embedded.
+  if (deadline !== null && Date.now() >= deadline) return reportDeadline({ totals: { indexed: 0, failed: 0, files: 0, stopped: "deadline" }, projectDir, logger, json: flags.json });
+  let heldLease = null;
+  const timer = deadline === null ? null : setTimer(() => {
+    if (heldLease) releaseLease(heldLease);
+    noteDeadline(projectDir);
+    logger.warn?.("[rag] KJ_RAG_DEADLINE passed with an embed call still hung: the index is partial and its commit is not recorded; run kj rag index --with-sources by hand");
+    exit(0);
+  }, Math.max(0, deadline + BACKSTOP_GRACE_MS - Date.now()));
+  timer?.unref?.();
   const db = openDb(config);
   try {
     const slug = projectSlug(projectDir);
@@ -76,13 +105,22 @@ export async function ragIndexCommand({ config, logger, flags = {} }) {
       }
     }
     if (!totals) {
-      totals = await indexProject(projectDir, {
-        db, embedder,
-        karajanHome: getKarajanHome(), logger,
-        withSources: Boolean(flags.withSources),
-        exclude: ragExclude(config),
-      });
+      // HU-2907: a full foreground index holds the machine lease, so background lanes defer to it.
+      // The bootstrap child finds its spawning parent in the lease (it names itself first): kin.
+      const lease = indexLeasePath();
+      const claim = claimLease(lease, { projectDir, slug }, { kin: [process.ppid] });
+      if (claim.holder) logger.warn?.(`[rag] another index is running (${claim.holder.slug ?? "?"}, pid ${claim.holder.pid}); indexing anyway`);
+      if (claim.claimed) heldLease = lease;
+      try {
+        totals = await indexProject(projectDir, {
+          db, embedder,
+          karajanHome: getKarajanHome(), logger,
+          withSources: Boolean(flags.withSources),
+          exclude: ragExclude(config), deadline,
+        });
+      } finally { if (claim.claimed) releaseLease(lease); }
     }
+    if (totals.stopped) return reportDeadline({ totals, projectDir, logger, json: flags.json });
     if (totals.head) setLastIndexedCommit(db, slug, totals.head);
     // KJC-TSK-0697 — the library corpus refreshes on every index run.
     // Best-effort and cheap: a handful of cards, content-hash dedup.
@@ -101,6 +139,7 @@ export async function ragIndexCommand({ config, logger, flags = {} }) {
     }
     return totals;
   } finally {
+    if (timer) clearTimer(timer);
     db.close();
   }
 }
@@ -128,71 +167,98 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
   // KJC-TSK-0883: --library abre la base del canon, no la del proyecto.
   const db = flags.library ? openLibraryStore({ dim: config?.rag?.embedder?.dim || 768 }) : openDb(config);
   try {
+    const projectDir = config?.projectDir || process.cwd();
     const topK = Math.max(1, Number(flags.topK) || 5);
     const scope = flags.scope || "all";
-    // KJC-TSK-0438 — project isolation. Auto-detect slug from projectDir
-    // (basename normalised); `--project all` disables the filter, `--project
-    // <slug>` overrides. Pre-v2.27 chunks with NULL slug are only visible
-    // when no filter is in effect.
-    const detected = projectSlug(config?.projectDir || process.cwd());
     // KJC-TSK-0697 — `--library` targets the distilled-canon collection:
     // its own project namespace plus the kind filter.
     const library = Boolean(flags.library);
-    let project = flags.project === "all" ? null : (flags.project || detected || null);
-    if (library) project = LIBRARY_PROJECT;
-    // KJC-BUG-0061 follow-up: align the CLI `--json` shape with the MCP
-    // handler. The MCP tool responds `{ hits: [], empty: true, topK, scope }`
-    // so agents (and the `/kj-rag-query` skill from Camino B) have a
-    // deterministic recovery signal. The CLI was emitting just `[]`, which
-    // is indistinguishable from "query returned zero hits over a populated
-    // store" and forced consumers to parse stderr.
-    // KJC-TSK-0882: cuenta lo de ESTE proyecto; chunks ajenos no son respuesta.
-    if (countChunks(db) === 0 || (project && countProjectChunks(db, project) === 0)) {
-      // La library se rellena en cada `kj rag index`; migrar la llevaria al proyecto.
-      const remedy = library ? "kj rag index" : emptyIndexRemedy({ slug: project, legacyPath: join(getKarajanHome(), "rag.db"), dim: config?.rag?.embedder?.dim || 768 });
-      logger.warn(`[rag] ${library ? "the library index" : "this project's index"} holds no chunks of ${project || "any project"}: this is not an answer, run ${remedy}`);
-      if (flags.json) process.stdout.write(`${JSON.stringify({ hits: [], empty: true, topK, scope, remedy })}\n`);
-      return [];
-    }
+    const project = library ? LIBRARY_PROJECT : queryProject(flags, projectDir);
+    if (reportEmptyIndex({ db, project, library, config, flags, topK, scope, logger })) return [];
     // KJC-BUG-0258: chunks of plans and briefs but none of the project's own files
     // answer nothing about the code; said now, not when kj review --staged blocks.
-    if (!library && project && countProjectSources(db, project, config?.projectDir || process.cwd()) === 0) {
+    if (!library && project && countProjectSources(db, project, projectDir) === 0) {
       logger.warn("[rag] this project's index holds none of its own files (only plans or briefs): run kj rag index --with-sources");
     }
-    const mode = flags.mode || "hybrid";
-    // KJC-TSK-1180 (RAG-C, core 1.6.0): hybrid fusion is reciprocal rank; `alpha` weighs nothing
-    // any more, so a value that is still around is said once, never silently ignored.
-    if ((flags.alpha !== undefined && flags.alpha !== "0.6") || config?.rag?.search?.alpha !== undefined) {
-      logger.warn("[rag] alpha no longer applies: hybrid fusion is reciprocal rank (RRF); tune rag.search.rrf_k instead and drop alpha from the config");
-    }
-    const rrfK = config?.rag?.search?.rrf_k;
-    if (rrfK !== undefined && !(Number.isInteger(rrfK) && rrfK > 0)) throw new Error(`rag.search.rrf_k must be a positive integer, got ${JSON.stringify(rrfK)}`);
+    const { mode, rrfK } = searchSettings({ config, flags, logger });
     // KJC-TSK-1180 (RAG-B3): a relative `source~=` prefix is placed under this project's root.
-    const whereRoot = config?.projectDir || process.cwd();
-    // Safe by grammar: parseWhere (core where-parser.js) accepts ONLY
-    // AND-joined key=value pairs — an OR or parens never parses, so the
-    // composed clause cannot leak past the kind filter; it fails loudly.
-    let where = flags.where || null;
-    if (library) where = flags.where ? `kind=library AND ${flags.where}` : "kind=library";
+    const where = whereFor(flags, library);
     const rerankOpts = rerankOptionsFor({ config, flags, logger });
-    const ranked = await query(db, makeGovernedEmbedder(config), text, { topK, scope, project, mode, rrfK, where, whereRoot, rerankOpts });
-    const hits = library ? ranked : withPathNamedHits(pathNamedHits(db, text, { project, projectDir: config?.projectDir || process.cwd() }), ranked, topK);
-    if (flags.json) {
-      process.stdout.write(`${JSON.stringify(hits)}\n`);
-    } else {
-      for (const h of hits) {
-        const label = h.metadata?.hu_id || h.metadata?.symbol || h.metadata?.headingPath?.join(" > ") || "block";
-        logger.info(`[${h.kind} · ${label} · score=${h.score.toFixed(4)}] ${h.source}`);
-        logger.info(h.text.length > 240 ? `${h.text.slice(0, 240)}…` : h.text);
-        logger.info("");
-      }
-    }
+    const ranked = await query(db, makeGovernedEmbedder(config), text, { topK, scope, project, mode, rrfK, where, whereRoot: projectDir, rerankOpts });
+    const hits = library ? ranked : withPathNamedHits(pathNamedHits(db, text, { project, projectDir }), ranked, topK);
+    printHits(hits, flags, logger);
     return hits;
   } finally {
     db.close();
   }
 }
 
+/**
+ * KJC-TSK-0438 — project isolation. Auto-detect slug from projectDir
+ * (basename normalised); `--project all` disables the filter, `--project
+ * <slug>` overrides. Pre-v2.27 chunks with NULL slug are only visible
+ * when no filter is in effect.
+ */
+function queryProject(flags, projectDir) {
+  if (flags.project === "all") return null;
+  return flags.project || projectSlug(projectDir) || null;
+}
+
+/**
+ * KJC-BUG-0061 follow-up: align the CLI `--json` shape with the MCP
+ * handler. The MCP tool responds `{ hits: [], empty: true, topK, scope }`
+ * so agents (and the `/kj-rag-query` skill from Camino B) have a
+ * deterministic recovery signal. The CLI was emitting just `[]`, which
+ * is indistinguishable from "query returned zero hits over a populated
+ * store" and forced consumers to parse stderr.
+ * KJC-TSK-0882: cuenta lo de ESTE proyecto; chunks ajenos no son respuesta.
+ * @returns {boolean} true when the index has nothing to answer with (and it was said)
+ */
+function reportEmptyIndex({ db, project, library, config, flags, topK, scope, logger }) {
+  if (countChunks(db) > 0 && !(project && countProjectChunks(db, project) === 0)) return false;
+  // La library se rellena en cada `kj rag index`; migrar la llevaria al proyecto.
+  const remedy = library ? "kj rag index" : emptyIndexRemedy({ slug: project, legacyPath: join(getKarajanHome(), "rag.db"), dim: config?.rag?.embedder?.dim || 768 });
+  logger.warn(`[rag] ${library ? "the library index" : "this project's index"} holds no chunks of ${project || "any project"}: this is not an answer, run ${remedy}`);
+  if (flags.json) process.stdout.write(`${JSON.stringify({ hits: [], empty: true, topK, scope, remedy })}\n`);
+  return true;
+}
+
+/**
+ * KJC-TSK-1180 (RAG-C, core 1.6.0): hybrid fusion is reciprocal rank; `alpha` weighs nothing
+ * any more, so a value that is still around is said once, never silently ignored.
+ */
+function searchSettings({ config, flags, logger }) {
+  if ((flags.alpha !== undefined && flags.alpha !== "0.6") || config?.rag?.search?.alpha !== undefined) {
+    logger.warn("[rag] alpha no longer applies: hybrid fusion is reciprocal rank (RRF); tune rag.search.rrf_k instead and drop alpha from the config");
+  }
+  const rrfK = config?.rag?.search?.rrf_k;
+  if (rrfK !== undefined && !(Number.isInteger(rrfK) && rrfK > 0)) throw new Error(`rag.search.rrf_k must be a positive integer, got ${JSON.stringify(rrfK)}`);
+  return { mode: flags.mode || "hybrid", rrfK };
+}
+
+/**
+ * Safe by grammar: parseWhere (core where-parser.js) accepts ONLY
+ * AND-joined key=value pairs — an OR or parens never parses, so the
+ * composed clause cannot leak past the kind filter; it fails loudly.
+ */
+function whereFor(flags, library) {
+  if (!library) return flags.where || null;
+  return flags.where ? `kind=library AND ${flags.where}` : "kind=library";
+}
+
+function printHits(hits, flags, logger) {
+  if (flags.json) {
+    process.stdout.write(`${JSON.stringify(hits)}\n`);
+    return;
+  }
+  for (const h of hits) {
+    const label = h.metadata?.hu_id || h.metadata?.symbol || h.metadata?.headingPath?.join(" > ") || "block";
+    logger.info(`[${h.kind} · ${label} · score=${h.score.toFixed(4)}] ${h.source}`);
+    logger.info(h.text.length > 240 ? `${h.text.slice(0, 240)}…` : h.text);
+    logger.info("");
+  }
+}
+
 // KJC-TSK-1181 (RAG-D2): the cross-encoder, from the flag or from rag.search.rerank in the config
 // (enabled, top_n, model). Without the library or the model the retrieval answers alone and the
 // install is said once per process, never on every query, never in silence.