karajan-code 4.62.1 → 4.63.0

package/src/lang/chunk-python.js

@@ -14,6 +14,7 @@
 // devuelve `[{ text, metadata: { source, kind, symbol, language } }]`.
 
 import { getLoaded, loadLanguage } from "./tree-sitter-loader.js";
+import { spanChunks } from "./chunk-span.js";
 
 const DEFAULT_LIMIT = 800;
 const DEFAULT_OVERLAP = 100;
@@ -23,16 +24,6 @@ export async function preparePython() {
   return loadLanguage("python");
 }
 
-function windowText(text, limit, overlap) {
-  if (text.length <= limit) return [text];
-  const step = Math.max(1, limit - overlap);
-  const out = [];
-  for (let i = 0; i < text.length; i += step) {
-    out.push(text.slice(i, i + limit));
-    if (i + limit >= text.length) break;
-  }
-  return out;
-}
 
 function marksViaAST(text) {
   const entry = getLoaded("python");
@@ -68,21 +59,5 @@ function marksViaRegex(text) {
 }
 
 export function chunkPython(text, { path = "<src>", limit = DEFAULT_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
-  const marks = marksViaAST(text) ?? marksViaRegex(text);
-  if (marks.length === 0) {
-    return windowText(text, limit, overlap).map((piece) => ({
-      text: piece,
-      metadata: { source: path, kind: "code", symbol: null, language: "python" },
-    }));
-  }
-  const out = [];
-  for (let i = 0; i < marks.length; i += 1) {
-    const start = marks[i].idx;
-    const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
-    const body = text.slice(start, end);
-    for (const piece of windowText(body, limit, overlap)) {
-      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol, language: "python" } });
-    }
-  }
-  return out;
+  return spanChunks(text, marksViaAST(text) ?? marksViaRegex(text), { path, language: "python", limit, overlap });
 }

package/src/lang/chunk-rust.js

@@ -7,6 +7,7 @@
 // que metadata.symbol apunte al tipo implementado (útil para grep en RAG).
 
 import { getLoaded, loadLanguage } from "./tree-sitter-loader.js";
+import { spanChunks } from "./chunk-span.js";
 
 const DEFAULT_LIMIT = 800;
 const DEFAULT_OVERLAP = 100;
@@ -25,16 +26,6 @@ export async function prepareRust() {
   return loadLanguage("rust");
 }
 
-function windowText(text, limit, overlap) {
-  if (text.length <= limit) return [text];
-  const step = Math.max(1, limit - overlap);
-  const out = [];
-  for (let i = 0; i < text.length; i += step) {
-    out.push(text.slice(i, i + limit));
-    if (i + limit >= text.length) break;
-  }
-  return out;
-}
 
 function symbolForImpl(node) {
   const typeNode = node.childForFieldName("type");
@@ -82,21 +73,5 @@ function marksViaRegex(text) {
 }
 
 export function chunkRust(text, { path = "<src>", limit = DEFAULT_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
-  const marks = marksViaAST(text) ?? marksViaRegex(text);
-  if (marks.length === 0) {
-    return windowText(text, limit, overlap).map((piece) => ({
-      text: piece,
-      metadata: { source: path, kind: "code", symbol: null, language: "rust" },
-    }));
-  }
-  const out = [];
-  for (let i = 0; i < marks.length; i += 1) {
-    const start = marks[i].idx;
-    const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
-    const body = text.slice(start, end);
-    for (const piece of windowText(body, limit, overlap)) {
-      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol, language: "rust" } });
-    }
-  }
-  return out;
+  return spanChunks(text, marksViaAST(text) ?? marksViaRegex(text), { path, language: "rust", limit, overlap });
 }

package/src/lang/chunk-span.js

@@ -0,0 +1,49 @@
+/**
+ * KJC-TSK-1171 (RAG-B, ADR 0026): the shared tail of every code chunker. A chunker
+ * finds marks ({ idx, symbol, parent? }); this slices the text from each mark to the
+ * next, windows what is too long for the embedder, and stamps each piece with its
+ * 1-based line range and the symbol it belongs to, so the embedded header and the
+ * citations can name file, symbol, parent and lines.
+ */
+const DEFAULT_LIMIT = 800;
+const DEFAULT_OVERLAP = 100;
+
+/** 1-based line of a character index. */
+export const lineAt = (text, idx) => {
+  let line = 1;
+  for (let i = 0; i < idx && i < text.length; i += 1) if (text.charCodeAt(i) === 10) line += 1;
+  return line;
+};
+
+// Trailing whitespace (a newline and the next line's indentation) is not a line of the symbol.
+const linesIn = (piece) => {
+  const body = piece.trimEnd();
+  let n = 0;
+  for (let i = 0; i < body.length; i += 1) if (body.charCodeAt(i) === 10) n += 1;
+  return n;
+};
+
+/** The slice at `startIdx`, windowed, each piece with its line range. */
+export function pieceSpans(text, startIdx, slice, { limit = DEFAULT_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  const step = Math.max(1, limit - overlap);
+  const out = [];
+  for (let i = 0; ; i += step) {
+    const piece = slice.slice(i, i + limit);
+    const startLine = lineAt(text, startIdx + i);
+    out.push({ text: piece, start_line: startLine, end_line: startLine + linesIn(piece) });
+    if (i + limit >= slice.length) break;
+  }
+  return out;
+}
+
+/** Chunks from marks; with no marks, the whole text as one anonymous symbol. */
+export function spanChunks(text, marks, { path = "<src>", kind = "code", language, limit, overlap } = {}) {
+  const base = { source: path, kind, ...(language === undefined ? {} : { language }) };
+  const ranges = marks.length === 0
+    ? [{ start: 0, end: text.length, symbol: null, parent: null }]
+    : marks.map((m, i) => ({ start: m.idx, end: i + 1 < marks.length ? marks[i + 1].idx : text.length, symbol: m.symbol, parent: m.parent ?? null }));
+  return ranges.flatMap(({ start, end, symbol, parent }) =>
+    pieceSpans(text, start, text.slice(start, end), { limit, overlap }).map(({ text: piece, start_line, end_line }) => ({
+      text: piece, metadata: { ...base, symbol, parent, start_line, end_line },
+    })));
+}

package/src/plan/plan-store.js

@@ -63,8 +63,11 @@ export function projectSlug(projectDir) {
     .slice(0, 120);
 }
 
+// KJC-TSK-1166: the board honours KJ_PLANS_DIR as its plans root; what kj writes must land
+// in the same root, or the board shows none of it.
 export function plansDir(projectDir) {
-  return path.join(getKjHome(), "plans", projectSlug(projectDir));
+  const root = process.env.KJ_PLANS_DIR || path.join(getKjHome(), "plans");
+  return path.join(root, projectSlug(projectDir));
 }
 
 /**

package/src/rag/auto-update.js

@@ -7,10 +7,25 @@ import { fileURLToPath } from "node:url";
 import { execa } from "execa";
 
 import { projectSlug, getLastIndexedCommit, setLastIndexedCommit } from "./vec-store.js";
-import { indexProjectDelta } from "./indexer.js";
+import { indexProject, indexProjectDelta } from "./indexer.js";
+import { readRefreshedPaths, untrackedFiles, writeRefreshedPaths } from "./worktree-files.js";
+import { startBootstrap } from "./bootstrap-index.js";
 import { ragExclude } from "./indexable.js";
 import { makeGovernedEmbedder } from "./governed-embedder.js";
 import { openProjectStore } from "./project-store.js";
+import { readEmbedSchema } from "./embed-schema.js";
+import { EMBED_SCHEMA } from "./embed-header.js";
+import { getKarajanHome } from "../utils/paths.js";
+
+// Said once per process and project: every query of a session would repeat it otherwise.
+const warnedFor = new Set();
+const warnOnce = (logger, key, msg) => { if (!warnedFor.has(key)) { warnedFor.add(key); logger.warn?.(msg); } };
+
+/** How many project files the index holds (a prefix compare: paths may hold % or _). */
+const countSources = (db, slug, projectDir) => {
+  const prefix = `${projectDir.replace(/[\\/]+$/, "")}/`;
+  return db.prepare("SELECT COUNT(DISTINCT source) AS n FROM chunks WHERE project_slug = ? AND substr(source, 1, ?) = ?").get(slug, prefix.length, prefix).n;
+};
 
 // The pre-query refresh reindexes at most this many changed files (config.rag.autoUpdate.maxFiles).
 export const DEFAULT_MAX_DRIFT_FILES = 25;
@@ -27,23 +42,56 @@ export async function maybeAutoUpdate({ projectDir, config, logger = console, fl
   const db = openProjectStore({ projectDir, dim: config?.rag?.embedder?.dim || 768 });
   try {
     const since = getLastIndexedCommit(db, slug);
-    if (!since || since === head) return { skipped: true, head };
+    if (!since) {
+      // KJC-TSK-1174 (RAG-A2): no index yet. It is built in the background, once; the session
+      // does not wait. A recent attempt that died (no Ollama) is not retried blindly.
+      const boot = startBootstrap(projectDir);
+      if (boot.recent) {
+        warnOnce(logger, `bootstrap:${projectDir}`, `[rag] ${slug} has no index and the background indexing did not finish: run kj rag index --with-sources`);
+        return { skipped: true, reason: "bootstrap-failed", head };
+      }
+      warnOnce(logger, `bootstrap:${projectDir}`, `[rag] ${slug} has no index yet: indexing it in the background (pid ${boot.pid}); queries answer from it once it is done`);
+      return { skipped: true, reason: "bootstrapping", pid: boot.pid, head };
+    }
     // KJC-BUG-0304: a drift of hundreds of files (an index weeks old) reindexed the
     // repo for half an hour before the query answered. Past the bound the query
     // answers from the index as it is, and says how to bring it up to date.
     const maxFiles = config?.rag?.autoUpdate?.maxFiles ?? DEFAULT_MAX_DRIFT_FILES;
-    const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-only", since, "HEAD"]);
-    const changed = stdout.split("\n").filter(Boolean).length;
+    // KJC-TSK-1171 (RAG-B): an index built before the embedded header holds vectors a
+    // query by file or symbol name cannot reach. Small, it is rebuilt here; large, the
+    // query answers from it and says once how to rebuild.
+    const schema = readEmbedSchema(projectDir);
+    if (schema < EMBED_SCHEMA) {
+      const sources = countSources(db, slug, projectDir);
+      if (sources > maxFiles) {
+        warnOnce(logger, projectDir, `[rag] the index of ${slug} predates the embedded header (schema ${schema} < ${EMBED_SCHEMA}): answering from it as it is; rebuild it with kj rag index --with-sources`);
+        return { skipped: true, reason: "embed-schema-old", sources, head };
+      }
+      logger.info?.(`[rag] the index of ${slug} predates the embedded header (schema ${schema} < ${EMBED_SCHEMA}); rebuilding ${sources} source(s)`);
+      const totals = await indexProject(projectDir, { db, embedder: makeGovernedEmbedder(config), karajanHome: getKarajanHome(), logger, withSources: true, exclude: ragExclude(config) });
+      if (totals.head) setLastIndexedCommit(db, slug, totals.head);
+      return { ran: true, rebuilt: true, totals };
+    }
+    // KJC-TSK-1174 (RAG-A1): the drift is measured up to the files on disk, untracked ones
+    // included, so a query sees what the session is writing before it is committed.
+    // The dirty paths of the last refresh count too: reverted or deleted since, they hold stale chunks.
+    const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-only", "-z", since]);
+    const dirty = [...stdout.split("\0").filter(Boolean), ...(await untrackedFiles(projectDir))];
+    const reconcile = readRefreshedPaths(projectDir).filter((p) => !dirty.includes(p));
+    const changed = dirty.length + reconcile.length;
+    if (changed === 0) return { skipped: true, head };
     if (changed > maxFiles) {
       logger.warn?.(`[rag] ${changed} file(s) changed since the last index (more than ${maxFiles}): answering from the index as it is; bring it up to date with kj rag index --since auto`);
       return { skipped: true, reason: "drift-too-large", changed, head };
     }
-    logger.info?.(`[rag] drift detected (${since.slice(0, 7)} → ${head.slice(0, 7)}); running delta update`);
-    const totals = await indexProjectDelta(projectDir, { db, embedder: makeGovernedEmbedder(config), since, logger, exclude: ragExclude(config) });
+    logger.info?.(`[rag] drift detected (${since.slice(0, 7)} → ${head.slice(0, 7)}, ${changed} file(s)); running delta update`);
+    const totals = await indexProjectDelta(projectDir, { db, embedder: makeGovernedEmbedder(config), since, logger, exclude: ragExclude(config), worktree: true, reconcile });
     if (totals.head) setLastIndexedCommit(db, slug, totals.head);
+    writeRefreshedPaths(projectDir, dirty);
     return { ran: true, totals };
   } catch (err) {
-    logger.warn?.(`[rag] auto-update failed (${err.message}); continuing without refresh`);
+    // Said once per process: an embedder that is down would repeat it on every query otherwise.
+    warnOnce(logger, `failed:${projectDir}`, `[rag] auto-update failed (${err.message}); continuing without refresh`);
     return { failed: true, error: err.message };
   } finally { db.close(); }
 }

package/src/rag/bootstrap-index.js

@@ -0,0 +1,86 @@
+// KJC-TSK-1174 (RAG-A2): a project with no index gets one in the background, once, at the first
+// command that needs it. The session goes on; queries answer from the index when it is done.
+import { spawn } from "node:child_process";
+import { mkdirSync, readFileSync, renameSync, rmdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { projectDbPath } from "./project-store.js";
+
+const KJ_BIN = resolve(fileURLToPath(import.meta.url), "../../../bin/kj.js");
+// An attempt that died (no Ollama, no model) is not repeated on every command: this long apart.
+export const RETRY_MS = 10 * 60 * 1000;
+// A marker that does not parse, or a lock, older than this was left by a crashed command: taken over.
+export const GRACE_MS = 30 * 1000;
+// An indexer is not trusted to run longer than this: past it, its pid may belong to someone else.
+export const MAX_RUN_MS = 60 * 60 * 1000;
+const markerPath = (projectDir) => `${projectDbPath(projectDir)}.bootstrap`;
+
+const alive = (pid) => { try { process.kill(pid, 0); return true; } catch { return false; } };
+
+/** What the marker says of the last attempt: `{ pid, at, running }`, or null when none was ever made. */
+const unlessGone = (fn) => {
+  try { return fn(); }
+  catch (err) {
+    if (err.code === "ENOENT") { return null; }
+    throw err;
+  }
+};
+
+export function readBootstrap(projectDir, { now = Date.now() } = {}) {
+  const file = markerPath(projectDir);
+  // Read first, never check-then-read: another command may set the marker aside in between.
+  const raw = unlessGone(() => readFileSync(file, "utf8"));
+  if (raw === null) return null;
+  try {
+    const note = JSON.parse(raw);
+    return { ...note, running: alive(note.pid) && now - note.at < MAX_RUN_MS };
+  } catch {
+    // Another command is writing it this very moment: running for a moment; past the grace, a crash.
+    const stat = unlessGone(() => statSync(file));
+    return stat === null ? null : { pid: null, at: 0, running: now - stat.mtimeMs < GRACE_MS };
+  }
+}
+
+/**
+ * Starts `kj rag index --with-sources` detached, or reuses the one running. Returns
+ * `{ started, pid }`, or `{ started: false, recent: true }` when the last attempt died less
+ * than RETRY_MS ago: the index is then asked for by hand instead of being retried blindly.
+ */
+export function startBootstrap(projectDir, { spawnFn = spawn, bin = KJ_BIN, now = Date.now() } = {}) {
+  const file = markerPath(projectDir);
+  mkdirSync(dirname(file), { recursive: true });
+  // One claimant at a time: the lock directory is created atomically, so of two commands arriving
+  // together only one reads the marker and decides.
+  const lock = `${file}.lock`;
+  const reuse = () => ({ started: false, pid: readBootstrap(projectDir, { now })?.pid ?? null });
+  if (!takeLock(lock, now)) return reuse();
+  try {
+    const last = readBootstrap(projectDir, { now });
+    if (last?.running) return { started: false, pid: last.pid };
+    if (last && now - last.at < RETRY_MS) return { started: false, recent: true, pid: null };
+    // The marker lands whole each time (written apart, then renamed): first with the claimant's
+    // pid, so it reads as running meanwhile, then with the indexer's.
+    const note = (pid) => { const tmp = `${file}.${process.pid}.tmp`; writeFileSync(tmp, JSON.stringify({ pid, at: now }), "utf8"); renameSync(tmp, file); };
+    note(process.pid);
+    const child = spawnFn(process.execPath, [bin, "rag", "index", "--with-sources"], { cwd: projectDir, detached: true, stdio: "ignore" });
+    child.unref?.();
+    note(child.pid);
+    return { started: true, pid: child.pid };
+  } finally { unlessGone(() => rmdirSync(lock)); }
+}
+
+const takeLock = (lock, now) => {
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try { mkdirSync(lock); return true; }
+    catch (err) {
+      if (err.code !== "EEXIST") throw err;
+      const held = unlessGone(() => statSync(lock));
+      if (held && now - held.mtimeMs < GRACE_MS) return false;
+      // The holder crashed. The lock is set aside under a unique name first: only the one caller
+      // whose rename succeeds removes it, so nobody removes a lock another caller has just taken.
+      const aside = `${lock}.${process.pid}.${now}.stale`;
+      if (unlessGone(() => renameSync(lock, aside)) !== null) rmdirSync(aside);
+    }
+  }
+  return false;
+};

package/src/rag/chunker-ast.js

@@ -9,6 +9,7 @@
 // file (errorRecovery: true catches most cases; a hard syntax error
 // returns null so the caller can fall back).
 import { parse } from "@babel/parser";
+import { pieceSpans } from "../lang/chunk-span.js";
 
 const PLUGINS = ["typescript", "jsx", "decorators-legacy", "classProperties", "topLevelAwait"];
 
@@ -24,8 +25,10 @@ export function chunkSourceAST(text, { path = "<src>", limit = 800, overlap = 10
     const start = leadingCommentsStart(node);
     const slice = text.slice(start, node.end ?? start);
     if (!slice.trim()) continue;
-    if (slice.length <= limit) out.push({ text: slice, metadata: { source: path, kind: "code", symbol: sym } });
-    else for (const piece of windowText(slice, limit, overlap)) out.push({ text: piece, metadata: { source: path, kind: "code", symbol: sym } });
+    // KJC-TSK-1171: each piece carries its line range; top-level declarations have no parent.
+    for (const { text: piece, start_line, end_line } of pieceSpans(text, start, slice, { limit, overlap })) {
+      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: sym, parent: null, start_line, end_line } });
+    }
   }
   return out.length === 0 ? null : out;
 }
@@ -44,13 +47,3 @@ function leadingCommentsStart(node) {
   return node.start ?? 0;
 }
 
-function windowText(text, limit, overlap) {
-  if (text.length <= limit) return [text];
-  const step = Math.max(1, limit - overlap);
-  const out = [];
-  for (let i = 0; i < text.length; i += step) {
-    out.push(text.slice(i, i + limit));
-    if (i + limit >= text.length) break;
-  }
-  return out;
-}

package/src/rag/chunker.js

@@ -93,6 +93,7 @@ export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT }
 
 import { chunkSourceAST } from "./chunker-ast.js";
 import { adapterForPath } from "../lang/registry.js";
+import { spanChunks } from "../lang/chunk-span.js";
 
 const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
 
@@ -113,20 +114,7 @@ function chunkSourceRegex(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT, o
   EXPORT_RE.lastIndex = 0;
   let m;
   while ((m = EXPORT_RE.exec(text)) !== null) marks.push({ idx: m.index, symbol: m[1] });
-  if (marks.length === 0) {
-    return windowText(text, { limit, overlap })
-      .map((piece) => ({ text: piece, metadata: { source: path, kind: "code", symbol: null } }));
-  }
-  const out = [];
-  for (let i = 0; i < marks.length; i += 1) {
-    const start = marks[i].idx;
-    const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
-    const body = text.slice(start, end);
-    for (const piece of windowText(body, { limit, overlap })) {
-      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol } });
-    }
-  }
-  return out;
+  return spanChunks(text, marks, { path, limit, overlap });
 }
 
 export { chunkSourceRegex };

package/src/rag/embed-header.js

@@ -0,0 +1,35 @@
+/**
+ * KJC-TSK-1171 (RAG-B, ADR 0026): what the embedder reads before a chunk's body.
+ * A query by module, class or file name ("the tax module in billing", "session.js")
+ * finds nothing in a body that never mentions them; the header puts path, language,
+ * parent, symbol and lines into the vector. Only the embedder sees it: the text
+ * stored and shown to the agent is the body as it is.
+ */
+import { extname, relative } from "node:path";
+
+/** Bumped when what the embedder reads changes: an index with an older one must be rebuilt. */
+export const EMBED_SCHEMA = 2;
+
+const LANGUAGES = { js: "javascript", mjs: "javascript", cjs: "javascript", jsx: "javascript", ts: "typescript", tsx: "typescript", py: "python", go: "go", rs: "rust", java: "java", md: "markdown", mdx: "markdown", json: "json", yml: "yaml", yaml: "yaml" };
+
+const languageOf = (metadata, source) => metadata.language ?? LANGUAGES[extname(source).slice(1).toLowerCase()] ?? null;
+
+/** The header line of a chunk, or "" when nothing beyond the path is known and the path is unknown. */
+export function embedHeader(metadata = {}, { root = process.cwd() } = {}) {
+  const source = metadata.source;
+  if (!source) return "";
+  const parts = [relative(root, source) || source];
+  const language = languageOf(metadata, source);
+  if (language) parts.push(language);
+  if (metadata.headingPath?.length) parts.push(metadata.headingPath.join(" > "));
+  if (metadata.hu_id) parts.push(metadata.title ? `${metadata.hu_id} ${metadata.title}` : metadata.hu_id);
+  if (metadata.symbol) parts.push(metadata.parent ? `${metadata.parent}.${metadata.symbol}` : metadata.symbol);
+  if (metadata.start_line) parts.push(metadata.end_line && metadata.end_line !== metadata.start_line ? `L${metadata.start_line}-${metadata.end_line}` : `L${metadata.start_line}`);
+  return parts.join(" · ");
+}
+
+/** The text the embedder reads: header, blank line, body. The body alone when there is no header. */
+export function embedText(chunk, opts) {
+  const header = embedHeader(chunk.metadata, opts);
+  return header ? `${header}\n\n${chunk.text}` : chunk.text;
+}

package/src/rag/embed-schema.js

@@ -0,0 +1,28 @@
+/**
+ * KJC-TSK-1171 (RAG-B): which embed schema a project's index was built with. The
+ * marker lives next to the project's store; an index built before the header (or with
+ * no marker at all) is schema 1 and is rebuilt once, bounded, before it serves queries.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { projectDbPath } from "./project-store.js";
+import { EMBED_SCHEMA } from "./embed-header.js";
+
+const PRE_HEADER = 1;
+const markerPath = (projectDir) => `${projectDbPath(projectDir)}.embed-schema`;
+
+/** The schema the index holds; 1 when nothing was ever recorded (every index before the header). */
+export function readEmbedSchema(projectDir) {
+  const file = markerPath(projectDir);
+  if (!existsSync(file)) return PRE_HEADER;
+  const n = Number(readFileSync(file, "utf8").trim());
+  if (!Number.isInteger(n) || n < 1) throw new Error(`${file} does not hold an embed schema number`);
+  return n;
+}
+
+/** Recorded after a full index, never after a partial one: only a complete rebuild carries the schema. */
+export function writeEmbedSchema(projectDir, schema = EMBED_SCHEMA) {
+  const file = markerPath(projectDir);
+  mkdirSync(dirname(file), { recursive: true });
+  writeFileSync(file, `${schema}\n`, "utf8");
+}

package/src/rag/eval.js

@@ -39,6 +39,16 @@ export function scoreQuery(hits, expected, ks = DEFAULT_KS) {
   return { recall, rr, firstRank };
 }
 
+// KJC-TSK-1170 (RAG-E): latency travels with recall and MRR. p95 is the nearest-rank
+// percentile over the measured queries; both are rounded to whole milliseconds.
+function latencyOf(results) {
+  const ms = results.map((r) => r.ms).filter((v) => Number.isFinite(v)).toSorted((a, b) => a - b);
+  if (ms.length === 0) return null;
+  const meanMs = Math.round(ms.reduce((acc, v) => acc + v, 0) / ms.length);
+  const p95Ms = Math.round(ms[Math.max(0, Math.ceil(ms.length * 0.95) - 1)]);
+  return { meanMs, p95Ms };
+}
+
 export function aggregate(results, ks = DEFAULT_KS) {
   const n = results.length || 1;
   const recall = {};
@@ -46,16 +56,42 @@ export function aggregate(results, ks = DEFAULT_KS) {
     recall[`@${k}`] = results.reduce((acc, r) => acc + (r.recall[`@${k}`] || 0), 0) / n;
   }
   const mrr = results.reduce((acc, r) => acc + (r.rr || 0), 0) / n;
-  return { n: results.length, recall, mrr };
+  const latency = latencyOf(results);
+  return { n: results.length, recall, mrr, ...(latency ? { latency } : {}) };
 }
 
-export async function runEval(queries, runQuery, { topK = 10, ks = DEFAULT_KS } = {}) {
+export async function runEval(queries, runQuery, { topK = 10, ks = DEFAULT_KS, now = () => performance.now() } = {}) {
   const results = [];
   for (const q of queries) {
+    const started = now();
     const hits = (await runQuery(q.query, topK)) || [];
+    const ms = now() - started;
     const expected = { sources: q.expected_sources || [], symbols: q.expected_symbols || [] };
     const scored = scoreQuery(hits, expected, ks);
-    results.push({ query: q.query, hits, ...scored });
+    results.push({ query: q.query, hits, ms, ...scored });
   }
   return { results, aggregate: aggregate(results, ks) };
 }
+
+/**
+ * KJC-TSK-1170: the current aggregate against a frozen one. Recall and MRR may not drop
+ * more than `tolerance`; p95 latency may not exceed the baseline's by `latencyFactor`,
+ * and is skipped when the baseline recorded none. Pure: the caller reads the files.
+ * @returns {{ok: boolean, problems: string[]}}
+ */
+export function compareToBaseline(current, baseline, { tolerance = 0, latencyFactor = 2, latencyFloorMs = 50 } = {}) {
+  const problems = [];
+  for (const k of Object.keys(baseline.recall ?? {})) {
+    const now = current.recall?.[k] ?? 0;
+    if (now < baseline.recall[k] - tolerance) problems.push(`recall${k} ${now.toFixed(3)} < ${baseline.recall[k].toFixed(3)}`);
+  }
+  if ((current.mrr ?? 0) < (baseline.mrr ?? 0) - tolerance) problems.push(`MRR ${(current.mrr ?? 0).toFixed(3)} < ${baseline.mrr.toFixed(3)}`);
+  const baseP95 = baseline.latency?.p95Ms;
+  if (Number.isFinite(baseP95) && Number.isFinite(current.latency?.p95Ms)) {
+    // KJC-BUG-0394: the factor alone turns a 2 ms baseline into a 4 ms limit, which machine
+    // noise crosses; the floor keeps a tiny baseline from failing on nothing.
+    const limit = Math.max(Math.round(baseP95 * latencyFactor), baseP95 + latencyFloorMs);
+    if (current.latency.p95Ms > limit) problems.push(`p95 ${current.latency.p95Ms} ms > ${limit} ms (${latencyFactor}x the baseline's ${baseP95} ms, or ${latencyFloorMs} ms over it)`);
+  }
+  return { ok: problems.length === 0, problems };
+}

package/src/rag/indexer.js

@@ -14,6 +14,9 @@ import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
 import { insertChunk, deleteChunksBySource, findChunkByHash, getEmbeddingsByIds } from "./vec-store.js";
 import { detectAdaptersForProject, getAllCodeExtensions } from "../lang/registry.js";
 import { indexableReason, listProjectFiles } from "./indexable.js";
+import { EMBED_SCHEMA, embedText } from "./embed-header.js";
+import { writeEmbedSchema } from "./embed-schema.js";
+import { untrackedFiles } from "./worktree-files.js";
 
 // KJC-PCS-0052 PR-A — antes vivían dos consts hard-coded para JS aquí
 // (CODE_EXT_RE + SKIP_SEGMENTS). Ahora vienen del registry de adapters,
@@ -59,7 +62,7 @@ export const sourceKind = () => "code";
  * fresh batch is written, so a second call against the same file
  * leaves the store with exactly the latest chunks (no duplicates).
  */
-export async function indexFile(path, { db, embedder, logger = console, project = null, kind: declaredKind = null } = {}) {
+export async function indexFile(path, { db, embedder, logger = console, project = null, kind: declaredKind = null, root = process.cwd() } = {}) {
   if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
   const kind = declaredKind ?? detectKind(path);
   if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
@@ -75,11 +78,17 @@ export async function indexFile(path, { db, embedder, logger = console, project
     // the stored vector: a file whose every chunk duplicates another (CLAUDE.md
     // next to the AGENTS.md kj harden writes) was left with no rows at all, and
     // the coverage check reported it missing after every reindex.
-    const contentHash = createHash("sha256").update(ch.text).digest("hex");
+    // KJC-TSK-1171 (RAG-B): the embedder reads the metadata header before the body, so a
+    // query by file, module or symbol name lands; the stored text stays the body alone.
+    // The dedup key is what the embedder read, header included, under the current schema: a
+    // vector is reused only when the same body sits under the same path, symbol and lines,
+    // never one embedded under another header or an older schema.
+    const embeddable = embedText(ch, { root });
+    const contentHash = createHash("sha256").update(`${EMBED_SCHEMA}\n${embeddable}`).digest("hex");
     try {
       const dup = findChunkByHash(db, contentHash, project);
       const stored = dup ? getEmbeddingsByIds(db, [dup.id]).get(Number(dup.id)) : null;
-      const embedding = stored ?? await embedder.embed(ch.text);
+      const embedding = stored ?? await embedder.embed(embeddable);
       insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project, contentHash });
       if (stored) skipped += 1; else indexed += 1;
     } catch (err) {
@@ -145,13 +154,13 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
   if (existsSync(planRoot)) {
     const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
     for (const p of plans) {
-      const r = await indexFile(p, { db, embedder, logger, project: slug });
+      const r = await indexFile(p, { db, embedder, logger, project: slug, root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }
   const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
   if (existsSync(onboarding)) {
-    const r = await indexFile(onboarding, { db, embedder, logger, project: slug });
+    const r = await indexFile(onboarding, { db, embedder, logger, project: slug, root: projectDir });
     totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
   }
   if (withSources) {
@@ -173,9 +182,13 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
       if (source.startsWith(prefix) && outOf(source) !== null) deleteChunksBySource(db, source);
     }
     for (const s of sources) {
-      const r = await indexFile(s, { db, embedder, logger, project: slug, kind: sourceKind() });
+      const r = await indexFile(s, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
+    // KJC-TSK-1171: only a COMPLETE index of the sources carries the embed schema; a file the
+    // embedder failed on leaves the marker as it was, so the rebuild is asked for again.
+    if (totals.failed === 0) writeEmbedSchema(projectDir);
+    else logger.warn?.(`[rag-indexer] ${totals.failed} chunk(s) failed: the embed schema is not recorded, the index will be rebuilt again`);
   }
   return totals;
 }
@@ -187,21 +200,30 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
  * `head` lets callers persist via setLastIndexedCommit. Throws on diff failure
  * (shallow clone, unknown ref) so the CLI can fall back to a full reindex.
  */
-export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console, exclude = [] } = {}) {
+export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console, exclude = [], worktree = false, reconcile = [] } = {}) {
   const totals = { indexed: 0, failed: 0, files: 0, deleted: 0, head: null };
   const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
   const { stdout: head } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
   totals.head = head.trim();
-  if (totals.head === since) return totals;
-  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, "HEAD"]);
+  if (totals.head === since && !worktree) return totals;
+  // KJC-TSK-1174 (RAG-A1): with `worktree`, the diff ends at the files on disk instead of HEAD,
+  // and untracked files join it, so what is being written is retrievable before it is committed.
+  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, ...(worktree ? [] : ["HEAD"])]);
   const parts = stdout.split("\0").filter(Boolean);
   const ops = []; // { kind: 'del'|'idx', p }
-  for (let i = 0; i < parts.length; i += 1) {
+  let i = 0;
+  while (i < parts.length) {
     const s = parts[i];
-    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[++i] }, { kind: "idx", p: parts[++i] }); }
-    else if (s === "D") { ops.push({ kind: "del", p: parts[++i] }); }
-    else { ops.push({ kind: "idx", p: parts[++i] }); }
+    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[i + 1] }, { kind: "idx", p: parts[i + 2] }); i += 3; }
+    else if (s.startsWith("C")) { ops.push({ kind: "idx", p: parts[i + 2] }); i += 3; } // a copy keeps its source
+    else if (s === "D") { ops.push({ kind: "del", p: parts[i + 1] }); i += 2; }
+    else { ops.push({ kind: "idx", p: parts[i + 1] }); i += 2; }
   }
+  if (worktree) for (const p of await untrackedFiles(projectDir)) ops.push({ kind: "idx", p });
+  // Paths a previous refresh took from a dirty tree: on disk they are reindexed as they are now
+  // (reverted or still dirty), gone they are dropped. Each path is handled once.
+  const seen = new Set(ops.map((o) => o.p));
+  for (const p of reconcile) if (!seen.has(p)) ops.push({ kind: existsSync(join(projectDir, p)) ? "idx" : "del", p });
   const adapters = detectAdaptersForProject(projectDir);
   await prepareAdapters(adapters, { logger });
   for (const { kind, p } of ops) {
@@ -211,7 +233,7 @@ export async function indexProjectDelta(projectDir, { db, embedder, since, logge
       logger.info?.(`[rag-indexer] delta delete ${relative(process.cwd(), abs)}`);
     } else if (indexableReason(relative(projectDir, abs), abs, { exclude, projectDir }) === null) {
       // KJC-TSK-0891: the same criterion as the full walk, not a list of extensions.
-      const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind() });
+      const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }