karajan-code 2.33.0 → 3.0.0

package/src/onboarder/collectors/framework-multilang.js

@@ -0,0 +1,64 @@
+// KJC-TSK-0477 — Detectores mínimos de framework por stack (Python/Rust/Go).
+// Consume el sniff de manifests (PR-C) para conocer deps + presencia de
+// archivos canónicos. Cero deps externas, fail-soft (devuelve null o lista
+// vacía). El brief markdown se sintetiza río abajo desde estos campos.
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+const PY_FW = [
+  { name: "django", deps: ["django"], file: "manage.py" },
+  { name: "flask", deps: ["flask"], file: "app.py" },
+  { name: "fastapi", deps: ["fastapi"], file: "main.py" },
+];
+const RUST_FW = [
+  { name: "axum", deps: ["axum"] },
+  { name: "actix-web", deps: ["actix-web"] },
+  { name: "rocket", deps: ["rocket"] },
+];
+const GO_FW = [
+  { name: "gin", deps: ["github.com/gin-gonic/gin"] },
+  { name: "echo", deps: ["github.com/labstack/echo"] },
+  { name: "fiber", deps: ["github.com/gofiber/fiber"] },
+];
+
+function matchFw(manifestRaw, projectDir, table) {
+  const found = [];
+  for (const fw of table) {
+    const hasDep = fw.deps.some((d) => manifestRaw?.includes(d));
+    const hasFile = fw.file ? existsSync(path.join(projectDir, fw.file)) : false;
+    if (hasDep || hasFile) found.push({ name: fw.name, viaDep: hasDep, viaFile: hasFile });
+  }
+  return found;
+}
+
+function detectLayout(projectDir, expected) {
+  return expected.filter((d) => existsSync(path.join(projectDir, d)));
+}
+
+// Detect frameworks + canonical layout per stack. Returns one entry per
+// detected manifest stack (skips Node — the existing JS path already covers
+// that with detectProjectStack).
+export function detectFrameworksMultiLang(projectDir, manifests = [], { readManifest } = {}) {
+  const out = [];
+  for (const m of manifests) {
+    if (m.stack === "node") continue;
+    const raw = readManifest ? readManifest(m) : "";
+    if (m.stack === "python") {
+      out.push({ stack: "python", manifest: m.manifest,
+        frameworks: matchFw(raw, projectDir, PY_FW),
+        layout: detectLayout(projectDir, ["apps", "src", "tests"]) });
+    } else if (m.stack === "rust") {
+      const isBin = existsSync(path.join(projectDir, "src", "main.rs"));
+      const isLib = existsSync(path.join(projectDir, "src", "lib.rs"));
+      out.push({ stack: "rust", manifest: m.manifest,
+        crateType: isBin && isLib ? "mixed" : isBin ? "binary" : isLib ? "library" : null,
+        frameworks: matchFw(raw, projectDir, RUST_FW),
+        layout: detectLayout(projectDir, ["src", "tests", "examples", "benches"]) });
+    } else if (m.stack === "go") {
+      out.push({ stack: "go", manifest: m.manifest, modulePath: m.name || null,
+        frameworks: matchFw(raw, projectDir, GO_FW),
+        layout: detectLayout(projectDir, ["cmd", "internal", "pkg"]) });
+    }
+  }
+  return out;
+}

package/src/onboarder/collectors/index.js

@@ -7,6 +7,8 @@ import { readdir } from "node:fs/promises";
 import { join, relative } from "node:path";
 
 import { detectProjectStack } from "../../utils/stack-detect.js";
+import { sniffManifests } from "../../utils/manifest-sniff.js";
+import { detectFrameworksMultiLang } from "./framework-multilang.js";
 
 const IGNORED_DIRS = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", "__pycache__"]);
 
@@ -100,16 +102,31 @@ export async function collectAdrs(projectDir) {
   return found;
 }
 
+// Read a manifest file as raw text (best-effort) for framework dep matching.
+function readManifestRaw(projectDir) {
+  return (m) => {
+    try { return readFileSync(join(projectDir, m.manifest), "utf8"); }
+    catch { return ""; }
+  };
+}
+
 // One-shot bundle: every slot is independent; missing slots → null/[].
+// KJC-TSK-0477: manifests (multi-stack) + frameworksMultiLang (Python/Rust/Go)
+// se añaden de forma aditiva para que el brief pueda incluir secciones
+// condicionales sin romper consumidores existentes.
 export async function collectAll(projectDir) {
-  const [stack, tree, adrs] = await Promise.all([
+  const [stack, tree, adrs, manifests] = await Promise.all([
     detectProjectStack(projectDir).catch(() => null),
     collectTree(projectDir).catch(() => []),
     collectAdrs(projectDir).catch(() => []),
+    sniffManifests(projectDir).catch(() => []),
   ]);
+  const frameworksMultiLang = detectFrameworksMultiLang(projectDir, manifests, { readManifest: readManifestRaw(projectDir) });
   return {
     projectDir,
     stack,
+    manifests,
+    frameworksMultiLang,
     tree,
     git: collectGitHistory(projectDir),
     configs: collectConfigs(projectDir),

package/src/orchestrator/stages/coder-stage.js

@@ -19,6 +19,7 @@ import { invokeSolomon } from "../solomon-escalation.js";
 import { detectRateLimit } from "../../utils/rate-limit-detector.js";
 import { createStallDetector } from "../../utils/stall-detector.js";
 import { buildStandbyState } from "../../brain/standby-store.js";
+import { applyQuotaSimulation } from "../../utils/quota-simulator.js";
 import { snapshotHomeTopLevel, detectNewHomeEntries, formatLeakMessage, verifyLeaksAgainstTranscript, detectTranscriptCdLeaks } from "../fs-leak-detector.js";
 
 export async function runCoderStage({ coderRoleInstance, coderRole, config, logger, emitter, eventBase, session, plannedTask, trackBudget, iteration, brainCtx, acceptanceTests = null, adrs = null, specSection = null, reviewerFindings = null, huId = null }) {
@@ -76,6 +77,7 @@ export async function runCoderStage({ coderRoleInstance, coderRole, config, logg
     coderStall.stop();
   }
   trackBudget({ role: "coder", provider: coderRole.provider, model: coderRole.model, result: coderExecResult.result, duration_ms: Date.now() - coderStart });
+  applyQuotaSimulation(coderExecResult, { iteration, agent: coderRole.provider });
 
   if (!coderExecResult.ok) {
     const details = coderExecResult.result?.error || coderExecResult.summary || "unknown error";
@@ -281,6 +283,7 @@ export async function runRefactorerStage({ refactorerRole, config, logger, emitt
     refactorerStall.stop();
   }
   trackBudget({ role: "refactorer", provider: refactorerRole.provider, model: refactorerRole.model, result: refResult.result, duration_ms: Date.now() - refactorerStart });
+  applyQuotaSimulation(refResult, { iteration, agent: refactorerRole.provider });
   if (!refResult.ok) {
     const details = refResult.result?.error || refResult.summary || "unknown error";
     // Quota cap → hibernate cleanly, same as the coder stage above.

package/src/orchestrator/stages/reviewer-stage.js

@@ -13,6 +13,7 @@ import { runReviewerWithFallback } from "../reviewer-fallback.js";
 import { invokeSolomon } from "../solomon-escalation.js";
 import { detectRateLimit } from "../../utils/rate-limit-detector.js";
 import { createStallDetector } from "../../utils/stall-detector.js";
+import { applyQuotaSimulation } from "../../utils/quota-simulator.js";
 
 function categorizeIssues(issues) {
   const categories = { security: 0, correctness: 0, tests: 0, style: 0, other: 0 };
@@ -269,6 +270,16 @@ export async function runReviewerStage({ reviewerRole, config, logger, emitter,
     reviewerStall.stop();
   }
 
+  if (applyQuotaSimulation(reviewerExec.execResult, { iteration, agent: reviewerRole.provider })) {
+    // Reviewer wraps the agent call: detectRateLimit reads `lastAttempt.result.error`,
+    // so propagate the synthetic line into the most recent attempt too.
+    const last = reviewerExec.attempts?.at(-1);
+    if (last) {
+      if (!last.result || typeof last.result !== "object") last.result = {};
+      last.result.error = reviewerExec.execResult.result.error;
+    }
+  }
+
   if (!reviewerExec.execResult?.ok) {
     const lastAttempt = reviewerExec.attempts.at(-1);
     const details =

package/src/rag/auto-update.js

@@ -0,0 +1,50 @@
+// KJC-TSK-0455 PR2 — Pre-run drift check + post-merge hook installer.
+// Keeps the local RAG index aligned with HEAD so retrieval never serves
+// stale code without the user having to run `kj rag index` by hand.
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { execa } from "execa";
+
+import { openVecStore, projectSlug, getLastIndexedCommit, setLastIndexedCommit } from "./vec-store.js";
+import { indexProjectDelta } from "./indexer.js";
+import { makeEmbedder } from "./embedders/factory.js";
+
+const HOOK_SRC = resolve(fileURLToPath(import.meta.url), "../../../scripts/git-hooks/post-merge");
+
+export async function maybeAutoUpdate({ projectDir, config, logger = console, flags = {} } = {}) {
+  // commander turns `--no-rag-update` into flags.ragUpdate === false.
+  if (flags?.ragUpdate === false || config?.rag?.autoUpdate?.onRun === false) return { skipped: true };
+  if (!projectDir || !existsSync(join(projectDir, ".git"))) return { skipped: true };
+  let head;
+  try { const r = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]); head = r.stdout.trim(); }
+  catch { return { skipped: true }; }
+  const slug = projectSlug(projectDir);
+  const db = openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
+  try {
+    const since = getLastIndexedCommit(db, slug);
+    if (!since || since === head) return { skipped: true, head };
+    logger.info?.(`[rag] drift detected (${since.slice(0, 7)} → ${head.slice(0, 7)}); running delta update`);
+    const totals = await indexProjectDelta(projectDir, { db, embedder: makeEmbedder(config), since, logger });
+    if (totals.head) setLastIndexedCommit(db, slug, totals.head);
+    return { ran: true, totals };
+  } catch (err) {
+    logger.warn?.(`[rag] auto-update failed (${err.message}); continuing without refresh`);
+    return { failed: true, error: err.message };
+  } finally { db.close(); }
+}
+
+export function installPostMergeHook({ projectDir, logger = console } = {}) {
+  const hooksDir = join(projectDir, ".git", "hooks");
+  if (!existsSync(join(projectDir, ".git"))) throw new Error(`Not a git repository: ${projectDir}`);
+  if (!existsSync(hooksDir)) mkdirSync(hooksDir, { recursive: true });
+  const target = join(hooksDir, "post-merge");
+  const src = readFileSync(HOOK_SRC, "utf8");
+  if (existsSync(target) && !readFileSync(target, "utf8").includes("KJC-TSK-0455")) {
+    logger.warn?.(`[rag] ${target} already exists and was not installed by kj; leaving untouched`);
+    return { skipped: true, target };
+  }
+  writeFileSync(target, src);
+  chmodSync(target, 0o755);
+  return { installed: true, target };
+}

package/src/rag/chunker.js

@@ -92,13 +92,17 @@ export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT }
 // ---------- source chunker (export-symbol granularity) ------------
 
 import { chunkSourceAST } from "./chunker-ast.js";
+import { adapterForPath } from "../lang/registry.js";
 
 const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
 
-// KJC-TSK-0444 — try AST chunker first (top-level declarations as whole
-// units, JSDoc folded in); fall back to the export-regex if @babel/parser
-// can't parse the file (returns null).
+// KJC-TSK-0444 + KJC-PCS-0052 PR-B — dispatch por adapter cuando el path
+// coincide con un lenguaje registrado que aporta su propio `chunkSource`
+// (p.ej. Python regex top-level). Si no, mantiene el camino JS histórico:
+// AST de @babel/parser primero, fallback a export-regex.
 export function chunkSource(text, opts = {}) {
+  const ad = adapterForPath(opts.path);
+  if (ad?.chunkSource) return ad.chunkSource(text, opts);
   const ast = chunkSourceAST(text, opts);
   if (ast) return ast;
   return chunkSourceRegex(text, opts);

package/src/rag/eval.js

@@ -0,0 +1,61 @@
+// KJC-TSK-0483 — Retrieval-quality harness. Loads a golden set of queries
+// (each with expected source paths and/or symbol names), runs the retriever
+// against each, and reports recall@k + MRR per query and aggregated. Pure:
+// the runner takes `runQuery` as a function so the CLI plugs in `query()`
+// while tests can use a stub.
+import { readFileSync } from "node:fs";
+
+const DEFAULT_KS = [5, 10];
+
+export function loadGoldenQueries(path) {
+  const data = JSON.parse(readFileSync(path, "utf8"));
+  if (!Array.isArray(data)) throw new Error(`golden queries: expected an array in ${path}`);
+  return data;
+}
+
+function hitMatchesExpected(hit, expected) {
+  if (!hit) return false;
+  const src = hit.source || "";
+  if (expected.sources?.some((s) => src.endsWith(s))) return true;
+  const sym = hit.metadata?.symbol;
+  if (sym && expected.symbols?.includes(sym)) return true;
+  return false;
+}
+
+// Per-query metrics. `recall@k` is binary — 1 if any expected match lands in
+// the top-k, else 0. `rr` is the reciprocal of the first relevant rank (1 if
+// the very first hit matches, 0 if none do). `firstRank` is null on miss so
+// callers can render "—" without re-checking.
+export function scoreQuery(hits, expected, ks = DEFAULT_KS) {
+  const recall = {};
+  for (const k of ks) {
+    recall[`@${k}`] = hits.slice(0, k).some((h) => hitMatchesExpected(h, expected)) ? 1 : 0;
+  }
+  let firstRank = null;
+  let rr = 0;
+  for (let i = 0; i < hits.length; i += 1) {
+    if (hitMatchesExpected(hits[i], expected)) { firstRank = i + 1; rr = 1 / firstRank; break; }
+  }
+  return { recall, rr, firstRank };
+}
+
+export function aggregate(results, ks = DEFAULT_KS) {
+  const n = results.length || 1;
+  const recall = {};
+  for (const k of ks) {
+    recall[`@${k}`] = results.reduce((acc, r) => acc + (r.recall[`@${k}`] || 0), 0) / n;
+  }
+  const mrr = results.reduce((acc, r) => acc + (r.rr || 0), 0) / n;
+  return { n: results.length, recall, mrr };
+}
+
+export async function runEval(queries, runQuery, { topK = 10, ks = DEFAULT_KS } = {}) {
+  const results = [];
+  for (const q of queries) {
+    const hits = (await runQuery(q.query, topK)) || [];
+    const expected = { sources: q.expected_sources || [], symbols: q.expected_symbols || [] };
+    const scored = scoreQuery(hits, expected, ks);
+    results.push({ query: q.query, hits, ...scored });
+  }
+  return { results, aggregate: aggregate(results, ks) };
+}

package/src/rag/indexer.js

@@ -6,11 +6,19 @@
 // failure mode simple (re-run to refresh).
 import { existsSync, readFileSync } from "node:fs";
 import { readdir } from "node:fs/promises";
-import { extname, join, relative } from "node:path";
+import { extname, isAbsolute, join, relative } from "node:path";
+import { createHash } from "node:crypto";
+import { execa } from "execa";
 
 import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
-import { insertChunk, deleteChunksBySource } from "./vec-store.js";
+import { insertChunk, deleteChunksBySource, findChunkByHash } from "./vec-store.js";
+import { detectAdaptersForProject, buildMatchers, getAllCodeExtensions } from "../lang/registry.js";
 
+// KJC-PCS-0052 PR-A — antes vivían dos consts hard-coded para JS aquí
+// (CODE_EXT_RE + SKIP_SEGMENTS). Ahora vienen del registry de adapters,
+// que se consulta una vez por proceso para el conjunto global y por
+// projectDir cuando hace falta filtrar el walk.
+const ALL_CODE_EXTENSIONS = new Set(getAllCodeExtensions());
 const ONBOARDING_DIR = "onboarding", PLANS_DIR = "plans";
 
 function detectKind(path) {
@@ -18,7 +26,7 @@ function detectKind(path) {
   const base = path.split("/").pop() || "";
   if (ext === ".json" && (/^plan-.+\.json$/i.test(base) || path.includes(`/${PLANS_DIR}/`))) return "plan";
   if (ext === ".md") return path.includes(`/${ONBOARDING_DIR}/`) ? "onboarding" : "plan";
-  if ([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx"].includes(ext)) return "code";
+  if (ALL_CODE_EXTENSIONS.has(ext)) return "code";
   return null;
 }
 
@@ -46,28 +54,50 @@ export async function indexFile(path, { db, embedder, logger = console, project
   const kind = detectKind(path);
   if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
   const chunks = chunksFor(path, kind);
-  if (!chunks.length) return { indexed: 0, failed: 0 };
+  if (!chunks.length) return { indexed: 0, failed: 0, skipped: 0 };
   deleteChunksBySource(db, path);
   let indexed = 0;
   let failed = 0;
+  let skipped = 0;
   for (const ch of chunks) {
+    // KJC-TSK-0484 — Skip re-embedding identical bodies for the same project.
+    // The chunks row is still rewritten under the new source so deletions of
+    // an alias don't orphan a row whose owner was the dedup target.
+    const contentHash = createHash("sha256").update(ch.text).digest("hex");
     try {
+      const dup = findChunkByHash(db, contentHash, project);
+      if (dup) { skipped += 1; continue; }
       const embedding = await embedder.embed(ch.text);
-      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project });
+      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project, contentHash });
       indexed += 1;
     } catch (err) {
       failed += 1;
       logger.warn?.(`[rag-indexer] embed failed for ${path} (${chunkLabel(ch)}): ${err.message}`);
     }
   }
-  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed)`);
-  return { indexed, failed };
+  const note = skipped ? `, ${skipped} dedup` : "";
+  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed${note})`);
+  return { indexed, failed, skipped };
 }
 
 function chunkLabel(ch) {
   return ch.metadata?.hu_id || ch.metadata?.symbol || ch.metadata?.headingPath?.join(" > ") || "block";
 }
 
+// KJC-PCS-0052 PR-B.2.4 — Activa el camino AST en chunkers cuyo adapter
+// expone `prepare()` (Python, Rust). Promise.allSettled aísla cada grammar:
+// si uno falla (red, wasm corrupto), el resto se carga y los .py/.rs sin
+// AST listo caen al fallback regex en chunkSource. JS no tiene `prepare`
+// porque chunker-ast.js usa @babel/parser síncrono (ya cargado).
+export async function prepareAdapters(adapters, { logger = console } = {}) {
+  const tasks = adapters.filter((a) => typeof a.prepare === "function");
+  if (tasks.length === 0) return;
+  const results = await Promise.allSettled(tasks.map((a) => a.prepare()));
+  results.forEach((r, i) => {
+    if (r.status === "rejected") logger.warn?.(`[rag-indexer] adapter ${tasks[i].id} prepare failed: ${r.reason?.message || r.reason}`);
+  });
+}
+
 async function listFiles(dir, predicate) {
   const out = [];
   async function walk(d) {
@@ -91,6 +121,7 @@ async function listFiles(dir, predicate) {
 export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false } = {}) {
   const totals = { indexed: 0, failed: 0, files: 0 };
   const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  await prepareAdapters(detectAdaptersForProject(projectDir), { logger });
   const planRoot = join(karajanHome, PLANS_DIR, slug);
   if (existsSync(planRoot)) {
     const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
@@ -109,9 +140,9 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
     // per HU during `kj run`; they hold complete copies of `src/` that
     // duplicate every chunk and contaminate retrieval scores. `_diet` is the
     // tests/_diet/ sandbox used by the test-diet audit harness — never user
-    // code. Skip both.
-    const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
-    const sources = await listFiles(projectDir, (p) => /\.(js|mjs|cjs|ts|tsx|jsx)$/.test(p) && !p.split("/").some((seg) => SKIP.has(seg)));
+    // code. Both viven en COMMON_SKIP_SEGMENTS dentro del registry.
+    const matchers = buildMatchers(detectAdaptersForProject(projectDir));
+    const sources = await listFiles(projectDir, (p) => matchers.isCodeFile(p) && !matchers.shouldSkip(p));
     for (const s of sources) {
       const r = await indexFile(s, { db, embedder, logger, project: slug });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
@@ -119,3 +150,43 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
   }
   return totals;
 }
+
+/**
+ * KJC-TSK-0455 — Re-index only the files changed between `since` and HEAD.
+ * `git diff --name-status -z` drives the work: A/M re-embed, D drop chunks,
+ * R drops old + indexes new. Returns `{ indexed, failed, files, deleted, head }`;
+ * `head` lets callers persist via setLastIndexedCommit. Throws on diff failure
+ * (shallow clone, unknown ref) so the CLI can fall back to a full reindex.
+ */
+export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console } = {}) {
+  const totals = { indexed: 0, failed: 0, files: 0, deleted: 0, head: null };
+  const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  const { stdout: head } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
+  totals.head = head.trim();
+  if (totals.head === since) return totals;
+  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, "HEAD"]);
+  const parts = stdout.split("\0").filter(Boolean);
+  const ops = []; // { kind: 'del'|'idx', p }
+  for (let i = 0; i < parts.length; i += 1) {
+    const s = parts[i];
+    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[++i] }, { kind: "idx", p: parts[++i] }); }
+    else if (s === "D") { ops.push({ kind: "del", p: parts[++i] }); }
+    else { ops.push({ kind: "idx", p: parts[++i] }); }
+  }
+  const adapters = detectAdaptersForProject(projectDir);
+  await prepareAdapters(adapters, { logger });
+  const matchers = buildMatchers(adapters);
+  for (const { kind, p } of ops) {
+    if (matchers.shouldSkip(p)) continue;
+    if (!matchers.isCodeFile(p)) continue;
+    const abs = isAbsolute(p) ? p : join(projectDir, p);
+    if (kind === "del") {
+      totals.deleted += deleteChunksBySource(db, abs);
+      logger.info?.(`[rag-indexer] delta delete ${relative(process.cwd(), abs)}`);
+    } else if (existsSync(abs)) {
+      const r = await indexFile(abs, { db, embedder, logger, project: slug });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  return totals;
+}

package/src/rag/retriever.js

@@ -1,5 +1,5 @@
 // KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
-import { searchSimilar, searchBM25 } from "./vec-store.js";
+import { searchSimilar, searchBM25, getEmbeddingsByIds } from "./vec-store.js";
 import { parseWhere, buildWhereSql } from "./where-parser.js";
 import { rerank } from "./rerank.js";
 
@@ -41,7 +41,46 @@ function fuseHits(semantic, keyword, alpha, mode) {
   return list;
 }
 
-export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null } = {}) {
+// KJC-TSK-0484 PR-B — Maximal Marginal Relevance. Given an ordered list of
+// candidates (best-first by `score`), pick `topK` that balance relevance
+// against intra-result diversity. `lambda=1` collapses to plain relevance;
+// `lambda=0` maximises diversity. Cosine sim between candidate embeddings
+// drives the redundancy penalty. Candidates without an embedding are kept
+// at the end (no penalty applies).
+function cosineSim(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0; let na = 0; let nb = 0;
+  for (let i = 0; i < a.length; i += 1) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export function mmrRerank(candidates, embeddings, { topK, lambda = 0.7 }) {
+  const remaining = candidates.slice();
+  const picked = [];
+  const relevance = (c) => 1 / (1 + Math.max(0, c.score ?? c.distance ?? 0));
+  while (picked.length < topK && remaining.length > 0) {
+    let bestIdx = 0; let bestVal = -Infinity;
+    for (let i = 0; i < remaining.length; i += 1) {
+      const c = remaining[i];
+      const rel = relevance(c);
+      let maxSim = 0;
+      const ce = embeddings.get(c.id);
+      if (ce && picked.length > 0) {
+        for (const p of picked) {
+          const pe = embeddings.get(p.id);
+          if (pe) maxSim = Math.max(maxSim, cosineSim(ce, pe));
+        }
+      }
+      const score = lambda * rel - (1 - lambda) * maxSim;
+      if (score > bestVal) { bestVal = score; bestIdx = i; }
+    }
+    picked.push(remaining.splice(bestIdx, 1)[0]);
+  }
+  return picked;
+}
+
+export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null, diversify = false, mmrLambda = 0.7 } = {}) {
   if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
   const fetchK = Math.min(50, topK * 2);
   const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
@@ -59,11 +98,24 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
   const raw = fuseHits(semanticHits, keywordHits, alpha, mode);
   if (raw.length === 0) return [];
   const boostSources = shouldBoostSources(text);
-  const reranked = raw.map((r) => {
+  const scored = raw.map((r) => {
     const kindB = kindBoost[r.kind] || 0;
     const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
     return { ...r, score: r.distance - kindB - sourceB };
-  }).sort((a, b) => a.score - b.score).slice(0, topK);
+  }).sort((a, b) => a.score - b.score);
+  // KJC-TSK-0484 PR-B — MMR diversification over topK*2 candidates. Fetches
+  // embeddings only when requested; cosine between candidate vectors drives
+  // the redundancy penalty so near-duplicates that survived dedup (e.g. a
+  // boilerplate chunk repeated under slightly different wording) get pushed
+  // down. Skipped entirely when `diversify=false` (default).
+  let reranked;
+  if (diversify) {
+    const pool = scored.slice(0, Math.min(scored.length, topK * 2));
+    const embeddings = getEmbeddingsByIds(db, pool.map((c) => c.id));
+    reranked = mmrRerank(pool, embeddings, { topK, lambda: mmrLambda });
+  } else {
+    reranked = scored.slice(0, topK);
+  }
   // KJC-TSK-0449 — optional cross-encoder rerank. When `rerankOpts` is set,
   // we re-score the topK survivors with a (query, passage) cross-encoder;
   // the kind+source boost has already been applied, so the rerank acts as
@@ -72,4 +124,4 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
   return reranked;
 }
 
-export { shouldBoostSources, isTestPath, fuseHits };
+export { shouldBoostSources, isTestPath, fuseHits, cosineSim };

package/src/rag/vec-store.js

@@ -53,6 +53,26 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
     db.exec("ALTER TABLE chunks ADD COLUMN project_slug TEXT;");
     db.exec("CREATE INDEX IF NOT EXISTS chunks_by_project ON chunks(project_slug);");
   }
+  // KJC-TSK-0484 — Content-hash dedup. SHA-256 of the chunk text lets the
+  // indexer skip embedding bodies it has already seen for the same project.
+  // Older rows keep NULL; the lookup ignores NULLs so they neither dedup
+  // nor get deduped until they are re-indexed and stamped.
+  if (!cols.includes("content_hash")) {
+    db.exec("ALTER TABLE chunks ADD COLUMN content_hash TEXT;");
+    db.exec("CREATE INDEX IF NOT EXISTS chunks_by_hash ON chunks(content_hash, project_slug);");
+  }
+  // KJC-TSK-0455 — Auto-update by commit diff. Per-project metadata tracks
+  // the last commit whose changes were reflected in the index, so subsequent
+  // `kj rag index --since auto` invocations (and the post-merge hook / pre-run
+  // drift check landed in PR2) only re-embed the actual delta instead of the
+  // whole project.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS vec_store_meta (
+      project_slug TEXT PRIMARY KEY,
+      last_indexed_commit TEXT,
+      last_indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
   // KJC-TSK-0443 — FTS5 keyword index for BM25 hybrid scoring. content='chunks'
   // makes it a contentless mirror linked by id (no double storage); triggers
@@ -85,13 +105,13 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
  * because the virtual table rowid is set explicitly to the freshly
  * minted chunks.id. Returns that id.
  */
-export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null }) {
+export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null, contentHash = null }) {
   if (!Array.isArray(embedding) && !ArrayBuffer.isView(embedding)) {
     throw new Error("insertChunk: embedding must be an array or typed array of floats");
   }
   const meta = metadata == null ? null : (typeof metadata === "string" ? metadata : JSON.stringify(metadata));
-  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug) VALUES (?, ?, ?, ?, ?)");
-  const info = ins.run(source, kind, text, meta, project);
+  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug, content_hash) VALUES (?, ?, ?, ?, ?, ?)");
+  const info = ins.run(source, kind, text, meta, project, contentHash);
   // sqlite-vec's vec0 virtual table requires the rowid to arrive as a BigInt
   // even for values that comfortably fit in a Number. Better-sqlite3 returns
   // lastInsertRowid as BigInt only when safeIntegers mode is on; force it.
@@ -153,6 +173,52 @@ export function deleteChunksBySource(db, source) {
   return ids.length;
 }
 
+// KJC-TSK-0455 — Per-project commit watermark for `kj rag index --since auto`
+// and the post-merge / pre-run drift hooks landing in PR2.
+export function getLastIndexedCommit(db, projectSlug) {
+  if (!projectSlug) return null;
+  const row = db.prepare("SELECT last_indexed_commit FROM vec_store_meta WHERE project_slug = ?").get(projectSlug);
+  return row?.last_indexed_commit ?? null;
+}
+
+export function setLastIndexedCommit(db, projectSlug, commit) {
+  if (!projectSlug || !commit) return;
+  db.prepare(`
+    INSERT INTO vec_store_meta (project_slug, last_indexed_commit, last_indexed_at)
+    VALUES (?, ?, datetime('now'))
+    ON CONFLICT(project_slug) DO UPDATE SET
+      last_indexed_commit = excluded.last_indexed_commit,
+      last_indexed_at = excluded.last_indexed_at
+  `).run(projectSlug, commit);
+}
+
+// KJC-TSK-0484 PR-B — Bulk fetch embeddings for an explicit set of chunk ids.
+// Returns Map<id, Float32Array>. Used by MMR diversification in retriever.js.
+export function getEmbeddingsByIds(db, ids) {
+  const out = new Map();
+  if (!ids?.length) return out;
+  const placeholders = ids.map(() => "?").join(",");
+  const rows = db.prepare(`SELECT rowid AS id, embedding FROM vec_chunks WHERE rowid IN (${placeholders})`).all(...ids);
+  for (const row of rows) {
+    if (!row.embedding) continue;
+    const buf = Buffer.isBuffer(row.embedding) ? row.embedding : Buffer.from(row.embedding);
+    out.set(Number(row.id), new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
+  }
+  return out;
+}
+
+// KJC-TSK-0484 — Lookup an already-indexed chunk by content_hash. Scoped to a
+// project so two repos with the same boilerplate file don't cross-pollute.
+// Returns `{ id, source }` or null. Pass `project=null` to match NULL slugs.
+export function findChunkByHash(db, hash, project = null) {
+  if (!hash) return null;
+  const sql = project
+    ? "SELECT id, source FROM chunks WHERE content_hash = ? AND project_slug = ? LIMIT 1"
+    : "SELECT id, source FROM chunks WHERE content_hash = ? AND project_slug IS NULL LIMIT 1";
+  const row = project ? db.prepare(sql).get(hash, project) : db.prepare(sql).get(hash);
+  return row || null;
+}
+
 export function countChunks(db, { kind = null } = {}) {
   if (kind) return db.prepare("SELECT COUNT(*) AS n FROM chunks WHERE kind = ?").get(kind).n;
   return db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;