karajan-code 2.32.0 → 2.34.0

package/src/rag/indexer.js

@@ -6,11 +6,19 @@
 // failure mode simple (re-run to refresh).
 import { existsSync, readFileSync } from "node:fs";
 import { readdir } from "node:fs/promises";
-import { extname, join, relative } from "node:path";
+import { extname, isAbsolute, join, relative } from "node:path";
+import { createHash } from "node:crypto";
+import { execa } from "execa";
 
 import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
-import { insertChunk, deleteChunksBySource } from "./vec-store.js";
+import { insertChunk, deleteChunksBySource, findChunkByHash } from "./vec-store.js";
+import { detectAdaptersForProject, buildMatchers, getAllCodeExtensions } from "../lang/registry.js";
 
+// KJC-PCS-0052 PR-A — antes vivían dos consts hard-coded para JS aquí
+// (CODE_EXT_RE + SKIP_SEGMENTS). Ahora vienen del registry de adapters,
+// que se consulta una vez por proceso para el conjunto global y por
+// projectDir cuando hace falta filtrar el walk.
+const ALL_CODE_EXTENSIONS = new Set(getAllCodeExtensions());
 const ONBOARDING_DIR = "onboarding", PLANS_DIR = "plans";
 
 function detectKind(path) {
@@ -18,7 +26,7 @@ function detectKind(path) {
   const base = path.split("/").pop() || "";
   if (ext === ".json" && (/^plan-.+\.json$/i.test(base) || path.includes(`/${PLANS_DIR}/`))) return "plan";
   if (ext === ".md") return path.includes(`/${ONBOARDING_DIR}/`) ? "onboarding" : "plan";
-  if ([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx"].includes(ext)) return "code";
+  if (ALL_CODE_EXTENSIONS.has(ext)) return "code";
   return null;
 }
 
@@ -46,28 +54,50 @@ export async function indexFile(path, { db, embedder, logger = console, project
   const kind = detectKind(path);
   if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
   const chunks = chunksFor(path, kind);
-  if (!chunks.length) return { indexed: 0, failed: 0 };
+  if (!chunks.length) return { indexed: 0, failed: 0, skipped: 0 };
   deleteChunksBySource(db, path);
   let indexed = 0;
   let failed = 0;
+  let skipped = 0;
   for (const ch of chunks) {
+    // KJC-TSK-0484 — Skip re-embedding identical bodies for the same project.
+    // The chunks row is still rewritten under the new source so deletions of
+    // an alias don't orphan a row whose owner was the dedup target.
+    const contentHash = createHash("sha256").update(ch.text).digest("hex");
     try {
+      const dup = findChunkByHash(db, contentHash, project);
+      if (dup) { skipped += 1; continue; }
       const embedding = await embedder.embed(ch.text);
-      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project });
+      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project, contentHash });
       indexed += 1;
     } catch (err) {
       failed += 1;
       logger.warn?.(`[rag-indexer] embed failed for ${path} (${chunkLabel(ch)}): ${err.message}`);
     }
   }
-  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed)`);
-  return { indexed, failed };
+  const note = skipped ? `, ${skipped} dedup` : "";
+  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed${note})`);
+  return { indexed, failed, skipped };
 }
 
 function chunkLabel(ch) {
   return ch.metadata?.hu_id || ch.metadata?.symbol || ch.metadata?.headingPath?.join(" > ") || "block";
 }
 
+// KJC-PCS-0052 PR-B.2.4 — Activa el camino AST en chunkers cuyo adapter
+// expone `prepare()` (Python, Rust). Promise.allSettled aísla cada grammar:
+// si uno falla (red, wasm corrupto), el resto se carga y los .py/.rs sin
+// AST listo caen al fallback regex en chunkSource. JS no tiene `prepare`
+// porque chunker-ast.js usa @babel/parser síncrono (ya cargado).
+export async function prepareAdapters(adapters, { logger = console } = {}) {
+  const tasks = adapters.filter((a) => typeof a.prepare === "function");
+  if (tasks.length === 0) return;
+  const results = await Promise.allSettled(tasks.map((a) => a.prepare()));
+  results.forEach((r, i) => {
+    if (r.status === "rejected") logger.warn?.(`[rag-indexer] adapter ${tasks[i].id} prepare failed: ${r.reason?.message || r.reason}`);
+  });
+}
+
 async function listFiles(dir, predicate) {
   const out = [];
   async function walk(d) {
@@ -91,6 +121,7 @@ async function listFiles(dir, predicate) {
 export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false } = {}) {
   const totals = { indexed: 0, failed: 0, files: 0 };
   const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  await prepareAdapters(detectAdaptersForProject(projectDir), { logger });
   const planRoot = join(karajanHome, PLANS_DIR, slug);
   if (existsSync(planRoot)) {
     const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
@@ -109,9 +140,9 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
     // per HU during `kj run`; they hold complete copies of `src/` that
     // duplicate every chunk and contaminate retrieval scores. `_diet` is the
     // tests/_diet/ sandbox used by the test-diet audit harness — never user
-    // code. Skip both.
-    const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
-    const sources = await listFiles(projectDir, (p) => /\.(js|mjs|cjs|ts|tsx|jsx)$/.test(p) && !p.split("/").some((seg) => SKIP.has(seg)));
+    // code. Both viven en COMMON_SKIP_SEGMENTS dentro del registry.
+    const matchers = buildMatchers(detectAdaptersForProject(projectDir));
+    const sources = await listFiles(projectDir, (p) => matchers.isCodeFile(p) && !matchers.shouldSkip(p));
     for (const s of sources) {
       const r = await indexFile(s, { db, embedder, logger, project: slug });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
@@ -119,3 +150,43 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
   }
   return totals;
 }
+
+/**
+ * KJC-TSK-0455 — Re-index only the files changed between `since` and HEAD.
+ * `git diff --name-status -z` drives the work: A/M re-embed, D drop chunks,
+ * R drops old + indexes new. Returns `{ indexed, failed, files, deleted, head }`;
+ * `head` lets callers persist via setLastIndexedCommit. Throws on diff failure
+ * (shallow clone, unknown ref) so the CLI can fall back to a full reindex.
+ */
+export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console } = {}) {
+  const totals = { indexed: 0, failed: 0, files: 0, deleted: 0, head: null };
+  const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  const { stdout: head } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
+  totals.head = head.trim();
+  if (totals.head === since) return totals;
+  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, "HEAD"]);
+  const parts = stdout.split("\0").filter(Boolean);
+  const ops = []; // { kind: 'del'|'idx', p }
+  for (let i = 0; i < parts.length; i += 1) {
+    const s = parts[i];
+    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[++i] }, { kind: "idx", p: parts[++i] }); }
+    else if (s === "D") { ops.push({ kind: "del", p: parts[++i] }); }
+    else { ops.push({ kind: "idx", p: parts[++i] }); }
+  }
+  const adapters = detectAdaptersForProject(projectDir);
+  await prepareAdapters(adapters, { logger });
+  const matchers = buildMatchers(adapters);
+  for (const { kind, p } of ops) {
+    if (matchers.shouldSkip(p)) continue;
+    if (!matchers.isCodeFile(p)) continue;
+    const abs = isAbsolute(p) ? p : join(projectDir, p);
+    if (kind === "del") {
+      totals.deleted += deleteChunksBySource(db, abs);
+      logger.info?.(`[rag-indexer] delta delete ${relative(process.cwd(), abs)}`);
+    } else if (existsSync(abs)) {
+      const r = await indexFile(abs, { db, embedder, logger, project: slug });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  return totals;
+}

package/src/rag/retriever.js

@@ -1,5 +1,5 @@
 // KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
-import { searchSimilar, searchBM25 } from "./vec-store.js";
+import { searchSimilar, searchBM25, getEmbeddingsByIds } from "./vec-store.js";
 import { parseWhere, buildWhereSql } from "./where-parser.js";
 import { rerank } from "./rerank.js";
 
@@ -41,7 +41,46 @@ function fuseHits(semantic, keyword, alpha, mode) {
   return list;
 }
 
-export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null } = {}) {
+// KJC-TSK-0484 PR-B — Maximal Marginal Relevance. Given an ordered list of
+// candidates (best-first by `score`), pick `topK` that balance relevance
+// against intra-result diversity. `lambda=1` collapses to plain relevance;
+// `lambda=0` maximises diversity. Cosine sim between candidate embeddings
+// drives the redundancy penalty. Candidates without an embedding are kept
+// at the end (no penalty applies).
+function cosineSim(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0; let na = 0; let nb = 0;
+  for (let i = 0; i < a.length; i += 1) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export function mmrRerank(candidates, embeddings, { topK, lambda = 0.7 }) {
+  const remaining = candidates.slice();
+  const picked = [];
+  const relevance = (c) => 1 / (1 + Math.max(0, c.score ?? c.distance ?? 0));
+  while (picked.length < topK && remaining.length > 0) {
+    let bestIdx = 0; let bestVal = -Infinity;
+    for (let i = 0; i < remaining.length; i += 1) {
+      const c = remaining[i];
+      const rel = relevance(c);
+      let maxSim = 0;
+      const ce = embeddings.get(c.id);
+      if (ce && picked.length > 0) {
+        for (const p of picked) {
+          const pe = embeddings.get(p.id);
+          if (pe) maxSim = Math.max(maxSim, cosineSim(ce, pe));
+        }
+      }
+      const score = lambda * rel - (1 - lambda) * maxSim;
+      if (score > bestVal) { bestVal = score; bestIdx = i; }
+    }
+    picked.push(remaining.splice(bestIdx, 1)[0]);
+  }
+  return picked;
+}
+
+export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null, diversify = false, mmrLambda = 0.7 } = {}) {
   if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
   const fetchK = Math.min(50, topK * 2);
   const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
@@ -59,11 +98,24 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
   const raw = fuseHits(semanticHits, keywordHits, alpha, mode);
   if (raw.length === 0) return [];
   const boostSources = shouldBoostSources(text);
-  const reranked = raw.map((r) => {
+  const scored = raw.map((r) => {
     const kindB = kindBoost[r.kind] || 0;
     const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
     return { ...r, score: r.distance - kindB - sourceB };
-  }).sort((a, b) => a.score - b.score).slice(0, topK);
+  }).sort((a, b) => a.score - b.score);
+  // KJC-TSK-0484 PR-B — MMR diversification over topK*2 candidates. Fetches
+  // embeddings only when requested; cosine between candidate vectors drives
+  // the redundancy penalty so near-duplicates that survived dedup (e.g. a
+  // boilerplate chunk repeated under slightly different wording) get pushed
+  // down. Skipped entirely when `diversify=false` (default).
+  let reranked;
+  if (diversify) {
+    const pool = scored.slice(0, Math.min(scored.length, topK * 2));
+    const embeddings = getEmbeddingsByIds(db, pool.map((c) => c.id));
+    reranked = mmrRerank(pool, embeddings, { topK, lambda: mmrLambda });
+  } else {
+    reranked = scored.slice(0, topK);
+  }
   // KJC-TSK-0449 — optional cross-encoder rerank. When `rerankOpts` is set,
   // we re-score the topK survivors with a (query, passage) cross-encoder;
   // the kind+source boost has already been applied, so the rerank acts as
@@ -72,4 +124,4 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
   return reranked;
 }
 
-export { shouldBoostSources, isTestPath, fuseHits };
+export { shouldBoostSources, isTestPath, fuseHits, cosineSim };

package/src/rag/vec-store.js

@@ -53,6 +53,26 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
     db.exec("ALTER TABLE chunks ADD COLUMN project_slug TEXT;");
     db.exec("CREATE INDEX IF NOT EXISTS chunks_by_project ON chunks(project_slug);");
   }
+  // KJC-TSK-0484 — Content-hash dedup. SHA-256 of the chunk text lets the
+  // indexer skip embedding bodies it has already seen for the same project.
+  // Older rows keep NULL; the lookup ignores NULLs so they neither dedup
+  // nor get deduped until they are re-indexed and stamped.
+  if (!cols.includes("content_hash")) {
+    db.exec("ALTER TABLE chunks ADD COLUMN content_hash TEXT;");
+    db.exec("CREATE INDEX IF NOT EXISTS chunks_by_hash ON chunks(content_hash, project_slug);");
+  }
+  // KJC-TSK-0455 — Auto-update by commit diff. Per-project metadata tracks
+  // the last commit whose changes were reflected in the index, so subsequent
+  // `kj rag index --since auto` invocations (and the post-merge hook / pre-run
+  // drift check landed in PR2) only re-embed the actual delta instead of the
+  // whole project.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS vec_store_meta (
+      project_slug TEXT PRIMARY KEY,
+      last_indexed_commit TEXT,
+      last_indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
   // KJC-TSK-0443 — FTS5 keyword index for BM25 hybrid scoring. content='chunks'
   // makes it a contentless mirror linked by id (no double storage); triggers
@@ -85,13 +105,13 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
  * because the virtual table rowid is set explicitly to the freshly
  * minted chunks.id. Returns that id.
  */
-export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null }) {
+export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null, contentHash = null }) {
   if (!Array.isArray(embedding) && !ArrayBuffer.isView(embedding)) {
     throw new Error("insertChunk: embedding must be an array or typed array of floats");
   }
   const meta = metadata == null ? null : (typeof metadata === "string" ? metadata : JSON.stringify(metadata));
-  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug) VALUES (?, ?, ?, ?, ?)");
-  const info = ins.run(source, kind, text, meta, project);
+  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug, content_hash) VALUES (?, ?, ?, ?, ?, ?)");
+  const info = ins.run(source, kind, text, meta, project, contentHash);
   // sqlite-vec's vec0 virtual table requires the rowid to arrive as a BigInt
   // even for values that comfortably fit in a Number. Better-sqlite3 returns
   // lastInsertRowid as BigInt only when safeIntegers mode is on; force it.
@@ -153,6 +173,52 @@ export function deleteChunksBySource(db, source) {
   return ids.length;
 }
 
+// KJC-TSK-0455 — Per-project commit watermark for `kj rag index --since auto`
+// and the post-merge / pre-run drift hooks landing in PR2.
+export function getLastIndexedCommit(db, projectSlug) {
+  if (!projectSlug) return null;
+  const row = db.prepare("SELECT last_indexed_commit FROM vec_store_meta WHERE project_slug = ?").get(projectSlug);
+  return row?.last_indexed_commit ?? null;
+}
+
+export function setLastIndexedCommit(db, projectSlug, commit) {
+  if (!projectSlug || !commit) return;
+  db.prepare(`
+    INSERT INTO vec_store_meta (project_slug, last_indexed_commit, last_indexed_at)
+    VALUES (?, ?, datetime('now'))
+    ON CONFLICT(project_slug) DO UPDATE SET
+      last_indexed_commit = excluded.last_indexed_commit,
+      last_indexed_at = excluded.last_indexed_at
+  `).run(projectSlug, commit);
+}
+
+// KJC-TSK-0484 PR-B — Bulk fetch embeddings for an explicit set of chunk ids.
+// Returns Map<id, Float32Array>. Used by MMR diversification in retriever.js.
+export function getEmbeddingsByIds(db, ids) {
+  const out = new Map();
+  if (!ids?.length) return out;
+  const placeholders = ids.map(() => "?").join(",");
+  const rows = db.prepare(`SELECT rowid AS id, embedding FROM vec_chunks WHERE rowid IN (${placeholders})`).all(...ids);
+  for (const row of rows) {
+    if (!row.embedding) continue;
+    const buf = Buffer.isBuffer(row.embedding) ? row.embedding : Buffer.from(row.embedding);
+    out.set(Number(row.id), new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
+  }
+  return out;
+}
+
+// KJC-TSK-0484 — Lookup an already-indexed chunk by content_hash. Scoped to a
+// project so two repos with the same boilerplate file don't cross-pollute.
+// Returns `{ id, source }` or null. Pass `project=null` to match NULL slugs.
+export function findChunkByHash(db, hash, project = null) {
+  if (!hash) return null;
+  const sql = project
+    ? "SELECT id, source FROM chunks WHERE content_hash = ? AND project_slug = ? LIMIT 1"
+    : "SELECT id, source FROM chunks WHERE content_hash = ? AND project_slug IS NULL LIMIT 1";
+  const row = project ? db.prepare(sql).get(hash, project) : db.prepare(sql).get(hash);
+  return row || null;
+}
+
 export function countChunks(db, { kind = null } = {}) {
   if (kind) return db.prepare("SELECT COUNT(*) AS n FROM chunks WHERE kind = ?").get(kind).n;
   return db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;

package/src/rag/watcher.js

@@ -1,23 +1,25 @@
 // KJC-TSK-0441 (RAG v2.28.0 PR1) — chokidar watcher. Live re-index of plans
 // + onboarding + (opt) sources, debounced. PID file arbitrates a single daemon.
+// KJC-TSK-0482 — source matcher derivado del registry (multi-lang: JS/Py/Rust/Go/Java).
 import { writeFileSync, readFileSync, existsSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
+import { extname, join } from "node:path";
 import chokidar from "chokidar";
 import { openVecStore, deleteChunksBySource, projectSlug } from "./vec-store.js";
 import { makeEmbedder } from "./embedders/factory.js";
 import { indexFile } from "./indexer.js";
 import { getKarajanHome } from "../utils/paths.js";
+import { getAllCodeExtensions } from "../lang/registry.js";
 
 const PIDFILE = () => join(getKarajanHome(), "watcher.pid");
 const DEFAULT_DEBOUNCE_MS = 1000;
 const SKIP_SEGMENTS = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
-const SOURCE_RE = /\.(js|mjs|cjs|ts|tsx|jsx)$/;
+const SOURCE_EXTS = new Set(getAllCodeExtensions().map((e) => e.toLowerCase()));
 
-function isWatchable(path) {
+export function isWatchable(path) {
   if (path.includes("/.karajan/onboarding/")) return /\.md$/.test(path);
   if (path.includes("/.karajan/plans/")) return /\.json$/.test(path);
   if (path.split("/").some((seg) => SKIP_SEGMENTS.has(seg))) return false;
-  return SOURCE_RE.test(path);
+  return SOURCE_EXTS.has(extname(path).toLowerCase());
 }
 
 export function startWatcher({ projectDir, config, logger = console, debounceMs = DEFAULT_DEBOUNCE_MS, withSources = false } = {}) {

package/src/utils/manifest-sniff.js

@@ -0,0 +1,82 @@
+// KJC-TSK-0476 — Multi-stack manifest sniffer. Sin libs externas: JSON.parse
+// para package.json, regex para TOML y go.mod. Lo consumen audit (basal-cost)
+// y, vía PR-D, el onboarder.
+import fs from "node:fs/promises";
+import path from "node:path";
+
+async function readMaybe(file) {
+  try { return await fs.readFile(file, "utf8"); } catch { return null; }
+}
+
+async function sniffNode(dir) {
+  const raw = await readMaybe(path.join(dir, "package.json"));
+  if (!raw) return null;
+  try {
+    const pkg = JSON.parse(raw);
+    return { stack: "node", manifest: "package.json", name: pkg.name || null,
+      deps: Object.keys(pkg.dependencies || {}).length,
+      devDeps: Object.keys(pkg.devDependencies || {}).length };
+  } catch {
+    return { stack: "node", manifest: "package.json", name: null, deps: 0, devDeps: 0, malformed: true };
+  }
+}
+
+// Cuerpo TOML de `[section]` hasta el siguiente `\n[` o EOF. Lookahead con `/m`
+// cambia el significado de `$` y producía capturas vacías.
+function tomlSection(toml, section) {
+  const i = toml.indexOf(`[${section}]`);
+  if (i === -1) return "";
+  const rest = toml.slice(i + section.length + 2);
+  const n = rest.search(/\n\[/);
+  return n === -1 ? rest : rest.slice(0, n);
+}
+
+const DEP_LINE_RE = /^\s*[A-Za-z0-9_-]+\s*=/;
+const countDepLines = (block, excludePython) => block.split("\n")
+  .filter((l) => DEP_LINE_RE.test(l) && !(excludePython && /^\s*python\s*=/.test(l))).length;
+
+async function sniffPython(dir) {
+  const py = await readMaybe(path.join(dir, "pyproject.toml"));
+  if (py) {
+    const name = /^name\s*=\s*["']([^"']+)["']/m.exec(py)?.[1] || null;
+    const projectDeps = /\[project\][\s\S]*?dependencies\s*=\s*\[([^\]]*)\]/m.exec(py)?.[1] || "";
+    const projectCount = projectDeps.split(",").map((s) => s.trim()).filter(Boolean).length;
+    const poetryCount = countDepLines(tomlSection(py, "tool.poetry.dependencies"), true);
+    return { stack: "python", manifest: "pyproject.toml", name, deps: projectCount + poetryCount, devDeps: 0 };
+  }
+  const req = await readMaybe(path.join(dir, "requirements.txt"));
+  if (!req) return null;
+  const lines = req.split("\n").map((l) => l.replace(/#.*$/, "").trim()).filter(Boolean);
+  return { stack: "python", manifest: "requirements.txt", name: null, deps: lines.length, devDeps: 0 };
+}
+
+async function sniffRust(dir) {
+  const toml = await readMaybe(path.join(dir, "Cargo.toml"));
+  if (!toml) return null;
+  return { stack: "rust", manifest: "Cargo.toml",
+    name: /^name\s*=\s*["']([^"']+)["']/m.exec(toml)?.[1] || null,
+    deps: countDepLines(tomlSection(toml, "dependencies")),
+    devDeps: countDepLines(tomlSection(toml, "dev-dependencies")) };
+}
+
+async function sniffGo(dir) {
+  const mod = await readMaybe(path.join(dir, "go.mod"));
+  if (!mod) return null;
+  const block = /require\s*\(([\s\S]*?)\)/m.exec(mod)?.[1] || "";
+  const blockCount = block.split("\n").filter((l) => /\S/.test(l) && !l.trim().startsWith("//")).length;
+  const singleCount = mod.split("\n").filter((l) => /^require\s+\S+\s+\S+/.test(l)).length;
+  return { stack: "go", manifest: "go.mod",
+    name: /^module\s+(\S+)/m.exec(mod)?.[1] || null,
+    deps: blockCount + singleCount, devDeps: 0 };
+}
+
+export async function sniffManifests(dir) {
+  const r = await Promise.all([sniffNode(dir), sniffPython(dir), sniffRust(dir), sniffGo(dir)]);
+  return r.filter(Boolean);
+}
+
+export function summariseStack(manifests) {
+  if (!manifests.length) return { primary: null, multi: false, manifests: [] };
+  const ranked = [...manifests].sort((a, b) => (b.deps + b.devDeps) - (a.deps + a.devDeps));
+  return { primary: ranked[0].stack, multi: manifests.length > 1, manifests };
+}

package/templates/roles/onboarder.md

@@ -14,7 +14,10 @@ If the project is greenfield (no git, no configs, empty tree) emit
 
 ## Input
 
-JSON bundle with `projectDir`, `stack`, `tree`, `git` (or null), `configs`,
+JSON bundle with `projectDir`, `stack`, `manifests` (per-stack: node /
+python / rust / go with deps count), `frameworksMultiLang` (Django / Flask /
+FastAPI / Axum / Actix / Rocket / Gin / Echo / Fiber + Rust crate type + Go
+module path + canonical layout dirs), `tree`, `git` (or null), `configs`,
 `adrs`. Slots may be null/empty.
 
 ## Output
@@ -24,6 +27,8 @@ Markdown only — skip any section without data:
 ```markdown
 # Architecture Brief — <projectDir>
 ## Stack
+## Manifests           # only when manifests has entries beyond node
+## Frameworks          # only when frameworksMultiLang has entries
 ## Layout
 ## Configs
 ## Git history
@@ -32,5 +37,9 @@ Markdown only — skip any section without data:
 ## Recommendations for new work
 ```
 
+For non-Node stacks (`frameworksMultiLang[].stack` in {python, rust, go}),
+mention the detected framework(s), crate type or module path, and canonical
+layout dirs (apps/, cmd/, internal/, src/, tests/) when present.
+
 Be conservative: only assert what the bundle shows. "Likely" / "appears to"
 are fine; invention is not. Aim for ~40–80 lines total.

package/vendor/tree-sitter-grammars/README.md

@@ -0,0 +1,21 @@
+# Vendored tree-sitter grammars
+
+WASM binaries used by `src/lang/tree-sitter-loader.js`. Vendored (rather
+than pulled as runtime deps) so the npm package and the SEA binary ship
+exactly the grammars we support — adding all of `@vscode/tree-sitter-wasm`
+would balloon the install by ~22 MB for languages we never chunk.
+
+Source: extracted from `@vscode/tree-sitter-wasm@0.3.1`. That package
+re-builds upstream `tree-sitter-*` grammars against the tree-sitter
+0.25.x ABI, which matches the runtime `web-tree-sitter@0.26.x` we depend
+on. Older prebuilds (e.g. `tree-sitter-wasms@0.1.13`) target an older ABI
+and fail to load.
+
+Re-extract with:
+
+    npm pack @vscode/tree-sitter-wasm
+    tar -xzf vscode-tree-sitter-wasm-*.tgz
+    cp package/wasm/tree-sitter-python.wasm vendor/tree-sitter-grammars/python.wasm
+    cp package/wasm/tree-sitter-rust.wasm   vendor/tree-sitter-grammars/rust.wasm
+    cp package/wasm/tree-sitter-go.wasm     vendor/tree-sitter-grammars/go.wasm
+    cp package/wasm/tree-sitter-java.wasm   vendor/tree-sitter-grammars/java.wasm