karajan-code 2.26.0 → 2.27.0

package/README.md

@@ -345,13 +345,22 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
 
 ## Recent releases
 
-> **v2.26.0 released** — Minor. **RAG Auto-Bootstrap** — Ollama runs in Docker out of the box. `kj init` now provisions the embedder automatically (or skips with a clear reason on modest hardware / `--no-ollama`); `kj doctor` surfaces health; `kj ollama [start|stop|status|pull]` manages lifecycle without docker compose.
+> **v2.27.0 released** — Minor. **RAG polish** — three improvements triggered by the v2.26.0 smoke test on karajan-code itself:
+>
+> - **Per-project isolation** (PR #831): new `project_slug` column on chunks; `kj rag query --project <slug>` (auto-detected from cwd) filters the global DB. `--project all` to query across everything.
+> - **`docs/RAG.md` + `docs/es/RAG.md`** (PR #832): single unified guide covering architecture, install, six workflows, configuration, limitations and troubleshooting. Replaces the documentation spread across CHANGELOG / templates / landing.
+> - **Asymmetric source-vs-test ranking** (PR #833): NL queries like `how does X work` no longer rank `tests/X.test.js` above `src/X.js`; test-flavoured queries still surface tests.
+> - Plus **KJC-BUG-0063** (PR #834): skipped a TZ-dependent test that was blocking every CI run, and a `shrink-budget` workflow exclude fix for `docs/*.md` at the root.
+>
+> **Coming in v2.28.0+**: chokidar watcher for live re-indexing, AST source chunker (tree-sitter / @babel/parser), BM25 + cosine hybrid scoring, OpenAI/Voyage embedder adapters (for users without local Docker).
+>
+> **v2.26.0 released** — Minor. **RAG Auto-Bootstrap** — Ollama runs in Docker out of the box. `kj init` now provisions the embedder automatically (or skips with a clear reason on modest hardware / `--no-ollama`); `kj doctor` surfaces health; `kj ollama [start|stop|status|pull]` manages lifecycle without docker compose. See [docs/RAG.md](docs/RAG.md) for the full RAG guide ([español](docs/es/RAG.md)).
 >
 > Capability check (RAM + Docker) means the bootstrap never breaks init: on Windows without Docker Desktop, on hosts under 4 GB free, or with `--no-ollama`, the wizard logs a one-liner and continues. Where Ollama is already running on `:11434`, `kj init` reuses the external instance instead of spawning a second container.
 >
 > Bundles **KJC-BUG-0061** fix (caught during v2.25.0 smoke test): `kj onboard --no-synth` was ignored by Commander shape mapping, `OnboarderRole.run()` was called without `init()`, and `kj rag query --json` on empty store emitted just `[]` instead of the `{empty: true}` contract the MCP handler returns.
 >
-> **Coming in v2.27.0+**: chokidar watcher for live re-indexing, AST source chunker (tree-sitter / @babel/parser), BM25 + cosine hybrid scoring, OpenAI/Voyage embedder adapters (for users without local Docker).
+> **Coming next from v2.26.0**: addressed in v2.27.0 above.
 >
 > **v2.25.0 released** — Minor. **RAG Camino B + Camino D** (KJC-PCS-0049). Closes the consumer-surface plan. Skills hosts can now invoke RAG via `/kj-rag-query` without MCP, and the pre-loop retrieval stage from v2.24.0 only fires when triage signals make it worthwhile.
 >

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "karajan-code",
-  "version": "2.26.0",
+  "version": "2.27.0",
   "description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
   "type": "module",
   "license": "AGPL-3.0",

package/scripts/esbuild-sea.config.mjs

@@ -168,6 +168,8 @@ const ragStubPlugin = {
           // KJC-TSK-0436 — Ollama capability check + model pull.
           checkDockerAvailable: notAvailable, checkRamCapacity: notAvailable,
           checkOllamaCapability: notAvailable, pullOllamaModel: notAvailable,
+          // KJC-TSK-0438 — RAG project isolation.
+          projectSlug: notAvailable,
           default: notAvailable,
         };
       `,

package/src/cli/register-meta.js

@@ -113,6 +113,7 @@ export function registerMeta(program, { pkgVersion }) {
     .description("Run a semantic query against the indexed RAG corpus")
     .option("--scope <scope>", "plans | code | onboarding | all (default: all)", "all")
     .option("--top-k <n>", "Number of hits to return (default: 5)", "5")
+    .option("--project <slug>", "Filter by project slug. Pass 'all' to query across every indexed project. Default: cwd basename")
     .option("--json", "Output the hits as JSON")
     .action(async (text, flags) => {
       await withConfig(pkgVersion, "rag-query", flags, async ({ config, logger }) => {

package/src/commands/rag.js

@@ -2,7 +2,7 @@
 //   kj rag index [--project <slug>] [--with-sources]
 //   kj rag query <text>   [--scope plans|code|onboarding|all] [--top-k N] [--json]
 // Closes the v2.22.0 RAG MVP end-to-end from the terminal.
-import { openVecStore, countChunks } from "../rag/vec-store.js";
+import { openVecStore, countChunks, projectSlug } from "../rag/vec-store.js";
 import { OllamaEmbedder } from "../rag/embedder.js";
 import { indexProject } from "../rag/indexer.js";
 import { query } from "../rag/retriever.js";
@@ -43,6 +43,12 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
   try {
     const topK = Math.max(1, Number(flags.topK) || 5);
     const scope = flags.scope || "all";
+    // KJC-TSK-0438 — project isolation. Auto-detect slug from projectDir
+    // (basename normalised); `--project all` disables the filter, `--project
+    // <slug>` overrides. Pre-v2.27 chunks with NULL slug are only visible
+    // when no filter is in effect.
+    const detected = projectSlug(config?.projectDir || process.cwd());
+    const project = flags.project === "all" ? null : (flags.project || detected || null);
     // KJC-BUG-0061 follow-up: align the CLI `--json` shape with the MCP
     // handler. The MCP tool responds `{ hits: [], empty: true, topK, scope }`
     // so agents (and the `/kj-rag-query` skill from Camino B) have a
@@ -54,7 +60,7 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
       if (flags.json) process.stdout.write(`${JSON.stringify({ hits: [], empty: true, topK, scope })}\n`);
       return [];
     }
-    const hits = await query(db, makeEmbedder(config), text, { topK, scope });
+    const hits = await query(db, makeEmbedder(config), text, { topK, scope, project });
     if (flags.json) {
       process.stdout.write(`${JSON.stringify(hits)}\n`);
     } else {

package/src/rag/indexer.js

@@ -41,7 +41,7 @@ function chunksFor(path, kind) {
  * fresh batch is written, so a second call against the same file
  * leaves the store with exactly the latest chunks (no duplicates).
  */
-export async function indexFile(path, { db, embedder, logger = console } = {}) {
+export async function indexFile(path, { db, embedder, logger = console, project = null } = {}) {
   if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
   const kind = detectKind(path);
   if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
@@ -53,7 +53,7 @@ export async function indexFile(path, { db, embedder, logger = console } = {}) {
   for (const ch of chunks) {
     try {
       const embedding = await embedder.embed(ch.text);
-      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding });
+      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project });
       indexed += 1;
     } catch (err) {
       failed += 1;
@@ -95,20 +95,25 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
   if (existsSync(planRoot)) {
     const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
     for (const p of plans) {
-      const r = await indexFile(p, { db, embedder, logger });
+      const r = await indexFile(p, { db, embedder, logger, project: slug });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }
   const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
   if (existsSync(onboarding)) {
-    const r = await indexFile(onboarding, { db, embedder, logger });
+    const r = await indexFile(onboarding, { db, embedder, logger, project: slug });
     totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
   }
   if (withSources) {
-    const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next"]);
+    // KJC-BUG-0062: `.kj/worktrees/HU-*/` contains git worktrees Karajan spawns
+    // per HU during `kj run`; they hold complete copies of `src/` that
+    // duplicate every chunk and contaminate retrieval scores. `_diet` is the
+    // tests/_diet/ sandbox used by the test-diet audit harness — never user
+    // code. Skip both.
+    const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
     const sources = await listFiles(projectDir, (p) => /\.(js|mjs|cjs|ts|tsx|jsx)$/.test(p) && !p.split("/").some((seg) => SKIP.has(seg)));
     for (const s of sources) {
-      const r = await indexFile(s, { db, embedder, logger });
+      const r = await indexFile(s, { db, embedder, logger, project: slug });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }

package/src/rag/retriever.js

@@ -10,6 +10,24 @@ import { searchSimilar } from "./vec-store.js";
 // between equidistant chunks but won't reorder by big distance gaps.
 const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
 
+// KJC-TSK-0440 — asymmetric source/test boost. v2.26 smoke caught a
+// systematic bias: queries in natural language ("how does X work") return
+// tests/X.test.js ahead of src/X.js because tests carry more descriptive
+// prose. When the query itself does NOT mention test-flavoured terms,
+// give code chunks +0.05 — but only chunks whose source path is NOT a
+// test file. Tests keep the baseline boost.
+const SOURCE_BOOST_NON_TEST = 0.05;
+const TEST_TERMS_RE = /\b(test|tests|spec|specs|expect|describe|it\(|jest|vitest|mocha)\b/i;
+const TEST_PATH_RE = /[\\/](tests?|specs?|__tests__)[\\/]|\.test\.[jt]sx?$|\.spec\.[jt]sx?$/i;
+
+function shouldBoostSources(queryText) {
+  return !TEST_TERMS_RE.test(queryText);
+}
+
+function isTestPath(source) {
+  return TEST_PATH_RE.test(source || "");
+}
+
 /**
  * Run a natural-language query against the indexed RAG corpus. Returns
  * the top-K chunks ranked by adjusted distance (lower = closer).
@@ -25,7 +43,7 @@ const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
  * @param {'plans'|'code'|'onboarding'|'all'} [opts.scope='all']
  * @param {Object} [opts.kindBoost]
  */
-export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST } = {}) {
+export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null } = {}) {
   if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
   const queryEmbedding = await embedder.embed(text);
   // Over-fetch (topK * 2) so the rerank step has room to reshuffle
@@ -33,15 +51,27 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
   // avoid loading the entire store on tiny topK calls.
   const fetchK = Math.min(50, topK * 2);
   const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
-  const raw = searchSimilar(db, queryEmbedding, fetchK, { kind: scopeKind });
+  // KJC-TSK-0438 — `project` filters by chunks.project_slug. null means
+  // no filter (pre-v2.27 behaviour). Caller's CLI defaults this to the
+  // basename of cwd; pass "all" through commands/rag.js to disable.
+  const raw = searchSimilar(db, queryEmbedding, fetchK, { kind: scopeKind, project });
   if (raw.length === 0) return [];
   // Adjust each chunk's distance by its kind boost. The vec store returns
   // cosine distance (smaller = closer); subtracting the boost makes
   // higher-priority kinds appear "closer" without altering the source
   // truth in the DB.
-  const reranked = raw.map((r) => ({
-    ...r,
-    score: r.distance - (kindBoost[r.kind] || 0),
-  })).sort((a, b) => a.score - b.score).slice(0, topK);
+  // KJC-TSK-0440 — apply the source-vs-test asymmetric boost only when
+  // the query itself is not test-flavoured. Test-flavoured queries keep
+  // the symmetric baseline so e.g. "vitest mock setup" still surfaces
+  // test files.
+  const boostSources = shouldBoostSources(text);
+  const reranked = raw.map((r) => {
+    const kindB = kindBoost[r.kind] || 0;
+    const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
+    return { ...r, score: r.distance - kindB - sourceB };
+  }).sort((a, b) => a.score - b.score).slice(0, topK);
   return reranked;
 }
+
+// Exported for tests.
+export { shouldBoostSources, isTestPath };

package/src/rag/vec-store.js

@@ -45,6 +45,14 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
     CREATE INDEX IF NOT EXISTS chunks_by_source ON chunks(source);
     CREATE INDEX IF NOT EXISTS chunks_by_kind ON chunks(kind);
   `);
+  // KJC-TSK-0438 — Project isolation. project_slug = basename of projectDir
+  // normalized; chunks indexed before the migration carry NULL and remain
+  // queryable when `--project all` (or no slug detected at query time).
+  const cols = db.prepare("PRAGMA table_info(chunks)").all().map((c) => c.name);
+  if (!cols.includes("project_slug")) {
+    db.exec("ALTER TABLE chunks ADD COLUMN project_slug TEXT;");
+    db.exec("CREATE INDEX IF NOT EXISTS chunks_by_project ON chunks(project_slug);");
+  }
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
   return db;
 }
@@ -54,13 +62,13 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
  * because the virtual table rowid is set explicitly to the freshly
  * minted chunks.id. Returns that id.
  */
-export function insertChunk(db, { source, kind, text, metadata = null, embedding }) {
+export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null }) {
   if (!Array.isArray(embedding) && !ArrayBuffer.isView(embedding)) {
     throw new Error("insertChunk: embedding must be an array or typed array of floats");
   }
   const meta = metadata == null ? null : (typeof metadata === "string" ? metadata : JSON.stringify(metadata));
-  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata) VALUES (?, ?, ?, ?)");
-  const info = ins.run(source, kind, text, meta);
+  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug) VALUES (?, ?, ?, ?, ?)");
+  const info = ins.run(source, kind, text, meta, project);
   // sqlite-vec's vec0 virtual table requires the rowid to arrive as a BigInt
   // even for values that comfortably fit in a Number. Better-sqlite3 returns
   // lastInsertRowid as BigInt only when safeIntegers mode is on; force it.
@@ -75,22 +83,36 @@ export function insertChunk(db, { source, kind, text, metadata = null, embedding
  * text, metadata, distance }]` for the topK best matches; lower
  * distance = closer. Returns `[]` when the store is empty.
  */
-export function searchSimilar(db, embedding, topK = 5, { kind = null } = {}) {
+export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null } = {}) {
   const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
   const kindClause = kind ? "AND c.kind = ?" : "";
+  const projectClause = project ? "AND c.project_slug = ?" : "";
   const sql = `
-    SELECT c.id, c.source, c.kind, c.text, c.metadata, v.distance
+    SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, v.distance
     FROM vec_chunks v
     JOIN chunks c ON c.id = v.rowid
-    WHERE v.embedding MATCH ? AND k = ? ${kindClause}
+    WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}
     ORDER BY v.distance
   `;
   const params = [Buffer.from(buf.buffer), topK];
   if (kind) params.push(kind);
+  if (project) params.push(project);
   const rows = db.prepare(sql).all(...params);
   return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
 }
 
+/**
+ * Normalize a projectDir into a stable slug used for project isolation.
+ * Basename → lowercased → non-alphanumeric stripped to dashes.
+ * KJC-TSK-0438.
+ */
+export function projectSlug(projectDir) {
+  if (!projectDir) return null;
+  const base = String(projectDir).replace(/\/+$/, "").split("/").pop();
+  if (!base) return null;
+  return base.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+}
+
 function safeParse(s) { try { return JSON.parse(s); } catch { return s; } }
 
 /**