karajan-code 4.3.1 → 4.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "karajan-code",
-  "version": "4.3.1",
+  "version": "4.4.1",
   "description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
   "type": "module",
   "license": "AGPL-3.0",
@@ -109,11 +109,13 @@
     "helmet": "^8.1.0",
     "js-yaml": "^4.2.0",
     "karajan-core": "^1.0.0",
+    "karajan-rag": "^1.2.0",
     "knip": "^6.15.0",
     "madge": "^8.0.0",
     "sqlite-vec": "^0.1.9",
     "valibot": "^1.4.1",
-    "web-tree-sitter": "^0.26.9"
+    "web-tree-sitter": "^0.26.9",
+    "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",

package/src/commands/env.js

@@ -6,11 +6,17 @@
  */
 import { installPlaybook } from "../environment/playbook.js";
 import { renderBrief, listBriefs } from "../environment/briefs.js";
-import { openVecStore, projectSlug, getLastIndexedCommit } from "../rag/vec-store.js";
+import { existsSync } from "node:fs";
+import { openVecStore, projectSlug, getLastIndexedCommit, dbPath } from "../rag/vec-store.js";
 import { ragIndexCommand } from "./rag.js";
 import { renderPendingBlock, PENDING_EXIT_CODE } from "../utils/pending-user-action.js";
+import { onnxConfig, persistOnnxChoice, resetEmptyStore } from "../rag/onnx-fallback.js";
 
 function hasRagIndex(config, projectDir) {
+  // KJC-BUG-0128: probing must never CREATE the store — openVecStore runs
+  // the DDL, and a table born at the default dim (768) breaks the ONNX
+  // fallback (384) later. No file → no index, without side effects.
+  if (!existsSync(dbPath())) return false;
   const db = openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
   try { return Boolean(getLastIndexedCommit(db, projectSlug(projectDir))); }
   finally { db.close(); }
@@ -59,6 +65,35 @@ export async function envInstallCommand({ config = null, logger = null, flags =
         : { tool: `${provider} embedder`, action: "needs-user", reason: `the RAG cannot index: ${why} — check rag.embedder in kj config`, manualUrl: "kj config" };
       console.log(renderPendingBlock([item], { retry: "kj rag index --with-sources   (then re-run: kj env install)" }));
     };
+    // KJC-TSK-0683: on limited machines (no Ollama, no Docker) the DEFAULT
+    // provider falls back to the built-in ONNX embedder instead of blocking.
+    // Only for the first index and only when the user didn't pick a provider
+    // explicitly — the election is persisted (dims differ, it must stick).
+    const explicitProvider = Boolean(config?.rag?.embedder?.provider);
+    const tryOnnxFallback = async (why) => {
+      if (explicitProvider) return false;
+      console.log(`⚠ default embedder unavailable (${why}) — trying the built-in ONNX embedder (no install needed)…`);
+      try {
+        // KJC-BUG-0128: an empty store may carry a vec table at the wrong
+        // dim (the old probe created it at 768) — reset it so ONNX (384)
+        // can index. A store with data is never touched.
+        if (!resetEmptyStore()) {
+          console.log("  the vector store already has data at another dimension — not switching automatically");
+          return false;
+        }
+        const totals = await ragIndexCommand({ config: onnxConfig(config), logger, flags: { withSources: true } });
+        if ((totals?.indexed ?? 0) > 0) {
+          const p = await persistOnnxChoice(projectDir);
+          console.log(`✓ RAG indexed with the built-in ONNX embedder — persisted in ${p}`);
+          console.log("  To move to Ollama later: install it, remove the rag.embedder block from .karajan/kj.config.yml,");
+          console.log("  delete the store (~/.karajan/rag.db) and run: kj rag index --with-sources");
+          return true;
+        }
+      } catch (fallbackErr) {
+        console.log(`  ONNX fallback also failed: ${fallbackErr.message}`);
+      }
+      return false;
+    };
     try {
       if (hasRagIndex(config, projectDir)) {
         console.log("✓ RAG index present");
@@ -66,11 +101,12 @@ export async function envInstallCommand({ config = null, logger = null, flags =
         console.log("⏳ no RAG index for this project — building it (first time only)…");
         const totals = await ragIndexCommand({ config, logger, flags: { withSources: true } });
         if ((totals?.indexed ?? 0) === 0 && (totals?.files ?? 0) > 0) {
-          blockRag(`0 of ${totals.files} files indexed — is the embedder running?`);
+          const why = `0 of ${totals.files} files indexed — is the embedder running?`;
+          if (!(await tryOnnxFallback(why))) blockRag(why);
         }
       }
     } catch (err) {
-      blockRag(err.message);
+      if (!(await tryOnnxFallback(err.message))) blockRag(err.message);
     }
   }
   if (result.exitCode !== PENDING_EXIT_CODE) {

package/src/commands/rag.js

@@ -3,7 +3,7 @@
 //   kj rag query <text>   [--scope plans|code|onboarding|all] [--top-k N] [--json]
 // Closes the v2.22.0 RAG MVP end-to-end from the terminal.
 import { openVecStore, countChunks, projectSlug, getLastIndexedCommit, setLastIndexedCommit } from "../rag/vec-store.js";
-import { makeEmbedder } from "../rag/embedders/factory.js";
+import { makeGovernedEmbedder } from "../rag/governed-embedder.js";
 import { indexProject, indexProjectDelta } from "../rag/indexer.js";
 import { query } from "../rag/retriever.js";
 import { installPostMergeHook, maybeAutoUpdate } from "../rag/auto-update.js";
@@ -19,7 +19,7 @@ export async function ragIndexCommand({ config, logger, flags = {} }) {
   const db = openDb(config);
   try {
     const slug = projectSlug(projectDir);
-    const embedder = makeEmbedder(config);
+    const embedder = makeGovernedEmbedder(config);
     const sinceFlag = flags.since;
     // KJC-TSK-0455 — `--since auto` resolves to the last commit we indexed;
     // an explicit ref is honoured as-is. Without a baseline (first-time
@@ -103,7 +103,7 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
     const alpha = Math.max(0, Math.min(1, Number(flags.alpha) || 0.6));
     const where = flags.where || null;
     const rerankOpts = flags.rerank ? { model: flags.rerankModel } : null;
-    const hits = await query(db, makeEmbedder(config), text, { topK, scope, project, mode, alpha, where, rerankOpts });
+    const hits = await query(db, makeGovernedEmbedder(config), text, { topK, scope, project, mode, alpha, where, rerankOpts });
     if (flags.json) {
       process.stdout.write(`${JSON.stringify(hits)}\n`);
     } else {
@@ -133,7 +133,7 @@ export async function ragEvalCommand({ config, logger, flags = {} }) {
     const topK = Math.max(1, Number(flags.topK) || 10);
     const detected = projectSlug(config?.projectDir || process.cwd());
     const project = flags.project === "all" ? null : (flags.project || detected || null);
-    const embedder = makeEmbedder(config);
+    const embedder = makeGovernedEmbedder(config);
     const runQuery = async (text, k) => query(db, embedder, text, { topK: k, scope: "all", project });
     const report = await runEval(queries, runQuery, { topK, ks: [5, 10] });
     if (flags.json) {

package/src/config/defaults.js

@@ -216,7 +216,11 @@ const DEFAULTS = {
     // delta re-index before every `kj run`; `onCommit` is consulted by
     // `kj rag install-hooks` to decide whether to install the post-merge
     // hook. Both default ON; flip to false to opt out without uninstalling.
-    autoUpdate: { onCommit: true, onRun: true }
+    autoUpdate: { onCommit: true, onRun: true },
+    // KJC-TSK-0682 — sensitivity of the indexed code. Cloud embedders
+    // (openai/voyage/cohere/mistral) require an explicit "public"; the
+    // safe default blocks them (local ollama/onnx always allowed).
+    sensitivity: "internal"
   },
   // Opt-in. Set by the `kj init` wizard (explicit consent prompt in the OS
   // locale). When the key is absent or false, no events are sent. See

package/src/mcp/handlers/rag-handler.js

@@ -2,7 +2,7 @@
 // from src/commands/* are forbidden by the layer-boundaries test
 // (MCP and CLI are peer layers); we hit the pure rag/* modules instead.
 import { countChunks, openVecStore } from "../../rag/vec-store.js";
-import { makeEmbedder } from "../../rag/embedders/factory.js";
+import { makeGovernedEmbedder } from "../../rag/governed-embedder.js";
 import { indexProject } from "../../rag/indexer.js";
 import { query } from "../../rag/retriever.js";
 import { getKarajanHome } from "../../utils/paths.js";
@@ -25,7 +25,7 @@ export async function handleRagQuery(args, server) {
     const db = openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
     try {
       if (countChunks(db) === 0) return responseText({ hits: [], empty: true, topK, scope });
-      const hits = await query(db, makeEmbedder(config), text, { topK, scope });
+      const hits = await query(db, makeGovernedEmbedder(config), text, { topK, scope });
       return responseText({ hits, empty: false, topK, scope });
     } finally { db.close(); }
   } catch (err) {
@@ -40,7 +40,7 @@ export async function handleRagIndex(args, server) {
     const db = openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
     try {
       const totals = await indexProject(projectDir, {
-        db, embedder: makeEmbedder(config),
+        db, embedder: makeGovernedEmbedder(config),
         karajanHome: getKarajanHome(), logger: silentLogger,
         withSources: Boolean(args?.withSources),
       });

package/src/orchestrator/stages/rag-context-stage.js

@@ -6,7 +6,7 @@
 // never sees an exception from this stage.
 import { emitProgress, makeEvent } from "../../utils/events.js";
 import { openVecStore, countChunks } from "../../rag/vec-store.js";
-import { makeEmbedder } from "../../rag/embedders/factory.js";
+import { makeGovernedEmbedder } from "../../rag/governed-embedder.js";
 import { query } from "../../rag/retriever.js";
 
 const DEFAULT_TOP_K = 5;
@@ -52,7 +52,7 @@ export async function runRagContextStage({ config, logger, emitter, eventBase, t
       }
       const topK = preload.topK || DEFAULT_TOP_K;
       const scope = preload.scope || DEFAULT_SCOPE;
-      const embedder = makeEmbedder(config);
+      const embedder = makeGovernedEmbedder(config);
       const searchOpts = resolveSearchOpts(config);
       const hits = await query(db, embedder, task, { topK, scope, ...searchOpts });
       if (hits.length === 0) return { skipped: true, reason: "no-hits" };

package/src/rag/auto-update.js

@@ -8,7 +8,7 @@ import { execa } from "execa";
 
 import { openVecStore, projectSlug, getLastIndexedCommit, setLastIndexedCommit } from "./vec-store.js";
 import { indexProjectDelta } from "./indexer.js";
-import { makeEmbedder } from "./embedders/factory.js";
+import { makeGovernedEmbedder } from "./governed-embedder.js";
 
 const HOOK_SRC = resolve(fileURLToPath(import.meta.url), "../../../scripts/git-hooks/post-merge");
 
@@ -25,7 +25,7 @@ export async function maybeAutoUpdate({ projectDir, config, logger = console, fl
     const since = getLastIndexedCommit(db, slug);
     if (!since || since === head) return { skipped: true, head };
     logger.info?.(`[rag] drift detected (${since.slice(0, 7)} → ${head.slice(0, 7)}); running delta update`);
-    const totals = await indexProjectDelta(projectDir, { db, embedder: makeEmbedder(config), since, logger });
+    const totals = await indexProjectDelta(projectDir, { db, embedder: makeGovernedEmbedder(config), since, logger });
     if (totals.head) setLastIndexedCommit(db, slug, totals.head);
     return { ran: true, totals };
   } catch (err) {

package/src/rag/governed-embedder.js

@@ -0,0 +1,39 @@
+/**
+ * Governed embedder (KJC-TSK-0682 — card filed by the karajan-rag session
+ * after its independent security audit closed the same risk there, H2).
+ * Chunks of project code must not travel to a cloud embedder without
+ * governance: local providers (ollama/onnx) pass through untouched; cloud
+ * providers (openai/voyage/cohere/mistral) require an EXPLICIT
+ * `rag.sensitivity: public` — the default (`internal`) blocks with an
+ * actionable error, never a silent downgrade. Even when allowed, every
+ * text is PII-redacted (karajan-rag's audited redactPII) before leaving
+ * the machine — defense in depth, not a substitute for the policy.
+ */
+import { redactPII } from "karajan-rag";
+import { makeEmbedder } from "./embedders/factory.js";
+
+const LOCAL_PROVIDERS = new Set(["ollama", "onnx"]);
+const redact = (text) => redactPII(String(text)).text;
+
+class RedactingEmbedder {
+  constructor(inner) { this.inner = inner; }
+  get dim() { return this.inner.dim; }
+  get model() { return this.inner.model; }
+  async embed(text) { return this.inner.embed(redact(text)); }
+  async embedBatch(texts) { return this.inner.embedBatch(texts.map(redact)); }
+}
+
+/** Drop-in replacement for makeEmbedder(config) at every kj call site. */
+export function makeGovernedEmbedder(config) {
+  const provider = config?.rag?.embedder?.provider || "ollama";
+  if (LOCAL_PROVIDERS.has(provider)) return makeEmbedder(config);
+
+  const sensitivity = config?.rag?.sensitivity || "internal";
+  if (sensitivity !== "public") {
+    throw new Error(
+      `embedder "${provider}" sends your code to a cloud provider, but rag.sensitivity is "${sensitivity}". `
+      + `Use a local embedder (rag.embedder.provider: ollama | onnx) or declare rag.sensitivity: public in kj.config.yml.`
+    );
+  }
+  return new RedactingEmbedder(makeEmbedder(config));
+}

package/src/rag/onnx-fallback.js

@@ -0,0 +1,69 @@
+/**
+ * ONNX embedder fallback (KJC-TSK-0683) — limited machines where installing
+ * Ollama/Docker is not viable still get a working RAG: the built-in ONNX
+ * embedder (all-MiniLM via @huggingface/transformers, in-process, CPU).
+ * The choice is PERSISTED in the project config: onnx (384) and ollama
+ * (768) have different vector dims, so the election must be sticky —
+ * an index can never be written by one and queried by the other.
+ */
+import fs from "node:fs/promises";
+import { existsSync, rmSync } from "node:fs";
+import path from "node:path";
+import Database from "better-sqlite3";
+import { parseDocument } from "yaml";
+import { getProjectConfigPath } from "../config/loader.js";
+import { dbPath } from "./vec-store.js";
+
+const ONNX_EMBEDDER = { provider: "onnx", dim: 384 };
+
+/**
+ * KJC-BUG-0128: the hasRagIndex probe could leave behind an EMPTY store
+ * whose vec table was created at the default dim (768) — ONNX inserts (384)
+ * then failed on every chunk. Before the fallback indexes, delete an empty
+ * store (plus WAL siblings) so it is recreated at the right dim. A store
+ * that already HAS chunks is never touched — switching dims there would
+ * destroy other projects' indexes.
+ */
+export function resetEmptyStore() {
+  const storePath = dbPath();
+  if (!existsSync(storePath)) return true;
+  // Read-only, schema-agnostic inspection: no DDL, no vec extension, no
+  // dimension assumptions — the store is only ever OPENED to be counted.
+  const db = new Database(storePath, { readonly: true });
+  try {
+    const hasChunksTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chunks'").get();
+    if (hasChunksTable && db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n > 0) return false;
+  } finally {
+    db.close();
+  }
+  for (const suffix of ["", "-wal", "-shm"]) rmSync(storePath + suffix, { force: true });
+  return true;
+}
+
+/** The same config, with the embedder swapped to the built-in ONNX. */
+export function onnxConfig(config) {
+  return {
+    ...config,
+    rag: { ...(config?.rag || {}), embedder: { ...(config?.rag?.embedder || {}), ...ONNX_EMBEDDER } },
+  };
+}
+
+/**
+ * Persist the ONNX election in the PROJECT config file. Surgical edit via
+ * the yaml Document API: only rag.embedder.{provider,dim} change — the
+ * user's comments, ordering and every other key survive untouched.
+ */
+export async function persistOnnxChoice(projectDir) {
+  const configPath = getProjectConfigPath(projectDir);
+  let source = "";
+  try {
+    source = await fs.readFile(configPath, "utf8");
+  } catch { /* no project config yet — create it */ }
+  const doc = parseDocument(source || "{}");
+  for (const [key, value] of Object.entries(ONNX_EMBEDDER)) {
+    doc.setIn(["rag", "embedder", key], value);
+  }
+  await fs.mkdir(path.dirname(configPath), { recursive: true });
+  await fs.writeFile(configPath, doc.toString(), "utf8");
+  return configPath;
+}

package/src/rag/watcher.js

@@ -5,7 +5,7 @@ import { writeFileSync, readFileSync, existsSync, unlinkSync } from "node:fs";
 import { extname, join } from "node:path";
 import chokidar from "chokidar";
 import { openVecStore, deleteChunksBySource, projectSlug } from "./vec-store.js";
-import { makeEmbedder } from "./embedders/factory.js";
+import { makeGovernedEmbedder } from "./governed-embedder.js";
 import { indexFile } from "./indexer.js";
 import { getKarajanHome } from "../utils/paths.js";
 import { getAllCodeExtensions } from "../lang/registry.js";
@@ -27,7 +27,7 @@ export function startWatcher({ projectDir, config, logger = console, debounceMs
   const slug = projectSlug(projectDir);
   const dim = config?.rag?.embedder?.dim || 768;
   const db = openVecStore({ dim });
-  const embedder = makeEmbedder(config);
+  const embedder = makeGovernedEmbedder(config);
   const paths = [join(getKarajanHome(), "onboarding"), join(getKarajanHome(), "plans")];
   if (withSources) paths.push(projectDir);
   const watcher = chokidar.watch(paths, { ignoreInitial: true, persistent: true });