karajan-code 2.21.0 → 2.22.0

package/README.md

@@ -345,6 +345,29 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
 
 ## Recent releases
 
+> **v2.22.0 released** — Minor. **Project RAG MVP** (KJC-PCS-0049). Karajan now indexes its plans + onboarding briefs (and optionally project sources) into a local vector store and lets you query them semantically from the CLI. Six PRs.
+>
+> | Step | PR | Module |
+> |---|---|---|
+> | 1 | #808 | Vector store on `better-sqlite3` + `sqlite-vec` at `~/.karajan/rag.db` |
+> | 2 | #809 | Ollama embedder adapter (`nomic-embed-text`, dim 768, `localhost:11434`) |
+> | 3 | #810 | Three chunkers: markdown heading hierarchy, plan JSON per-HU, JS/TS export-symbol |
+> | 4 | #811 | Indexer (`indexFile` + `indexProject`), idempotent, embedder failures = warn + continue |
+> | 5 | #812 | Retriever + ranking, kind boost breaks ties (plan +0.05, onboarding +0.03, code 0) |
+> | 6 | #813 | `kj rag` CLI: `index [--with-sources]` + `query <text> [--scope] [--top-k]` |
+>
+> ```bash
+> cd ~/your-project
+> kj onboard                       # Architecture Brief
+> kj plan generate task.md -y      # Plans
+> kj rag index                     # Seed the vec store
+> kj rag query "how did I handle auth in module X?"
+> ```
+>
+> **Coming in v2.23.0**: MCP tool `kj_rag_query` (other agents query the RAG), HU Board search panel, chokidar watcher for live re-indexing, AST-aware source chunker, BM25 + cosine hybrid scoring.
+>
+> The SEA binary stubs out `src/rag/*` + `src/commands/rag.js` (same pattern as the HU Board) — `kj rag` requires `npm install -g karajan-code`.
+>
 > **v2.21.0 released** — Minor. **Brownfield Onboarder role**. Karajan now ships a dedicated path to analyze any existing codebase and produce a Markdown Architecture Brief that the planner can consume as automatic context. Closes KJC-TSK-0384 (3 PRs).
 >
 > KJC-TSK-0384 (PRs #804 + #805 + #806): **`kj onboard`** runs five deterministic collectors over a project root — directory walk (ignoring `node_modules` / `.git` / `dist` / `build`), git log (commits, branches, hot files via `--name-only` over the last 200 commits), 18 well-known config patterns + `package.json` scripts, ADR-style filenames under `docs/adr/`, `docs/adrs/`, `docs/architecture/`, plus a one-shot bundle. Then optionally synthesises a Markdown Architecture Brief via the new OnboarderRole. Output lands at `~/.karajan/onboarding/<slug>.md`. Flags: `--no-synth` (skip the LLM call, dump the raw collectors — useful for CI), `--output <path>` (override default target). Greenfield projects produce `# Project is greenfield` instead of erroring.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "karajan-code",
-  "version": "2.21.0",
+  "version": "2.22.0",
   "description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
   "type": "module",
   "license": "AGPL-3.0",
@@ -80,6 +80,7 @@
     "js-yaml": "^4.1.0",
     "knip": "^6.14.1",
     "madge": "^8.0.0",
+    "sqlite-vec": "^0.1.9",
     "valibot": "^1.3.1"
   },
   "devDependencies": {

package/scripts/esbuild-sea.config.mjs

@@ -139,6 +139,36 @@ const huBoardStubPlugin = {
   },
 };
 
+// Same reasoning as the HU Board stub: src/rag/* + src/commands/rag.js
+// depend on better-sqlite3 + sqlite-vec, native modules that esbuild
+// cannot bundle. The SEA binary stubs them out and tells the user to
+// install via npm to get RAG. KJC-PCS-0049 / Step 6.
+const ragStubPlugin = {
+  name: "rag-stub",
+  setup(build) {
+    build.onResolve({ filter: /[\\/](rag[\\/][^\\/]+|commands[\\/]rag)\.js$/ }, (args) => ({
+      path: args.path, namespace: "rag-stub",
+    }));
+    build.onLoad({ filter: /.*/, namespace: "rag-stub" }, () => ({
+      contents: `
+        const NOT_AVAILABLE = "The RAG subsystem is not available in this standalone binary.\\n" +
+          "Install karajan-code from npm to get \\\`kj rag\\\`:\\n" +
+          "  npm install -g karajan-code\\n";
+        function notAvailable() { throw new Error(NOT_AVAILABLE); }
+        module.exports = {
+          __esModule: false,
+          ragIndexCommand: notAvailable, ragQueryCommand: notAvailable,
+          openVecStore: notAvailable, OllamaEmbedder: notAvailable,
+          indexFile: notAvailable, indexProject: notAvailable,
+          query: notAvailable, chunkMarkdown: notAvailable, chunkPlan: notAvailable, chunkSource: notAvailable,
+          default: notAvailable,
+        };
+      `,
+      loader: "js",
+    }));
+  },
+};
+
 /** @type {import('esbuild').BuildOptions} */
 export const seaBuildOptions = {
   entryPoints: ["src/cli.js"],
@@ -169,6 +199,6 @@ export const seaBuildOptions = {
   // throws → collector returns available:false. npm installs get knip
   // resolved normally from node_modules.
   external: ["better-sqlite3", "madge", "knip", "oxc-parser", "oxc-resolver"],
-  plugins: [seaTransformPlugin, huBoardStubPlugin],
+  plugins: [seaTransformPlugin, huBoardStubPlugin, ragStubPlugin],
   logLevel: "info",
 };

package/src/cli/register-meta.js

@@ -3,6 +3,7 @@ import { triageCommand } from "../commands/triage.js";
 import { researcherCommand } from "../commands/researcher.js";
 import { architectCommand } from "../commands/architect.js";
 import { onboardCommand } from "../commands/onboard.js";
+import { ragIndexCommand, ragQueryCommand } from "../commands/rag.js";
 import { auditCommand } from "../commands/audit.js";
 import { resumeCommand } from "../commands/resume.js";
 import { boardCommand } from "../commands/board.js";
@@ -98,6 +99,27 @@ export function registerMeta(program, { pkgVersion }) {
       });
     });
 
+  const rag = program.command("rag").description("Retrieval-augmented search over Karajan plans, onboarding briefs and project code");
+  rag.command("index")
+    .description("Index plans + onboarding (and optionally project sources) into the local vector store")
+    .option("--with-sources", "Also index the projectDir's JS/TS files")
+    .option("--json", "Output the totals as JSON")
+    .action(async (flags) => {
+      await withConfig(pkgVersion, "rag-index", flags, async ({ config, logger }) => {
+        await ragIndexCommand({ config, logger, flags });
+      });
+    });
+  rag.command("query <text>")
+    .description("Run a semantic query against the indexed RAG corpus")
+    .option("--scope <scope>", "plans | code | onboarding | all (default: all)", "all")
+    .option("--top-k <n>", "Number of hits to return (default: 5)", "5")
+    .option("--json", "Output the hits as JSON")
+    .action(async (text, flags) => {
+      await withConfig(pkgVersion, "rag-query", flags, async ({ config, logger }) => {
+        await ragQueryCommand({ text, config, logger, flags });
+      });
+    });
+
   program
     .command("audit")
     .description("Analyze codebase health (read-only)")

package/src/commands/rag.js

@@ -0,0 +1,66 @@
+// KJC-PCS-0049 Step 6 — `kj rag` command group. Two subcommands:
+//   kj rag index [--project <slug>] [--with-sources]
+//   kj rag query <text>   [--scope plans|code|onboarding|all] [--top-k N] [--json]
+// Closes the v2.22.0 RAG MVP end-to-end from the terminal.
+import { openVecStore, countChunks } from "../rag/vec-store.js";
+import { OllamaEmbedder } from "../rag/embedder.js";
+import { indexProject } from "../rag/indexer.js";
+import { query } from "../rag/retriever.js";
+import { getKarajanHome } from "../utils/paths.js";
+
+function makeEmbedder(config) {
+  const cfg = config?.rag?.embedder || {};
+  return new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim: cfg.dim });
+}
+
+function openDb(config) {
+  return openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
+}
+
+export async function ragIndexCommand({ config, logger, flags = {} }) {
+  const projectDir = config?.projectDir || process.cwd();
+  const db = openDb(config);
+  try {
+    const totals = await indexProject(projectDir, {
+      db, embedder: makeEmbedder(config),
+      karajanHome: getKarajanHome(), logger,
+      withSources: Boolean(flags.withSources),
+    });
+    if (flags.json) {
+      process.stdout.write(`${JSON.stringify(totals)}\n`);
+    } else {
+      logger.info(`[rag] indexed ${totals.indexed} chunk(s) across ${totals.files} file(s) (${totals.failed} failed)`);
+    }
+    return totals;
+  } finally {
+    db.close();
+  }
+}
+
+export async function ragQueryCommand({ text, config, logger, flags = {} }) {
+  if (!text) throw new Error("kj rag query: text argument required");
+  const db = openDb(config);
+  try {
+    if (countChunks(db) === 0) {
+      logger.warn("[rag] No chunks indexed yet. Run 'kj rag index' first.");
+      if (flags.json) process.stdout.write("[]\n");
+      return [];
+    }
+    const topK = Math.max(1, Number(flags.topK) || 5);
+    const scope = flags.scope || "all";
+    const hits = await query(db, makeEmbedder(config), text, { topK, scope });
+    if (flags.json) {
+      process.stdout.write(`${JSON.stringify(hits)}\n`);
+    } else {
+      for (const h of hits) {
+        const label = h.metadata?.hu_id || h.metadata?.symbol || h.metadata?.headingPath?.join(" > ") || "block";
+        logger.info(`[${h.kind} · ${label} · score=${h.score.toFixed(4)}] ${h.source}`);
+        logger.info(h.text.length > 240 ? `${h.text.slice(0, 240)}…` : h.text);
+        logger.info("");
+      }
+    }
+    return hits;
+  } finally {
+    db.close();
+  }
+}

package/src/rag/chunker.js

@@ -0,0 +1,119 @@
+// KJC-PCS-0049 Step 3 — Chunkers for the RAG indexer.
+// Each chunker is pure: takes text + opts, returns
+// `[{ text, metadata: { source, kind, ...extras } }]`. The indexer
+// (Step 4) hands every chunk to the embedder + vec store; the
+// retriever (Step 5) reads metadata back to enrich results.
+
+const DEFAULT_CHUNK_LIMIT = 800;
+const DEFAULT_OVERLAP = 100;
+
+// ---------- shared splitter ---------------------------------------
+
+/**
+ * Split a long string into overlapping windows. Used by every chunker
+ * that produces a single section too big for the embedder's context.
+ */
+function windowText(text, { limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  if (text.length <= limit) return [text];
+  const step = Math.max(1, limit - overlap);
+  const out = [];
+  for (let i = 0; i < text.length; i += step) {
+    out.push(text.slice(i, i + limit));
+    if (i + limit >= text.length) break;
+  }
+  return out;
+}
+
+// ---------- markdown chunker (plans / onboarding briefs) ----------
+
+const HEADING_RE = /^(#{1,6})\s+(.+)$/gm;
+
+/**
+ * Chunk a markdown document by heading hierarchy. The headingPath
+ * metadata tracks ancestors so a retrieval result can show "from
+ * '# Plan' › '## Auth' › '### Login'".
+ */
+export function chunkMarkdown(text, { path = "<inline>", kind = "plan", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  const sections = [];
+  const stack = []; // [{ depth, title }]
+  let cursor = 0;
+  let pending = { headingPath: [], start: 0 };
+  HEADING_RE.lastIndex = 0;
+  let m;
+  while ((m = HEADING_RE.exec(text)) !== null) {
+    if (m.index > cursor) {
+      pending.body = text.slice(pending.start, m.index).trim();
+      if (pending.body) sections.push({ ...pending });
+    }
+    const depth = m[1].length;
+    const title = m[2].trim();
+    while (stack.length > 0 && stack[stack.length - 1].depth >= depth) stack.pop();
+    stack.push({ depth, title });
+    pending = { headingPath: stack.map((s) => s.title), start: HEADING_RE.lastIndex };
+    cursor = HEADING_RE.lastIndex;
+  }
+  const tail = text.slice(pending.start).trim();
+  if (tail) sections.push({ ...pending, body: tail });
+  const chunks = [];
+  for (const sec of sections) {
+    const pieces = windowText(sec.body, { limit, overlap });
+    for (const piece of pieces) {
+      chunks.push({ text: piece, metadata: { source: path, kind, headingPath: sec.headingPath } });
+    }
+  }
+  return chunks;
+}
+
+// ---------- plan chunker (one chunk per HU) -----------------------
+
+/**
+ * Chunk a Karajan plan JSON. One chunk per HU; an extra chunk for the
+ * plan-level summary if present. The retriever uses hu_id to deep-link
+ * back to the board.
+ */
+export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT } = {}) {
+  const out = [];
+  if (plan?.summary) {
+    const pieces = windowText(String(plan.summary), { limit });
+    for (const piece of pieces) {
+      out.push({ text: piece, metadata: { source: path, kind: "plan", section: "summary" } });
+    }
+  }
+  for (const hu of plan?.hus || []) {
+    const parts = [hu.title, hu.description, hu.scope].filter(Boolean).join("\n\n");
+    if (!parts.trim()) continue;
+    for (const piece of windowText(parts, { limit })) {
+      out.push({ text: piece, metadata: { source: path, kind: "plan", hu_id: hu.id || null, title: hu.title || null } });
+    }
+  }
+  return out;
+}
+
+// ---------- source chunker (export-symbol granularity) ------------
+
+const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
+
+/**
+ * Chunk a JS/TS source file by top-level export symbol. Falls back to
+ * the windowed splitter on files without explicit exports.
+ */
+export function chunkSource(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  const marks = [];
+  EXPORT_RE.lastIndex = 0;
+  let m;
+  while ((m = EXPORT_RE.exec(text)) !== null) marks.push({ idx: m.index, symbol: m[1] });
+  if (marks.length === 0) {
+    return windowText(text, { limit, overlap })
+      .map((piece) => ({ text: piece, metadata: { source: path, kind: "code", symbol: null } }));
+  }
+  const out = [];
+  for (let i = 0; i < marks.length; i += 1) {
+    const start = marks[i].idx;
+    const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
+    const body = text.slice(start, end);
+    for (const piece of windowText(body, { limit, overlap })) {
+      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol } });
+    }
+  }
+  return out;
+}

package/src/rag/embedder.js

@@ -0,0 +1,81 @@
+// KJC-PCS-0049 Step 2 — Ollama embedder adapter for the RAG pipeline.
+// Talks to a local Ollama server (POST /api/embeddings) and returns
+// Float32Array vectors. Cero deps externas; usa fetch global.
+//
+// Defaults:
+//   url       = process.env.KJ_OLLAMA_URL or "http://localhost:11434"
+//   model     = process.env.KJ_OLLAMA_EMBED_MODEL or "nomic-embed-text"
+//   dim       = 768  (matches nomic-embed-text)
+//   timeoutMs = 30000
+
+const DEFAULT_URL = "http://localhost:11434";
+const DEFAULT_MODEL = "nomic-embed-text";
+const DEFAULT_DIM = 768;
+const DEFAULT_TIMEOUT_MS = 30000;
+
+export class OllamaEmbedderError extends Error {
+  constructor(message, { cause, status } = {}) {
+    super(message);
+    this.name = "OllamaEmbedderError";
+    if (cause) this.cause = cause;
+    if (status != null) this.status = status;
+  }
+}
+
+export class OllamaEmbedder {
+  constructor({
+    url = process.env.KJ_OLLAMA_URL || DEFAULT_URL,
+    model = process.env.KJ_OLLAMA_EMBED_MODEL || DEFAULT_MODEL,
+    dim = DEFAULT_DIM,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+    fetchFn = globalThis.fetch,
+  } = {}) {
+    this.url = url.replace(/\/$/, "");
+    this.model = model;
+    this.dim = dim;
+    this.timeoutMs = timeoutMs;
+    this.fetch = fetchFn;
+  }
+
+  /** Single-text embedding. Returns Float32Array(dim). */
+  async embed(text) {
+    if (typeof text !== "string" || text.length === 0) {
+      throw new OllamaEmbedderError("embed: text must be a non-empty string");
+    }
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), this.timeoutMs);
+    let res;
+    try {
+      res = await this.fetch(`${this.url}/api/embeddings`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: this.model, prompt: text }),
+        signal: ctrl.signal,
+      });
+    } catch (err) {
+      throw new OllamaEmbedderError(`Ollama embed request failed (${this.url}): ${err.message}`, { cause: err });
+    } finally {
+      clearTimeout(timer);
+    }
+    if (!res.ok) {
+      throw new OllamaEmbedderError(`Ollama embed HTTP ${res.status} from ${this.url}`, { status: res.status });
+    }
+    const body = await res.json();
+    const arr = body?.embedding;
+    if (!Array.isArray(arr) || arr.length === 0) {
+      throw new OllamaEmbedderError("Ollama response missing 'embedding' array");
+    }
+    if (arr.length !== this.dim) {
+      throw new OllamaEmbedderError(`Ollama dim mismatch: got ${arr.length}, expected ${this.dim} for model ${this.model}`);
+    }
+    return Float32Array.from(arr);
+  }
+
+  /** Batch embedding (sequential — Ollama /api/embeddings is single-text). */
+  async embedBatch(texts) {
+    if (!Array.isArray(texts)) throw new OllamaEmbedderError("embedBatch: texts must be an array");
+    const out = [];
+    for (const t of texts) out.push(await this.embed(t));
+    return out;
+  }
+}

package/src/rag/indexer.js

@@ -0,0 +1,116 @@
+// KJC-PCS-0049 Step 4 — Indexer for the RAG pipeline. Wires the three
+// previous modules together: chunker → embedder → vec store. The CLI
+// (Step 6) drives this; Step 5's retriever reads what gets persisted
+// here. A chokidar watcher version stays out of scope for v2.22.0 —
+// indexProject() is on-demand from `kj rag index`, which keeps the
+// failure mode simple (re-run to refresh).
+import { existsSync, readFileSync } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { extname, join, relative } from "node:path";
+
+import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
+import { insertChunk, deleteChunksBySource } from "./vec-store.js";
+
+const ONBOARDING_DIR = "onboarding", PLANS_DIR = "plans";
+
+function detectKind(path) {
+  const ext = extname(path).toLowerCase();
+  const base = path.split("/").pop() || "";
+  if (ext === ".json" && (/^plan-.+\.json$/i.test(base) || path.includes(`/${PLANS_DIR}/`))) return "plan";
+  if (ext === ".md") return path.includes(`/${ONBOARDING_DIR}/`) ? "onboarding" : "plan";
+  if ([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx"].includes(ext)) return "code";
+  return null;
+}
+
+function chunksFor(path, kind) {
+  const text = readFileSync(path, "utf8");
+  if (kind === "plan" && extname(path) === ".json") {
+    try { return chunkPlan(JSON.parse(text), { path }); }
+    catch { return []; }
+  }
+  if (kind === "plan" || kind === "onboarding") {
+    return chunkMarkdown(text, { path, kind });
+  }
+  if (kind === "code") return chunkSource(text, { path });
+  return [];
+}
+
+/**
+ * Index a single file. Returns `{ indexed, failed }`. Idempotent —
+ * pre-existing chunks for this `source` path are deleted before the
+ * fresh batch is written, so a second call against the same file
+ * leaves the store with exactly the latest chunks (no duplicates).
+ */
+export async function indexFile(path, { db, embedder, logger = console } = {}) {
+  if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
+  const kind = detectKind(path);
+  if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
+  const chunks = chunksFor(path, kind);
+  if (!chunks.length) return { indexed: 0, failed: 0 };
+  deleteChunksBySource(db, path);
+  let indexed = 0;
+  let failed = 0;
+  for (const ch of chunks) {
+    try {
+      const embedding = await embedder.embed(ch.text);
+      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding });
+      indexed += 1;
+    } catch (err) {
+      failed += 1;
+      logger.warn?.(`[rag-indexer] embed failed for ${path} (${chunkLabel(ch)}): ${err.message}`);
+    }
+  }
+  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed)`);
+  return { indexed, failed };
+}
+
+function chunkLabel(ch) {
+  return ch.metadata?.hu_id || ch.metadata?.symbol || ch.metadata?.headingPath?.join(" > ") || "block";
+}
+
+async function listFiles(dir, predicate) {
+  const out = [];
+  async function walk(d) {
+    let ents; try { ents = await readdir(d, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      const p = join(d, e.name);
+      if (e.isDirectory()) await walk(p); else if (e.isFile() && predicate(p)) out.push(p);
+    }
+  }
+  await walk(dir); return out;
+}
+
+/**
+ * Index every Karajan-managed asset for the project: every plan-*.json
+ * under `<karajanHome>/plans/<slug>/`, the onboarding brief under
+ * `<karajanHome>/onboarding/`, and (optionally) source files under the
+ * projectDir. Sources are gated behind `withSources: true` because
+ * indexing tens of thousands of files is expensive — the CLI exposes
+ * it as `--with-sources`.
+ */
+export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false } = {}) {
+  const totals = { indexed: 0, failed: 0, files: 0 };
+  const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  const planRoot = join(karajanHome, PLANS_DIR, slug);
+  if (existsSync(planRoot)) {
+    const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
+    for (const p of plans) {
+      const r = await indexFile(p, { db, embedder, logger });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
+  if (existsSync(onboarding)) {
+    const r = await indexFile(onboarding, { db, embedder, logger });
+    totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+  }
+  if (withSources) {
+    const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next"]);
+    const sources = await listFiles(projectDir, (p) => /\.(js|mjs|cjs|ts|tsx|jsx)$/.test(p) && !p.split("/").some((seg) => SKIP.has(seg)));
+    for (const s of sources) {
+      const r = await indexFile(s, { db, embedder, logger });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  return totals;
+}

package/src/rag/retriever.js

@@ -0,0 +1,47 @@
+// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline. Composes the
+// embedder (Step 2) + vec store (Step 1) into a one-call query API.
+// The CLI (Step 6) and the future MCP tool are the only consumers.
+
+import { searchSimilar } from "./vec-store.js";
+
+// Kind boosts for rerank: a plan chunk is more "intentional" than a
+// raw code chunk, an onboarding brief sits between them. Tunable via
+// the third argument; the defaults are conservative — they break ties
+// between equidistant chunks but won't reorder by big distance gaps.
+const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
+
+/**
+ * Run a natural-language query against the indexed RAG corpus. Returns
+ * the top-K chunks ranked by adjusted distance (lower = closer).
+ *
+ *   query(db, embedder, "how did I handle auth in module X?",
+ *         { topK: 5, scope: 'plans' })
+ *
+ * @param {Database} db           — opened via openVecStore()
+ * @param {OllamaEmbedder} embedder
+ * @param {string} text           — the query
+ * @param {Object} [opts]
+ * @param {number} [opts.topK=5]
+ * @param {'plans'|'code'|'onboarding'|'all'} [opts.scope='all']
+ * @param {Object} [opts.kindBoost]
+ */
+export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST } = {}) {
+  if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
+  const queryEmbedding = await embedder.embed(text);
+  // Over-fetch (topK * 2) so the rerank step has room to reshuffle
+  // without dropping a relevant chunk past the cut. Clamped at 50 to
+  // avoid loading the entire store on tiny topK calls.
+  const fetchK = Math.min(50, topK * 2);
+  const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
+  const raw = searchSimilar(db, queryEmbedding, fetchK, { kind: scopeKind });
+  if (raw.length === 0) return [];
+  // Adjust each chunk's distance by its kind boost. The vec store returns
+  // cosine distance (smaller = closer); subtracting the boost makes
+  // higher-priority kinds appear "closer" without altering the source
+  // truth in the DB.
+  const reranked = raw.map((r) => ({
+    ...r,
+    score: r.distance - (kindBoost[r.kind] || 0),
+  })).sort((a, b) => a.score - b.score).slice(0, topK);
+  return reranked;
+}

package/src/rag/vec-store.js

@@ -0,0 +1,113 @@
+// KJC-PCS-0049 Step 1 — Vector store sobre better-sqlite3 + sqlite-vec.
+// El RAG indexer (Step 4) escribe; el retriever (Step 5) lee. Una sola
+// DB en `~/.karajan/rag.db` por instalación.
+import { mkdirSync, existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+
+import { getKarajanHome } from "../utils/paths.js";
+
+const DEFAULT_DIM = 768;
+
+export function dbPath() {
+  return process.env.KJ_RAG_DB || join(getKarajanHome(), "rag.db");
+}
+
+/**
+ * Open the rag.db, load the sqlite-vec extension, and ensure the
+ * chunks + vec_chunks tables exist. Idempotent: a second call against
+ * the same file is a no-op past the schema creation.
+ *
+ * Returns the open Database handle. Caller owns close().
+ *
+ * @param {Object} [opts]
+ * @param {number} [opts.dim=768] — embedding dimension. Must match
+ *   the embedder configured for the project (Step 2). Changing it
+ *   later requires a fresh DB; the constant is captured in the
+ *   virtual table DDL.
+ */
+export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
+  if (!existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true });
+  const db = new Database(path);
+  sqliteVec.load(db);
+  db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      source TEXT NOT NULL,
+      kind TEXT NOT NULL CHECK (kind IN ('plan', 'code', 'onboarding')),
+      text TEXT NOT NULL,
+      metadata TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE INDEX IF NOT EXISTS chunks_by_source ON chunks(source);
+    CREATE INDEX IF NOT EXISTS chunks_by_kind ON chunks(kind);
+  `);
+  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
+  return db;
+}
+
+/**
+ * Persist one chunk. The chunks.id and vec_chunks.rowid stay in sync
+ * because the virtual table rowid is set explicitly to the freshly
+ * minted chunks.id. Returns that id.
+ */
+export function insertChunk(db, { source, kind, text, metadata = null, embedding }) {
+  if (!Array.isArray(embedding) && !ArrayBuffer.isView(embedding)) {
+    throw new Error("insertChunk: embedding must be an array or typed array of floats");
+  }
+  const meta = metadata == null ? null : (typeof metadata === "string" ? metadata : JSON.stringify(metadata));
+  const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata) VALUES (?, ?, ?, ?)");
+  const info = ins.run(source, kind, text, meta);
+  // sqlite-vec's vec0 virtual table requires the rowid to arrive as a BigInt
+  // even for values that comfortably fit in a Number. Better-sqlite3 returns
+  // lastInsertRowid as BigInt only when safeIntegers mode is on; force it.
+  const id = typeof info.lastInsertRowid === "bigint" ? info.lastInsertRowid : BigInt(info.lastInsertRowid);
+  const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
+  db.prepare("INSERT INTO vec_chunks(rowid, embedding) VALUES (?, ?)").run(id, Buffer.from(buf.buffer));
+  return Number(id);
+}
+
+/**
+ * Cosine-ordered nearest-neighbour search. Returns `[{ id, source, kind,
+ * text, metadata, distance }]` for the topK best matches; lower
+ * distance = closer. Returns `[]` when the store is empty.
+ */
+export function searchSimilar(db, embedding, topK = 5, { kind = null } = {}) {
+  const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
+  const kindClause = kind ? "AND c.kind = ?" : "";
+  const sql = `
+    SELECT c.id, c.source, c.kind, c.text, c.metadata, v.distance
+    FROM vec_chunks v
+    JOIN chunks c ON c.id = v.rowid
+    WHERE v.embedding MATCH ? AND k = ? ${kindClause}
+    ORDER BY v.distance
+  `;
+  const params = [Buffer.from(buf.buffer), topK];
+  if (kind) params.push(kind);
+  const rows = db.prepare(sql).all(...params);
+  return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
+}
+
+function safeParse(s) { try { return JSON.parse(s); } catch { return s; } }
+
+/**
+ * Remove every chunk that came from a given source path. Both tables
+ * are touched explicitly because vec0 virtual tables do not honour
+ * SQLite foreign-key cascades.
+ */
+export function deleteChunksBySource(db, source) {
+  const ids = db.prepare("SELECT id FROM chunks WHERE source = ?").all(source).map((r) => r.id);
+  if (ids.length === 0) return 0;
+  const placeholders = ids.map(() => "?").join(",");
+  db.prepare(`DELETE FROM vec_chunks WHERE rowid IN (${placeholders})`).run(...ids);
+  db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...ids);
+  return ids.length;
+}
+
+export function countChunks(db, { kind = null } = {}) {
+  if (kind) return db.prepare("SELECT COUNT(*) AS n FROM chunks WHERE kind = ?").get(kind).n;
+  return db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
+}