@karajan-family/code 4.21.0

package/src/rag/chunker.js

@@ -0,0 +1,132 @@
+// KJC-PCS-0049 Step 3 — Chunkers for the RAG indexer.
+// Each chunker is pure: takes text + opts, returns
+// `[{ text, metadata: { source, kind, ...extras } }]`. The indexer
+// (Step 4) hands every chunk to the embedder + vec store; the
+// retriever (Step 5) reads metadata back to enrich results.
+
+const DEFAULT_CHUNK_LIMIT = 800;
+const DEFAULT_OVERLAP = 100;
+
+// ---------- shared splitter ---------------------------------------
+
+/**
+ * Split a long string into overlapping windows. Used by every chunker
+ * that produces a single section too big for the embedder's context.
+ */
+function windowText(text, { limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  if (text.length <= limit) return [text];
+  const step = Math.max(1, limit - overlap);
+  const out = [];
+  for (let i = 0; i < text.length; i += step) {
+    out.push(text.slice(i, i + limit));
+    if (i + limit >= text.length) break;
+  }
+  return out;
+}
+
+// ---------- markdown chunker (plans / onboarding briefs) ----------
+
+const HEADING_RE = /^(#{1,6})\s+(.+)$/gm;
+
+/**
+ * Chunk a markdown document by heading hierarchy. The headingPath
+ * metadata tracks ancestors so a retrieval result can show "from
+ * '# Plan' › '## Auth' › '### Login'".
+ */
+export function chunkMarkdown(text, { path = "<inline>", kind = "plan", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  const sections = [];
+  const stack = []; // [{ depth, title }]
+  let cursor = 0;
+  let pending = { headingPath: [], start: 0 };
+  HEADING_RE.lastIndex = 0;
+  let m;
+  while ((m = HEADING_RE.exec(text)) !== null) {
+    if (m.index > cursor) {
+      pending.body = text.slice(pending.start, m.index).trim();
+      if (pending.body) sections.push({ ...pending });
+    }
+    const depth = m[1].length;
+    const title = m[2].trim();
+    while (stack.length > 0 && stack[stack.length - 1].depth >= depth) stack.pop();
+    stack.push({ depth, title });
+    pending = { headingPath: stack.map((s) => s.title), start: HEADING_RE.lastIndex };
+    cursor = HEADING_RE.lastIndex;
+  }
+  const tail = text.slice(pending.start).trim();
+  if (tail) sections.push({ ...pending, body: tail });
+  const chunks = [];
+  for (const sec of sections) {
+    const pieces = windowText(sec.body, { limit, overlap });
+    for (const piece of pieces) {
+      chunks.push({ text: piece, metadata: { source: path, kind, headingPath: sec.headingPath } });
+    }
+  }
+  return chunks;
+}
+
+// ---------- plan chunker (one chunk per HU) -----------------------
+
+/**
+ * Chunk a Karajan plan JSON. One chunk per HU; an extra chunk for the
+ * plan-level summary if present. The retriever uses hu_id to deep-link
+ * back to the board.
+ */
+export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT } = {}) {
+  const out = [];
+  if (plan?.summary) {
+    const pieces = windowText(String(plan.summary), { limit });
+    for (const piece of pieces) {
+      out.push({ text: piece, metadata: { source: path, kind: "plan", section: "summary" } });
+    }
+  }
+  for (const hu of plan?.hus || []) {
+    const parts = [hu.title, hu.description, hu.scope].filter(Boolean).join("\n\n");
+    if (!parts.trim()) continue;
+    for (const piece of windowText(parts, { limit })) {
+      out.push({ text: piece, metadata: { source: path, kind: "plan", hu_id: hu.id || null, title: hu.title || null } });
+    }
+  }
+  return out;
+}
+
+// ---------- source chunker (export-symbol granularity) ------------
+
+import { chunkSourceAST } from "./chunker-ast.js";
+import { adapterForPath } from "../lang/registry.js";
+
+const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
+
+// KJC-TSK-0444 + KJC-PCS-0052 PR-B — dispatch por adapter cuando el path
+// coincide con un lenguaje registrado que aporta su propio `chunkSource`
+// (p.ej. Python regex top-level). Si no, mantiene el camino JS histórico:
+// AST de @babel/parser primero, fallback a export-regex.
+export function chunkSource(text, opts = {}) {
+  const ad = adapterForPath(opts.path);
+  if (ad?.chunkSource) return ad.chunkSource(text, opts);
+  const ast = chunkSourceAST(text, opts);
+  if (ast) return ast;
+  return chunkSourceRegex(text, opts);
+}
+
+function chunkSourceRegex(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
+  const marks = [];
+  EXPORT_RE.lastIndex = 0;
+  let m;
+  while ((m = EXPORT_RE.exec(text)) !== null) marks.push({ idx: m.index, symbol: m[1] });
+  if (marks.length === 0) {
+    return windowText(text, { limit, overlap })
+      .map((piece) => ({ text: piece, metadata: { source: path, kind: "code", symbol: null } }));
+  }
+  const out = [];
+  for (let i = 0; i < marks.length; i += 1) {
+    const start = marks[i].idx;
+    const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
+    const body = text.slice(start, end);
+    for (const piece of windowText(body, { limit, overlap })) {
+      out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol } });
+    }
+  }
+  return out;
+}
+
+export { chunkSourceRegex };

package/src/rag/embedder.js

@@ -0,0 +1,2 @@
+// Shim: embedder now lives in karajan-core/rag (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedder";

package/src/rag/embedders/_cloud-base.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/_cloud-base";

package/src/rag/embedders/cohere.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/cohere";

package/src/rag/embedders/factory.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/factory";

package/src/rag/embedders/mistral.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/mistral";

package/src/rag/embedders/onnx.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/onnx";

package/src/rag/embedders/openai.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/openai";

package/src/rag/embedders/voyage.js

@@ -0,0 +1,2 @@
+// Shim: moved to karajan-core/rag/embedders (KJC-TSK-0632 PR3).
+export * from "karajan-core/rag/embedders/voyage";

package/src/rag/eval.js

@@ -0,0 +1,61 @@
+// KJC-TSK-0483 — Retrieval-quality harness. Loads a golden set of queries
+// (each with expected source paths and/or symbol names), runs the retriever
+// against each, and reports recall@k + MRR per query and aggregated. Pure:
+// the runner takes `runQuery` as a function so the CLI plugs in `query()`
+// while tests can use a stub.
+import { readFileSync } from "node:fs";
+
+const DEFAULT_KS = [5, 10];
+
+export function loadGoldenQueries(path) {
+  const data = JSON.parse(readFileSync(path, "utf8"));
+  if (!Array.isArray(data)) throw new Error(`golden queries: expected an array in ${path}`);
+  return data;
+}
+
+function hitMatchesExpected(hit, expected) {
+  if (!hit) return false;
+  const src = hit.source || "";
+  if (expected.sources?.some((s) => src.endsWith(s))) return true;
+  const sym = hit.metadata?.symbol;
+  if (sym && expected.symbols?.includes(sym)) return true;
+  return false;
+}
+
+// Per-query metrics. `recall@k` is binary — 1 if any expected match lands in
+// the top-k, else 0. `rr` is the reciprocal of the first relevant rank (1 if
+// the very first hit matches, 0 if none do). `firstRank` is null on miss so
+// callers can render "—" without re-checking.
+export function scoreQuery(hits, expected, ks = DEFAULT_KS) {
+  const recall = {};
+  for (const k of ks) {
+    recall[`@${k}`] = hits.slice(0, k).some((h) => hitMatchesExpected(h, expected)) ? 1 : 0;
+  }
+  let firstRank = null;
+  let rr = 0;
+  for (let i = 0; i < hits.length; i += 1) {
+    if (hitMatchesExpected(hits[i], expected)) { firstRank = i + 1; rr = 1 / firstRank; break; }
+  }
+  return { recall, rr, firstRank };
+}
+
+export function aggregate(results, ks = DEFAULT_KS) {
+  const n = results.length || 1;
+  const recall = {};
+  for (const k of ks) {
+    recall[`@${k}`] = results.reduce((acc, r) => acc + (r.recall[`@${k}`] || 0), 0) / n;
+  }
+  const mrr = results.reduce((acc, r) => acc + (r.rr || 0), 0) / n;
+  return { n: results.length, recall, mrr };
+}
+
+export async function runEval(queries, runQuery, { topK = 10, ks = DEFAULT_KS } = {}) {
+  const results = [];
+  for (const q of queries) {
+    const hits = (await runQuery(q.query, topK)) || [];
+    const expected = { sources: q.expected_sources || [], symbols: q.expected_symbols || [] };
+    const scored = scoreQuery(hits, expected, ks);
+    results.push({ query: q.query, hits, ...scored });
+  }
+  return { results, aggregate: aggregate(results, ks) };
+}

package/src/rag/governed-embedder.js

@@ -0,0 +1,39 @@
+/**
+ * Governed embedder (KJC-TSK-0682 — card filed by the karajan-rag session
+ * after its independent security audit closed the same risk there, H2).
+ * Chunks of project code must not travel to a cloud embedder without
+ * governance: local providers (ollama/onnx) pass through untouched; cloud
+ * providers (openai/voyage/cohere/mistral) require an EXPLICIT
+ * `rag.sensitivity: public` — the default (`internal`) blocks with an
+ * actionable error, never a silent downgrade. Even when allowed, every
+ * text is PII-redacted (karajan-rag's audited redactPII) before leaving
+ * the machine — defense in depth, not a substitute for the policy.
+ */
+import { redactPII } from "karajan-rag";
+import { makeEmbedder } from "./embedders/factory.js";
+
+const LOCAL_PROVIDERS = new Set(["ollama", "onnx"]);
+const redact = (text) => redactPII(String(text)).text;
+
+class RedactingEmbedder {
+  constructor(inner) { this.inner = inner; }
+  get dim() { return this.inner.dim; }
+  get model() { return this.inner.model; }
+  async embed(text) { return this.inner.embed(redact(text)); }
+  async embedBatch(texts) { return this.inner.embedBatch(texts.map(redact)); }
+}
+
+/** Drop-in replacement for makeEmbedder(config) at every kj call site. */
+export function makeGovernedEmbedder(config) {
+  const provider = config?.rag?.embedder?.provider || "ollama";
+  if (LOCAL_PROVIDERS.has(provider)) return makeEmbedder(config);
+
+  const sensitivity = config?.rag?.sensitivity || "internal";
+  if (sensitivity !== "public") {
+    throw new Error(
+      `embedder "${provider}" sends your code to a cloud provider, but rag.sensitivity is "${sensitivity}". `
+      + `Use a local embedder (rag.embedder.provider: ollama | onnx) or declare rag.sensitivity: public in kj.config.yml.`
+    );
+  }
+  return new RedactingEmbedder(makeEmbedder(config));
+}

package/src/rag/indexer.js

@@ -0,0 +1,199 @@
+// KJC-PCS-0049 Step 4 — Indexer for the RAG pipeline. Wires the three
+// previous modules together: chunker → embedder → vec store. The CLI
+// (Step 6) drives this; Step 5's retriever reads what gets persisted
+// here. A chokidar watcher version stays out of scope for v2.22.0 —
+// indexProject() is on-demand from `kj rag index`, which keeps the
+// failure mode simple (re-run to refresh).
+import { existsSync, readFileSync } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { extname, isAbsolute, join, relative } from "node:path";
+import { createHash } from "node:crypto";
+import { execa } from "execa";
+
+import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
+import { insertChunk, deleteChunksBySource, findChunkByHash } from "./vec-store.js";
+import { detectAdaptersForProject, buildMatchers, getAllCodeExtensions } from "../lang/registry.js";
+
+// KJC-PCS-0052 PR-A — antes vivían dos consts hard-coded para JS aquí
+// (CODE_EXT_RE + SKIP_SEGMENTS). Ahora vienen del registry de adapters,
+// que se consulta una vez por proceso para el conjunto global y por
+// projectDir cuando hace falta filtrar el walk.
+const ALL_CODE_EXTENSIONS = new Set(getAllCodeExtensions());
+const ONBOARDING_DIR = "onboarding", PLANS_DIR = "plans";
+
+function detectKind(path) {
+  const ext = extname(path).toLowerCase();
+  const base = path.split("/").pop() || "";
+  if (ext === ".json" && (/^plan-.+\.json$/i.test(base) || path.includes(`/${PLANS_DIR}/`))) return "plan";
+  if (ext === ".md") return path.includes(`/${ONBOARDING_DIR}/`) ? "onboarding" : "plan";
+  if (ALL_CODE_EXTENSIONS.has(ext)) return "code";
+  return null;
+}
+
+function chunksFor(path, kind) {
+  const text = readFileSync(path, "utf8");
+  if (kind === "plan" && extname(path) === ".json") {
+    try { return chunkPlan(JSON.parse(text), { path }); }
+    catch { return []; }
+  }
+  if (kind === "plan" || kind === "onboarding") {
+    return chunkMarkdown(text, { path, kind });
+  }
+  if (kind === "code") return chunkSource(text, { path });
+  return [];
+}
+
+/**
+ * Index a single file. Returns `{ indexed, failed }`. Idempotent —
+ * pre-existing chunks for this `source` path are deleted before the
+ * fresh batch is written, so a second call against the same file
+ * leaves the store with exactly the latest chunks (no duplicates).
+ */
+export async function indexFile(path, { db, embedder, logger = console, project = null } = {}) {
+  if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
+  const kind = detectKind(path);
+  if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
+  const chunks = chunksFor(path, kind);
+  if (!chunks.length) return { indexed: 0, failed: 0, skipped: 0 };
+  deleteChunksBySource(db, path);
+  let indexed = 0;
+  let failed = 0;
+  let skipped = 0;
+  for (const ch of chunks) {
+    // KJC-TSK-0484 — Skip re-embedding identical bodies for the same project.
+    // The chunks row is still rewritten under the new source so deletions of
+    // an alias don't orphan a row whose owner was the dedup target.
+    const contentHash = createHash("sha256").update(ch.text).digest("hex");
+    try {
+      const dup = findChunkByHash(db, contentHash, project);
+      if (dup) { skipped += 1; continue; }
+      const embedding = await embedder.embed(ch.text);
+      insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project, contentHash });
+      indexed += 1;
+    } catch (err) {
+      failed += 1;
+      logger.warn?.(`[rag-indexer] embed failed for ${path} (${chunkLabel(ch)}): ${err.message}`);
+    }
+  }
+  const note = skipped ? `, ${skipped} dedup` : "";
+  logger.info?.(`[rag-indexer] ${relative(process.cwd(), path)} → ${indexed} chunk(s) (${failed} failed${note})`);
+  return { indexed, failed, skipped };
+}
+
+function chunkLabel(ch) {
+  return ch.metadata?.hu_id || ch.metadata?.symbol || ch.metadata?.headingPath?.join(" > ") || "block";
+}
+
+// KJC-PCS-0052 PR-B.2.4 — Activa el camino AST en chunkers cuyo adapter
+// expone `prepare()` (Python, Rust). Promise.allSettled aísla cada grammar:
+// si uno falla (red, wasm corrupto), el resto se carga y los .py/.rs sin
+// AST listo caen al fallback regex en chunkSource. JS no tiene `prepare`
+// porque chunker-ast.js usa @babel/parser síncrono (ya cargado).
+export async function prepareAdapters(adapters, { logger = console } = {}) {
+  const tasks = adapters.filter((a) => typeof a.prepare === "function");
+  if (tasks.length === 0) return;
+  const results = await Promise.allSettled(tasks.map((a) => a.prepare()));
+  results.forEach((r, i) => {
+    if (r.status === "rejected") logger.warn?.(`[rag-indexer] adapter ${tasks[i].id} prepare failed: ${r.reason?.message || r.reason}`);
+  });
+}
+
+async function listFiles(dir, predicate) {
+  const out = [];
+  async function walk(d) {
+    let ents; try { ents = await readdir(d, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      const p = join(d, e.name);
+      if (e.isDirectory()) await walk(p); else if (e.isFile() && predicate(p)) out.push(p);
+    }
+  }
+  await walk(dir); return out;
+}
+
+/**
+ * Index every Karajan-managed asset for the project: every plan-*.json
+ * under `<karajanHome>/plans/<slug>/`, the onboarding brief under
+ * `<karajanHome>/onboarding/`, and (optionally) source files under the
+ * projectDir. Sources are gated behind `withSources: true` because
+ * indexing tens of thousands of files is expensive — the CLI exposes
+ * it as `--with-sources`.
+ */
+export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false } = {}) {
+  // KJC-TSK-0640: a full index must stamp HEAD too — without it,
+  // last_indexed_commit stayed null after the FIRST index, so the drift
+  // delta-update (maybeAutoUpdate) never engaged until a manual --since.
+  const totals = { indexed: 0, failed: 0, files: 0, head: null };
+  try {
+    const { stdout } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
+    totals.head = stdout.trim();
+  } catch { /* not a git repo (or zero commits) — nothing to stamp */ }
+  const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  await prepareAdapters(detectAdaptersForProject(projectDir), { logger });
+  const planRoot = join(karajanHome, PLANS_DIR, slug);
+  if (existsSync(planRoot)) {
+    const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
+    for (const p of plans) {
+      const r = await indexFile(p, { db, embedder, logger, project: slug });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
+  if (existsSync(onboarding)) {
+    const r = await indexFile(onboarding, { db, embedder, logger, project: slug });
+    totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+  }
+  if (withSources) {
+    // KJC-BUG-0062: `.kj/worktrees/HU-*/` contains git worktrees Karajan spawns
+    // per HU during `kj run`; they hold complete copies of `src/` that
+    // duplicate every chunk and contaminate retrieval scores. `_diet` is the
+    // tests/_diet/ sandbox used by the test-diet audit harness — never user
+    // code. Both viven en COMMON_SKIP_SEGMENTS dentro del registry.
+    const matchers = buildMatchers(detectAdaptersForProject(projectDir));
+    const sources = await listFiles(projectDir, (p) => matchers.isCodeFile(p) && !matchers.shouldSkip(p));
+    for (const s of sources) {
+      const r = await indexFile(s, { db, embedder, logger, project: slug });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  return totals;
+}
+
+/**
+ * KJC-TSK-0455 — Re-index only the files changed between `since` and HEAD.
+ * `git diff --name-status -z` drives the work: A/M re-embed, D drop chunks,
+ * R drops old + indexes new. Returns `{ indexed, failed, files, deleted, head }`;
+ * `head` lets callers persist via setLastIndexedCommit. Throws on diff failure
+ * (shallow clone, unknown ref) so the CLI can fall back to a full reindex.
+ */
+export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console } = {}) {
+  const totals = { indexed: 0, failed: 0, files: 0, deleted: 0, head: null };
+  const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
+  const { stdout: head } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
+  totals.head = head.trim();
+  if (totals.head === since) return totals;
+  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, "HEAD"]);
+  const parts = stdout.split("\0").filter(Boolean);
+  const ops = []; // { kind: 'del'|'idx', p }
+  for (let i = 0; i < parts.length; i += 1) {
+    const s = parts[i];
+    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[++i] }, { kind: "idx", p: parts[++i] }); }
+    else if (s === "D") { ops.push({ kind: "del", p: parts[++i] }); }
+    else { ops.push({ kind: "idx", p: parts[++i] }); }
+  }
+  const adapters = detectAdaptersForProject(projectDir);
+  await prepareAdapters(adapters, { logger });
+  const matchers = buildMatchers(adapters);
+  for (const { kind, p } of ops) {
+    if (matchers.shouldSkip(p)) continue;
+    if (!matchers.isCodeFile(p)) continue;
+    const abs = isAbsolute(p) ? p : join(projectDir, p);
+    if (kind === "del") {
+      totals.deleted += deleteChunksBySource(db, abs);
+      logger.info?.(`[rag-indexer] delta delete ${relative(process.cwd(), abs)}`);
+    } else if (existsSync(abs)) {
+      const r = await indexFile(abs, { db, embedder, logger, project: slug });
+      totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
+    }
+  }
+  return totals;
+}

package/src/rag/library.js

@@ -0,0 +1,62 @@
+/**
+ * library — the distilled engineering canon as its own RAG collection
+ * (KJC-TSK-0697). Markdown cards from `<pkg>/library/`, `~/.karajan/library/`
+ * and `<project>/.karajan/library/` index into the SAME global rag.db,
+ * isolated by project="library" + kind="library", so `kj rag query --library`
+ * reaches the canon and normal project queries never see it. The architect
+ * consults it to ground the greenfield alternative (KJC-TSK-0696).
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { extname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import os from "node:os";
+
+import { chunkMarkdown } from "./chunker.js";
+import { insertChunk, deleteChunksBySource, findChunkByHash } from "./vec-store.js";
+
+export const LIBRARY_PROJECT = "library";
+
+const SHIPPED_LIBRARY_DIR = fileURLToPath(new URL("../../library", import.meta.url));
+
+/** Existing library dirs, shipped canon first. */
+export function libraryDirs({ pkgLibraryDir = SHIPPED_LIBRARY_DIR, home = os.homedir(), projectDir = process.cwd() } = {}) {
+  const candidates = [pkgLibraryDir, join(home, ".karajan", "library"), join(projectDir, ".karajan", "library")];
+  return candidates.filter((d) => existsSync(d));
+}
+
+/**
+ * Index every markdown card found in the library dirs. Idempotent: chunks
+ * re-index per source and identical bodies skip the embedder (content-hash
+ * dedup within the library project).
+ */
+export async function indexLibrary({ db, embedder, logger = console, pkgLibraryDir, home, projectDir } = {}) {
+  let indexed = 0, failed = 0, files = 0;
+  for (const dir of libraryDirs({ pkgLibraryDir, home, projectDir })) {
+    for (const name of readdirSync(dir)) {
+      if (extname(name).toLowerCase() !== ".md") continue;
+      const path = join(dir, name);
+      files += 1;
+      const chunks = chunkMarkdown(readFileSync(path, "utf8"), { path, kind: "library" });
+      const hashed = chunks.map((ch) => ({ ch, contentHash: createHash("sha256").update(ch.text).digest("hex") }));
+      // Unchanged card (every chunk body already known) → skip before the
+      // delete, so idempotent re-runs never touch the embedder.
+      if (hashed.length && hashed.every(({ contentHash }) => findChunkByHash(db, contentHash, LIBRARY_PROJECT))) continue;
+      deleteChunksBySource(db, path);
+      for (const { ch, contentHash } of hashed) {
+        try {
+          if (findChunkByHash(db, contentHash, LIBRARY_PROJECT)) continue;
+          const embedding = await embedder.embed(ch.text);
+          insertChunk(db, { source: path, kind: "library", text: ch.text, metadata: ch.metadata, embedding, project: LIBRARY_PROJECT, contentHash });
+          indexed += 1;
+        } catch (err) {
+          failed += 1;
+          logger.warn?.(`[rag-library] embed failed for ${path}: ${err.message}`);
+        }
+      }
+    }
+  }
+  if (files) logger.info?.(`[rag-library] ${files} card(s) → ${indexed} chunk(s) (${failed} failed)`);
+  return { files, indexed, failed };
+}

package/src/rag/ollama-capability.js

@@ -0,0 +1,59 @@
+// KJC-TSK-0436 (RAG Auto-Bootstrap PR2) — Capability check for Ollama
+// bootstrap. Returns whether the host can reasonably run a local Ollama
+// container: Docker installed, daemon reachable, and enough RAM
+// (default >= 4 GB free). The caller (kj init) decides whether to
+// bootstrap, warn, or skip.
+import os from "node:os";
+import { runCommand } from "../utils/process.js";
+
+const DEFAULT_MIN_RAM_BYTES = 4 * 1024 * 1024 * 1024; // 4 GB free
+const DEFAULT_TIMEOUT_MS = 5000;
+
+export async function checkDockerAvailable(timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const v = await runCommand("docker", ["--version"], { timeout: timeoutMs }).catch(() => ({ exitCode: 127 }));
+  if (v.exitCode !== 0) return { ok: false, reason: "docker:not-installed" };
+  const info = await runCommand("docker", ["info"], { timeout: timeoutMs }).catch(() => ({ exitCode: 1 }));
+  if (info.exitCode !== 0) return { ok: false, reason: "docker:daemon-unreachable" };
+  return { ok: true };
+}
+
+export function checkRamCapacity(minBytes = DEFAULT_MIN_RAM_BYTES) {
+  const free = os.freemem();
+  const total = os.totalmem();
+  if (free < minBytes) {
+    return {
+      ok: false,
+      reason: `ram:insufficient (${formatGB(free)} free < ${formatGB(minBytes)} required)`,
+      freeBytes: free,
+      totalBytes: total,
+    };
+  }
+  return { ok: true, freeBytes: free, totalBytes: total };
+}
+
+/**
+ * Combined capability check. Returns `{ capable, reasons[] }`.
+ * Caller decides what to do (bootstrap, warn, skip).
+ */
+export async function checkOllamaCapability({ minRamBytes = DEFAULT_MIN_RAM_BYTES } = {}) {
+  const reasons = [];
+  const docker = await checkDockerAvailable();
+  if (!docker.ok) reasons.push(docker.reason);
+  const ram = checkRamCapacity(minRamBytes);
+  if (!ram.ok) reasons.push(ram.reason);
+  return { capable: reasons.length === 0, reasons, docker, ram };
+}
+
+function formatGB(bytes) {
+  return `${(bytes / (1024 ** 3)).toFixed(1)} GB`;
+}
+
+/**
+ * Pull a model into the Ollama container. Uses `docker exec` so we do
+ * not depend on the host having `ollama` CLI installed. Wired by `kj
+ * init` after `ollamaUp`, and by `kj ollama pull <model>` (PR3).
+ */
+export async function pullOllamaModel(model, { containerName = "kj-ollama", timeoutMs = 10 * 60 * 1000 } = {}) {
+  if (!model || typeof model !== "string") throw new Error("pullOllamaModel: model is required");
+  return runCommand("docker", ["exec", containerName, "ollama", "pull", model], { timeout: timeoutMs });
+}