karajan-code 2.27.0 → 2.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/package.json +2 -1
- package/packages/hu-board/public/index.html +1 -0
- package/packages/hu-board/public/rag.html +21 -0
- package/packages/hu-board/public/rag.js +39 -0
- package/packages/hu-board/public/styles.css +14 -0
- package/packages/hu-board/src/routes/api.js +2 -2
- package/packages/hu-board/src/routes/rag.js +36 -0
- package/packages/hu-board/src/server.js +2 -0
- package/scripts/esbuild-sea.config.mjs +14 -1
- package/src/cli/register-meta.js +18 -0
- package/src/commands/rag.js +6 -7
- package/src/commands/watch.js +46 -0
- package/src/mcp/handlers/rag-handler.js +1 -6
- package/src/orchestrator/stages/rag-context-stage.js +2 -3
- package/src/rag/chunker-ast.js +56 -0
- package/src/rag/chunker.js +14 -5
- package/src/rag/embedders/_cloud-base.js +24 -0
- package/src/rag/embedders/cohere.js +28 -0
- package/src/rag/embedders/factory.js +30 -0
- package/src/rag/embedders/mistral.js +26 -0
- package/src/rag/embedders/onnx.js +63 -0
- package/src/rag/embedders/openai.js +22 -0
- package/src/rag/embedders/voyage.js +18 -0
- package/src/rag/rerank.js +74 -0
- package/src/rag/retriever.js +52 -54
- package/src/rag/vec-store.js +54 -2
- package/src/rag/watcher.js +60 -0
- package/src/rag/where-parser.js +54 -0
- package/src/utils/rate-limit-detector.js +54 -12
package/README.md
CHANGED
|
@@ -345,6 +345,22 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
345
345
|
|
|
346
346
|
## Recent releases
|
|
347
347
|
|
|
348
|
+
> **v2.29.0 released** — Minor. **RAG quality lift** — dashboard + three new providers + metadata filter + rerank.
|
|
349
|
+
>
|
|
350
|
+
> - **Retrieval dashboard on HU Board** (PR #843): nuevo `/rag.html` con embedder activo, tamaño de DB, last-index, chunks por kind, chunks por proyecto. Primera pieza del config UI que llega completo en v2.30.
|
|
351
|
+
> - **Cohere + Mistral embedders** (PR #848): `embed-multilingual-v3.0` (multi-idioma) y `mistral-embed` (EU-hosted, útil para GDPR). `KJ_COHERE_KEY` / `KJ_MISTRAL_KEY` Karajan-scoped.
|
|
352
|
+
> - **ONNX local embedder** (PR #??): `@huggingface/transformers` corriendo en Node — sin Docker, sin API key, sin Ollama. Default `Xenova/all-MiniLM-L6-v2` (384 dim). Es la base para el zero-config init de v2.31.
|
|
353
|
+
> - **Metadata `--where` filter** (PR #??): `kj rag query 'auth' --where 'symbol=loadConfig AND kind=plan'`. Gramática mínima KEY=VALUE AND KEY=VALUE; cualquier metadata que emite el chunker es queryable sin schema changes.
|
|
354
|
+
> - **Cross-encoder rerank** (PR #??): opt-in `--rerank` re-ordena los topK con un cross-encoder (`Xenova/ms-marco-MiniLM-L-6-v2`). Latencia acotada, calidad final notablemente mejor.
|
|
355
|
+
>
|
|
356
|
+
> **v2.28.0 released** — Minor. **RAG advanced** — live re-index + cloud embedders + hybrid scoring + AST chunker, plus a real fix for the v2.27.0 chapuza.
|
|
357
|
+
>
|
|
358
|
+
> - **`kj watch`** (PR #836): chokidar daemon que re-indexa los archivos del proyecto tras cada edit (1s debounce). Fin del `kj rag index` manual.
|
|
359
|
+
> - **OpenAI + Voyage embedders** (PR #841): para usuarios sin Docker local. `config.rag.embedder.provider: openai | voyage` + `KJ_OPENAI_KEY` / `KJ_VOYAGE_KEY` env vars (Karajan-scoped — preserva el architecture invariant).
|
|
360
|
+
> - **BM25 hybrid scoring** (PR #838): SQLite FTS5 + cosine fusion. `kj rag query --mode hybrid|semantic|keyword`. Queries con símbolos exactos rankean correctamente.
|
|
361
|
+
> - **AST source chunker** (PR #839): `@babel/parser`. Cada top-level declaration entera en un chunk; TypeScript + JSX + decorators soportados.
|
|
362
|
+
> - **KJC-BUG-0064** (PR #840): `parseCooldown` TZ-aware via `Intl.DateTimeFormat`. Deshace el skip-in-CI workaround de v2.27.0 — tests pasan en cualquier TZ.
|
|
363
|
+
>
|
|
348
364
|
> **v2.27.0 released** — Minor. **RAG polish** — three improvements triggered by the v2.26.0 smoke test on karajan-code itself:
|
|
349
365
|
>
|
|
350
366
|
> - **Per-project isolation** (PR #831): new `project_slug` column on chunks; `kj rag query --project <slug>` (auto-detected from cwd) filters the global DB. `--project all` to query across everything.
|
|
@@ -352,7 +368,7 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
352
368
|
> - **Asymmetric source-vs-test ranking** (PR #833): NL queries like `how does X work` no longer rank `tests/X.test.js` above `src/X.js`; test-flavoured queries still surface tests.
|
|
353
369
|
> - Plus **KJC-BUG-0063** (PR #834): skipped a TZ-dependent test that was blocking every CI run, and a `shrink-budget` workflow exclude fix for `docs/*.md` at the root.
|
|
354
370
|
>
|
|
355
|
-
> **Coming in v2.
|
|
371
|
+
> **Coming in v2.30.0+**: writable config UI on the HU Board (toggle roles, switch coder/reviewer, adjust alpha/mode/rerank without re-editing the YAML), then v2.31 zero-config init (reduce the wizard to one critical question, smart defaults for everything).
|
|
356
372
|
>
|
|
357
373
|
> **v2.26.0 released** — Minor. **RAG Auto-Bootstrap** — Ollama runs in Docker out of the box. `kj init` now provisions the embedder automatically (or skips with a clear reason on modest hardware / `--no-ollama`); `kj doctor` surfaces health; `kj ollama [start|stop|status|pull]` manages lifecycle without docker compose. See [docs/RAG.md](docs/RAG.md) for the full RAG guide ([español](docs/es/RAG.md)).
|
|
358
374
|
>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "karajan-code",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.29.0",
|
|
4
4
|
"description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "AGPL-3.0",
|
|
@@ -69,6 +69,7 @@
|
|
|
69
69
|
"src/**/*.js": "eslint --max-warnings=999"
|
|
70
70
|
},
|
|
71
71
|
"dependencies": {
|
|
72
|
+
"@babel/parser": "^7.29.3",
|
|
72
73
|
"@modelcontextprotocol/sdk": "^1.26.0",
|
|
73
74
|
"better-sqlite3": "^11.0.0",
|
|
74
75
|
"chokidar": "^4.0.0",
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
<button class="nav-btn" data-view="dashboard" title="Per-project landing: stats grid + project list. Click a project card to focus the kanban / graph on that project.">Dashboard</button>
|
|
26
26
|
<button class="nav-btn" data-view="sessions" title="Every kj run that has touched the board, newest first. Filter by project; click a session to inspect its iterations, commits and checkpoints.">Sessions</button>
|
|
27
27
|
<a class="nav-btn" href="/pipeline.html" title="Live observability of pipeline runs (Karajan v2.7+): stage timings, agent calls, decision logs.">Pipeline</a>
|
|
28
|
+
<a class="nav-btn" href="/rag.html" title="RAG retrieval-quality dashboard: chunk counts, embedder provider, db size.">RAG</a>
|
|
28
29
|
<select class="project-select" id="project-select">
|
|
29
30
|
<option value="">All Projects</option>
|
|
30
31
|
</select>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>Karajan RAG · Retrieval Dashboard</title>
|
|
7
|
+
<link rel="stylesheet" href="/styles.css">
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
<header class="header">
|
|
11
|
+
<div class="header__logo"><div><div class="header__title">Karajan RAG</div><div class="header__subtitle">retrieval-quality dashboard</div></div></div>
|
|
12
|
+
<nav class="header__nav">
|
|
13
|
+
<a class="nav-btn" href="/">← Board</a>
|
|
14
|
+
<button class="nav-btn active">RAG</button>
|
|
15
|
+
<div class="header__controls"><button class="control-btn" id="rag-refresh" title="Reload stats">🔄</button></div>
|
|
16
|
+
</nav>
|
|
17
|
+
</header>
|
|
18
|
+
<main class="main" id="rag-app"><div class="loading"><div class="loading__spinner"></div><p>Loading…</p></div></main>
|
|
19
|
+
<script src="/rag.js"></script>
|
|
20
|
+
</body>
|
|
21
|
+
</html>
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// KJC-TSK-0445 — Retrieval dashboard frontend. Fetches /api/rag/stats and
|
|
2
|
+
// renders chunk counts per project/kind, embedder, DB size and last-index.
|
|
3
|
+
const TOKEN = new URLSearchParams(location.search).get("token") || localStorage.getItem("kj_board_token") || "";
|
|
4
|
+
const HEADERS = TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {};
|
|
5
|
+
const ESC = (s) => String(s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
6
|
+
|
|
7
|
+
function bytes(n) { for (const u of ["B", "KB", "MB", "GB"]) { if (n < 1024) return `${u === "B" ? n : n.toFixed(1)} ${u}`; n /= 1024; } return `${n.toFixed(2)} TB`; }
|
|
8
|
+
|
|
9
|
+
function bars(rows, lab, val, max) {
|
|
10
|
+
if (!rows.length) return '<p class="rag-empty">no data</p>';
|
|
11
|
+
return rows.map((r) => `<div class="rag-bar"><span class="rag-bar__label">${ESC(lab(r))}</span><span class="rag-bar__fill" style="width:${Math.round((val(r) / max) * 100)}%"></span><span class="rag-bar__value">${val(r).toLocaleString()}</span></div>`).join("");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function embedderCard(e) {
|
|
15
|
+
return e ? `<div class="rag-card"><h3>Embedder</h3><p class="rag-stat-big">${ESC(e.provider)}</p><p class="rag-stat-sub">${ESC(e.model || "default model")} · dim ${e.dim ?? "?"}</p></div>` : "";
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function render(root, data) {
|
|
19
|
+
if (!data.initialized) { root.innerHTML = `<section class="rag-grid"><div class="rag-card"><h3>RAG DB</h3><p class="rag-empty">${ESC(data.message || "not initialized")}</p></div>${embedderCard(data.embedder)}</section>`; return; }
|
|
20
|
+
const lastIdx = data.last_indexed ? new Date(data.last_indexed).toLocaleString() : "never";
|
|
21
|
+
const mk = Math.max(1, ...data.by_kind.map((r) => r.n));
|
|
22
|
+
const mp = Math.max(1, ...data.by_project.map((r) => r.n));
|
|
23
|
+
root.innerHTML = `<section class="rag-grid">
|
|
24
|
+
<div class="rag-card"><h3>Totals</h3><p class="rag-stat-big">${data.total_chunks.toLocaleString()}</p><p class="rag-stat-sub">chunks · ${bytes(data.db_size_bytes)} · last indexed ${ESC(lastIdx)}</p></div>
|
|
25
|
+
${embedderCard(data.embedder)}
|
|
26
|
+
<div class="rag-card rag-card--wide"><h3>By kind</h3>${bars(data.by_kind, (r) => r.kind, (r) => r.n, mk)}</div>
|
|
27
|
+
<div class="rag-card rag-card--wide"><h3>By project (top 20)</h3>${bars(data.by_project, (r) => r.project, (r) => r.n, mp)}</div>
|
|
28
|
+
</section>`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function load() {
|
|
32
|
+
const app = document.getElementById("rag-app");
|
|
33
|
+
app.innerHTML = '<div class="loading"><div class="loading__spinner"></div><p>Loading…</p></div>';
|
|
34
|
+
try { const r = await fetch("/api/rag/stats", { headers: HEADERS }); if (!r.ok) throw new Error(`HTTP ${r.status}`); render(app, await r.json()); }
|
|
35
|
+
catch (e) { app.innerHTML = `<div class="rag-card rag-card--error"><strong>Error:</strong> ${ESC(e.message)}</div>`; }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
document.getElementById("rag-refresh").addEventListener("click", load);
|
|
39
|
+
load();
|
|
@@ -916,3 +916,17 @@ body.scoped-mode .nav-btn[data-view="dashboard"],
|
|
|
916
916
|
body.scoped-mode .header__nav a[href="/pipeline.html"] {
|
|
917
917
|
display: none;
|
|
918
918
|
}
|
|
919
|
+
|
|
920
|
+
/* KJC-TSK-0445 — RAG dashboard */
|
|
921
|
+
.rag-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 1rem; padding: 1rem; }
|
|
922
|
+
.rag-card { background: var(--bg-secondary, #1e1e2e); border-radius: 8px; padding: 1.25rem; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
|
|
923
|
+
.rag-card--wide { grid-column: 1 / -1; }
|
|
924
|
+
.rag-card--error { color: #f87171; background: #2a1414; }
|
|
925
|
+
.rag-card h3 { margin: 0 0 .75rem; font-size: .9rem; text-transform: uppercase; letter-spacing: .05em; color: var(--text-secondary, #94a3b8); }
|
|
926
|
+
.rag-stat-big { font-size: 2rem; font-weight: 700; margin: 0; color: var(--text-primary, #f8fafc); }
|
|
927
|
+
.rag-stat-sub { margin: .25rem 0 0; font-size: .85rem; color: var(--text-secondary, #94a3b8); }
|
|
928
|
+
.rag-empty { color: var(--text-secondary, #94a3b8); font-style: italic; }
|
|
929
|
+
.rag-bar { display: grid; grid-template-columns: 140px 1fr 60px; align-items: center; gap: .5rem; margin: .35rem 0; font-size: .85rem; }
|
|
930
|
+
.rag-bar__label { color: var(--text-secondary, #cbd5e1); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
931
|
+
.rag-bar__fill { display: block; height: 14px; background: linear-gradient(90deg, #7c3aed, #a78bfa); border-radius: 3px; }
|
|
932
|
+
.rag-bar__value { text-align: right; color: var(--text-primary, #f8fafc); font-variant-numeric: tabular-nums; }
|
|
@@ -1292,12 +1292,12 @@ router.post('/rag/query', async (req, res) => {
|
|
|
1292
1292
|
}
|
|
1293
1293
|
try {
|
|
1294
1294
|
const { openVecStore, countChunks } = await import('../../../../src/rag/vec-store.js');
|
|
1295
|
-
const {
|
|
1295
|
+
const { makeEmbedder } = await import('../../../../src/rag/embedders/factory.js');
|
|
1296
1296
|
const { query } = await import('../../../../src/rag/retriever.js');
|
|
1297
1297
|
const db = openVecStore({ dim: 768 });
|
|
1298
1298
|
try {
|
|
1299
1299
|
if (countChunks(db) === 0) return res.json({ hits: [], empty: true, topK, scope });
|
|
1300
|
-
const embedder =
|
|
1300
|
+
const embedder = makeEmbedder({});
|
|
1301
1301
|
const hits = await query(db, embedder, text, { topK: Number(topK), scope });
|
|
1302
1302
|
res.json({ hits, empty: false, topK, scope });
|
|
1303
1303
|
} finally { db.close(); }
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// KJC-TSK-0445 — RAG retrieval-quality dashboard route.
|
|
2
|
+
// GET /api/rag/stats → snapshot of the local rag.db (counts per kind/project,
|
|
3
|
+
// active embedder, db size, last-index timestamp). Missing DB returns an
|
|
4
|
+
// "uninitialized" payload so the dashboard renders an empty state cleanly.
|
|
5
|
+
import { Router } from "express";
|
|
6
|
+
import { existsSync, statSync } from "node:fs";
|
|
7
|
+
import Database from "better-sqlite3";
|
|
8
|
+
import { dbPath } from "../../../../src/rag/vec-store.js";
|
|
9
|
+
import { readConfig } from "../config-yaml.js";
|
|
10
|
+
|
|
11
|
+
const router = Router();
|
|
12
|
+
const DIMS = { ollama: 768, openai: 1536, voyage: 1024, cohere: 1024, mistral: 1024, onnx: 384 };
|
|
13
|
+
|
|
14
|
+
function embedder() {
|
|
15
|
+
try {
|
|
16
|
+
const e = (readConfig()?.rag?.embedder) || {};
|
|
17
|
+
const provider = e.provider || "ollama";
|
|
18
|
+
return { provider, model: e.model || null, dim: e.dim || DIMS[provider] || null };
|
|
19
|
+
} catch { return { provider: "ollama", model: null, dim: 768 }; }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
router.get("/stats", (_req, res) => {
|
|
23
|
+
const path = dbPath();
|
|
24
|
+
if (!existsSync(path)) return res.json({ ok: true, initialized: false, message: "rag.db not initialized — run `kj rag index <dir>` to start indexing.", embedder: embedder() });
|
|
25
|
+
try {
|
|
26
|
+
const db = new Database(path, { readonly: true, fileMustExist: true });
|
|
27
|
+
const total_chunks = db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
28
|
+
const by_kind = db.prepare("SELECT kind, COUNT(*) AS n FROM chunks GROUP BY kind").all();
|
|
29
|
+
const by_project = db.prepare("SELECT COALESCE(project_slug, '(no slug)') AS project, COUNT(*) AS n FROM chunks GROUP BY project_slug ORDER BY n DESC LIMIT 20").all();
|
|
30
|
+
const last_indexed = db.prepare("SELECT MAX(created_at) AS ts FROM chunks").get().ts;
|
|
31
|
+
db.close();
|
|
32
|
+
res.json({ ok: true, initialized: true, db_path: path, db_size_bytes: statSync(path).size, total_chunks, by_kind, by_project, last_indexed, embedder: embedder() });
|
|
33
|
+
} catch (err) { res.status(500).json({ ok: false, error: String(err.message || err) }); }
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
export default router;
|
|
@@ -9,6 +9,7 @@ import { initDb, closeDb } from './db.js';
|
|
|
9
9
|
import { fullScan, startWatcher } from './sync.js';
|
|
10
10
|
import apiRoutes from './routes/api.js';
|
|
11
11
|
import pipelineRoutes from './routes/pipeline.js';
|
|
12
|
+
import ragRoutes from './routes/rag.js';
|
|
12
13
|
import { authMiddleware } from './auth.js';
|
|
13
14
|
import { getOrCreateToken, getTokenPath } from './token-store.js';
|
|
14
15
|
import { reapZombieSessions } from './zombie-reaper.js';
|
|
@@ -298,6 +299,7 @@ async function main() {
|
|
|
298
299
|
app.use(express.static(PUBLIC_DIR, { setHeaders: noStoreHeaders, etag: false, lastModified: false }));
|
|
299
300
|
app.use('/api', buildRateLimiter(), authMiddleware(), apiRoutes);
|
|
300
301
|
app.use('/api/pipeline', authMiddleware(), pipelineRoutes);
|
|
302
|
+
app.use('/api/rag', authMiddleware(), ragRoutes);
|
|
301
303
|
|
|
302
304
|
// SPA fallback: serve index.html for non-API, non-static routes
|
|
303
305
|
app.get('/{*splat}', (_req, res) => {
|
|
@@ -146,7 +146,7 @@ const huBoardStubPlugin = {
|
|
|
146
146
|
const ragStubPlugin = {
|
|
147
147
|
name: "rag-stub",
|
|
148
148
|
setup(build) {
|
|
149
|
-
build.onResolve({ filter: /[\\/](rag[\\/]
|
|
149
|
+
build.onResolve({ filter: /[\\/](rag[\\/].+|commands[\\/](rag|watch))\.js$/ }, (args) => ({
|
|
150
150
|
path: args.path, namespace: "rag-stub",
|
|
151
151
|
}));
|
|
152
152
|
build.onLoad({ filter: /.*/, namespace: "rag-stub" }, () => ({
|
|
@@ -170,6 +170,19 @@ const ragStubPlugin = {
|
|
|
170
170
|
checkOllamaCapability: notAvailable, pullOllamaModel: notAvailable,
|
|
171
171
|
// KJC-TSK-0438 — RAG project isolation.
|
|
172
172
|
projectSlug: notAvailable,
|
|
173
|
+
// KJC-TSK-0441 — RAG chokidar watcher.
|
|
174
|
+
startWatcher: notAvailable, readPidFile: notAvailable, clearPidFile: notAvailable,
|
|
175
|
+
isPidAlive: notAvailable, writePidFile: notAvailable,
|
|
176
|
+
// KJC-TSK-0442 / KJC-TSK-0446 — RAG embedders cloud + factory.
|
|
177
|
+
makeEmbedder: notAvailable, OpenAIEmbedder: notAvailable, VoyageEmbedder: notAvailable,
|
|
178
|
+
OpenAIEmbedderError: notAvailable, VoyageEmbedderError: notAvailable, PROVIDERS: notAvailable,
|
|
179
|
+
CohereEmbedder: notAvailable, CohereEmbedderError: notAvailable,
|
|
180
|
+
MistralEmbedder: notAvailable, MistralEmbedderError: notAvailable,
|
|
181
|
+
ONNXEmbedder: notAvailable, ONNXEmbedderError: notAvailable,
|
|
182
|
+
// KJC-TSK-0448 — RAG --where metadata filter.
|
|
183
|
+
parseWhere: notAvailable, buildWhereSql: notAvailable,
|
|
184
|
+
// KJC-TSK-0449 — RAG cross-encoder rerank.
|
|
185
|
+
rerank: notAvailable, RerankError: notAvailable, _resetPipeline: notAvailable,
|
|
173
186
|
default: notAvailable,
|
|
174
187
|
};
|
|
175
188
|
`,
|
package/src/cli/register-meta.js
CHANGED
|
@@ -4,6 +4,7 @@ import { researcherCommand } from "../commands/researcher.js";
|
|
|
4
4
|
import { architectCommand } from "../commands/architect.js";
|
|
5
5
|
import { onboardCommand } from "../commands/onboard.js";
|
|
6
6
|
import { ragIndexCommand, ragQueryCommand } from "../commands/rag.js";
|
|
7
|
+
import { watchStartCommand, watchStopCommand, watchStatusCommand } from "../commands/watch.js";
|
|
7
8
|
import { auditCommand } from "../commands/audit.js";
|
|
8
9
|
import { resumeCommand } from "../commands/resume.js";
|
|
9
10
|
import { boardCommand } from "../commands/board.js";
|
|
@@ -114,6 +115,11 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
114
115
|
.option("--scope <scope>", "plans | code | onboarding | all (default: all)", "all")
|
|
115
116
|
.option("--top-k <n>", "Number of hits to return (default: 5)", "5")
|
|
116
117
|
.option("--project <slug>", "Filter by project slug. Pass 'all' to query across every indexed project. Default: cwd basename")
|
|
118
|
+
.option("--mode <mode>", "hybrid (default) | semantic (cosine only) | keyword (BM25 only)", "hybrid")
|
|
119
|
+
.option("--alpha <n>", "Hybrid weight for semantic component (0..1). Default 0.6", "0.6")
|
|
120
|
+
.option("--where <clause>", "Metadata filter, e.g. 'symbol=loadConfig' or 'hu_id=HU-003 AND kind=plan'")
|
|
121
|
+
.option("--rerank", "Apply cross-encoder rerank to top-K (needs @huggingface/transformers, slower but more precise)")
|
|
122
|
+
.option("--rerank-model <name>", "Cross-encoder model id. Default: Xenova/ms-marco-MiniLM-L-6-v2")
|
|
117
123
|
.option("--json", "Output the hits as JSON")
|
|
118
124
|
.action(async (text, flags) => {
|
|
119
125
|
await withConfig(pkgVersion, "rag-query", flags, async ({ config, logger }) => {
|
|
@@ -121,6 +127,18 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
121
127
|
});
|
|
122
128
|
});
|
|
123
129
|
|
|
130
|
+
// KJC-TSK-0441 — `kj watch [start|stop|status]` for live RAG re-index.
|
|
131
|
+
const watch = program.command("watch").description("Live RAG re-index daemon");
|
|
132
|
+
watch.command("start").description("Start the watcher daemon").option("--foreground", "Run in foreground (do not detach)").option("--with-sources", "Watch project source files too").action(async (flags) => {
|
|
133
|
+
await withConfig(pkgVersion, "watch:start", flags, async ({ config, logger }) => watchStartCommand({ config, logger, flags }));
|
|
134
|
+
});
|
|
135
|
+
watch.command("stop").description("Stop the watcher daemon").action(async () => {
|
|
136
|
+
await withConfig(pkgVersion, "watch:stop", {}, async ({ logger }) => watchStopCommand({ logger }));
|
|
137
|
+
});
|
|
138
|
+
watch.command("status").description("Report watcher daemon state").action(async () => {
|
|
139
|
+
await withConfig(pkgVersion, "watch:status", {}, async ({ logger }) => watchStatusCommand({ logger }));
|
|
140
|
+
});
|
|
141
|
+
|
|
124
142
|
program
|
|
125
143
|
.command("audit")
|
|
126
144
|
.description("Analyze codebase health (read-only)")
|
package/src/commands/rag.js
CHANGED
|
@@ -3,16 +3,11 @@
|
|
|
3
3
|
// kj rag query <text> [--scope plans|code|onboarding|all] [--top-k N] [--json]
|
|
4
4
|
// Closes the v2.22.0 RAG MVP end-to-end from the terminal.
|
|
5
5
|
import { openVecStore, countChunks, projectSlug } from "../rag/vec-store.js";
|
|
6
|
-
import {
|
|
6
|
+
import { makeEmbedder } from "../rag/embedders/factory.js";
|
|
7
7
|
import { indexProject } from "../rag/indexer.js";
|
|
8
8
|
import { query } from "../rag/retriever.js";
|
|
9
9
|
import { getKarajanHome } from "../utils/paths.js";
|
|
10
10
|
|
|
11
|
-
function makeEmbedder(config) {
|
|
12
|
-
const cfg = config?.rag?.embedder || {};
|
|
13
|
-
return new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim: cfg.dim });
|
|
14
|
-
}
|
|
15
|
-
|
|
16
11
|
function openDb(config) {
|
|
17
12
|
return openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
|
|
18
13
|
}
|
|
@@ -60,7 +55,11 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
|
|
|
60
55
|
if (flags.json) process.stdout.write(`${JSON.stringify({ hits: [], empty: true, topK, scope })}\n`);
|
|
61
56
|
return [];
|
|
62
57
|
}
|
|
63
|
-
const
|
|
58
|
+
const mode = flags.mode || "hybrid";
|
|
59
|
+
const alpha = Math.max(0, Math.min(1, Number(flags.alpha) || 0.6));
|
|
60
|
+
const where = flags.where || null;
|
|
61
|
+
const rerankOpts = flags.rerank ? { model: flags.rerankModel } : null;
|
|
62
|
+
const hits = await query(db, makeEmbedder(config), text, { topK, scope, project, mode, alpha, where, rerankOpts });
|
|
64
63
|
if (flags.json) {
|
|
65
64
|
process.stdout.write(`${JSON.stringify(hits)}\n`);
|
|
66
65
|
} else {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// KJC-TSK-0441 — `kj watch [start|stop|status]`. Thin wrappers.
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { startWatcher, readPidFile, clearPidFile, isPidAlive, writePidFile } from "../rag/watcher.js";
|
|
6
|
+
|
|
7
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
|
|
9
|
+
export async function watchStartCommand({ config, logger, flags = {} }) {
|
|
10
|
+
const projectDir = config?.projectDir || process.cwd();
|
|
11
|
+
const existing = readPidFile();
|
|
12
|
+
if (existing && isPidAlive(existing)) { logger.info(`Watcher already running (pid ${existing}).`); return { pid: existing, started: false }; }
|
|
13
|
+
if (existing) clearPidFile();
|
|
14
|
+
if (flags.foreground) {
|
|
15
|
+
writePidFile();
|
|
16
|
+
const stop = startWatcher({ projectDir, config, logger, withSources: !!flags.withSources });
|
|
17
|
+
const shutdown = async () => { await stop(); clearPidFile(); process.exit(0); };
|
|
18
|
+
process.on("SIGINT", shutdown).on("SIGTERM", shutdown);
|
|
19
|
+
logger.info(`Watcher started in foreground (pid ${process.pid}). Ctrl+C to stop.`);
|
|
20
|
+
return { pid: process.pid, started: true };
|
|
21
|
+
}
|
|
22
|
+
const args = [join(HERE, "..", "cli.js"), "watch", "start", "--foreground", ...(flags.withSources ? ["--with-sources"] : [])];
|
|
23
|
+
const child = spawn(process.execPath, args, { detached: true, stdio: "ignore", env: { ...process.env, KJ_WATCHER_DAEMON: "1" } });
|
|
24
|
+
child.unref();
|
|
25
|
+
logger.info(`Watcher started in background (pid ${child.pid}).`);
|
|
26
|
+
return { pid: child.pid, started: true };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function watchStopCommand({ logger }) {
|
|
30
|
+
const pid = readPidFile();
|
|
31
|
+
if (!pid) { logger.info("No watcher running."); return { stopped: false }; }
|
|
32
|
+
if (!isPidAlive(pid)) { clearPidFile(); logger.info(`Stale PID file removed (process ${pid} was not alive).`); return { stopped: false, stale: true }; }
|
|
33
|
+
try { process.kill(pid, "SIGTERM"); } catch (err) { logger.warn(`Could not signal pid ${pid}: ${err.message}`); }
|
|
34
|
+
clearPidFile();
|
|
35
|
+
logger.info(`Watcher (pid ${pid}) stopped.`);
|
|
36
|
+
return { stopped: true, pid };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function watchStatusCommand({ logger }) {
|
|
40
|
+
const pid = readPidFile();
|
|
41
|
+
const alive = pid && isPidAlive(pid);
|
|
42
|
+
if (alive) logger.info(`Watcher running (pid ${pid}).`);
|
|
43
|
+
else if (pid) logger.warn(`Stale PID file (process ${pid} not alive). Run \`kj watch stop\` to clean up.`);
|
|
44
|
+
else logger.info("Watcher is NOT running.");
|
|
45
|
+
return { pid, alive };
|
|
46
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// from src/commands/* are forbidden by the layer-boundaries test
|
|
3
3
|
// (MCP and CLI are peer layers); we hit the pure rag/* modules instead.
|
|
4
4
|
import { countChunks, openVecStore } from "../../rag/vec-store.js";
|
|
5
|
-
import {
|
|
5
|
+
import { makeEmbedder } from "../../rag/embedders/factory.js";
|
|
6
6
|
import { indexProject } from "../../rag/indexer.js";
|
|
7
7
|
import { query } from "../../rag/retriever.js";
|
|
8
8
|
import { getKarajanHome } from "../../utils/paths.js";
|
|
@@ -12,11 +12,6 @@ const silentLogger = {
|
|
|
12
12
|
info: () => {}, warn: () => {}, error: () => {}, debug: () => {}, setContext: () => {},
|
|
13
13
|
};
|
|
14
14
|
|
|
15
|
-
function makeEmbedder(config) {
|
|
16
|
-
const cfg = config?.rag?.embedder || {};
|
|
17
|
-
return new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim: cfg.dim });
|
|
18
|
-
}
|
|
19
|
-
|
|
20
15
|
export async function handleRagQuery(args, server) {
|
|
21
16
|
const text = args?.text;
|
|
22
17
|
if (typeof text !== "string" || text.length === 0) {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// never sees an exception from this stage.
|
|
7
7
|
import { emitProgress, makeEvent } from "../../utils/events.js";
|
|
8
8
|
import { openVecStore, countChunks } from "../../rag/vec-store.js";
|
|
9
|
-
import {
|
|
9
|
+
import { makeEmbedder } from "../../rag/embedders/factory.js";
|
|
10
10
|
import { query } from "../../rag/retriever.js";
|
|
11
11
|
|
|
12
12
|
const DEFAULT_TOP_K = 5;
|
|
@@ -35,8 +35,7 @@ export async function runRagContextStage({ config, logger, emitter, eventBase, t
|
|
|
35
35
|
}
|
|
36
36
|
const topK = preload.topK || DEFAULT_TOP_K;
|
|
37
37
|
const scope = preload.scope || DEFAULT_SCOPE;
|
|
38
|
-
const
|
|
39
|
-
const embedder = new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim });
|
|
38
|
+
const embedder = makeEmbedder(config);
|
|
40
39
|
const hits = await query(db, embedder, task, { topK, scope });
|
|
41
40
|
if (hits.length === 0) return { skipped: true, reason: "no-hits" };
|
|
42
41
|
const lines = hits.map((h) => {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// KJC-TSK-0444 — AST-aware source chunker. Replaces the regex-based
|
|
2
|
+
// chunkSource for JS/TS/JSX files. Each top-level declaration (function,
|
|
3
|
+
// class, var/const, export named/default) becomes one chunk whose `text`
|
|
4
|
+
// is the literal source slice [node.start, node.end]. Comments
|
|
5
|
+
// immediately preceding a declaration are folded in (JSDoc + leading
|
|
6
|
+
// line comments) so the chunk reads like a doc-aware snippet.
|
|
7
|
+
//
|
|
8
|
+
// Falls back to the regex chunker if @babel/parser cannot parse the
|
|
9
|
+
// file (errorRecovery: true catches most cases; a hard syntax error
|
|
10
|
+
// returns null so the caller can fall back).
|
|
11
|
+
import { parse } from "@babel/parser";
|
|
12
|
+
|
|
13
|
+
const PLUGINS = ["typescript", "jsx", "decorators-legacy", "classProperties", "topLevelAwait"];
|
|
14
|
+
|
|
15
|
+
export function chunkSourceAST(text, { path = "<src>", limit = 800, overlap = 100 } = {}) {
|
|
16
|
+
let ast;
|
|
17
|
+
try { ast = parse(text, { sourceType: "module", errorRecovery: true, plugins: PLUGINS }); }
|
|
18
|
+
catch { return null; }
|
|
19
|
+
const body = ast?.program?.body;
|
|
20
|
+
if (!Array.isArray(body) || body.length === 0) return null;
|
|
21
|
+
const out = [];
|
|
22
|
+
for (const node of body) {
|
|
23
|
+
const sym = extractSymbol(node);
|
|
24
|
+
const start = leadingCommentsStart(node);
|
|
25
|
+
const slice = text.slice(start, node.end ?? start);
|
|
26
|
+
if (!slice.trim()) continue;
|
|
27
|
+
if (slice.length <= limit) out.push({ text: slice, metadata: { source: path, kind: "code", symbol: sym } });
|
|
28
|
+
else for (const piece of windowText(slice, limit, overlap)) out.push({ text: piece, metadata: { source: path, kind: "code", symbol: sym } });
|
|
29
|
+
}
|
|
30
|
+
return out.length === 0 ? null : out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function extractSymbol(node) {
|
|
34
|
+
if (node.type === "ExportNamedDeclaration" && node.declaration) return extractSymbol(node.declaration);
|
|
35
|
+
if (node.type === "ExportDefaultDeclaration") return "default";
|
|
36
|
+
if (node.id?.name) return node.id.name;
|
|
37
|
+
if (node.declarations?.[0]?.id?.name) return node.declarations[0].id.name;
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function leadingCommentsStart(node) {
|
|
42
|
+
const lc = node.leadingComments;
|
|
43
|
+
if (Array.isArray(lc) && lc.length > 0) return lc[0].start ?? node.start ?? 0;
|
|
44
|
+
return node.start ?? 0;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function windowText(text, limit, overlap) {
|
|
48
|
+
if (text.length <= limit) return [text];
|
|
49
|
+
const step = Math.max(1, limit - overlap);
|
|
50
|
+
const out = [];
|
|
51
|
+
for (let i = 0; i < text.length; i += step) {
|
|
52
|
+
out.push(text.slice(i, i + limit));
|
|
53
|
+
if (i + limit >= text.length) break;
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
package/src/rag/chunker.js
CHANGED
|
@@ -91,13 +91,20 @@ export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT }
|
|
|
91
91
|
|
|
92
92
|
// ---------- source chunker (export-symbol granularity) ------------
|
|
93
93
|
|
|
94
|
+
import { chunkSourceAST } from "./chunker-ast.js";
|
|
95
|
+
|
|
94
96
|
const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
|
|
95
97
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
// KJC-TSK-0444 — try AST chunker first (top-level declarations as whole
|
|
99
|
+
// units, JSDoc folded in); fall back to the export-regex if @babel/parser
|
|
100
|
+
// can't parse the file (returns null).
|
|
101
|
+
export function chunkSource(text, opts = {}) {
|
|
102
|
+
const ast = chunkSourceAST(text, opts);
|
|
103
|
+
if (ast) return ast;
|
|
104
|
+
return chunkSourceRegex(text, opts);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function chunkSourceRegex(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
|
|
101
108
|
const marks = [];
|
|
102
109
|
EXPORT_RE.lastIndex = 0;
|
|
103
110
|
let m;
|
|
@@ -117,3 +124,5 @@ export function chunkSource(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT,
|
|
|
117
124
|
}
|
|
118
125
|
return out;
|
|
119
126
|
}
|
|
127
|
+
|
|
128
|
+
export { chunkSourceRegex };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// KJC-TSK-0442 — shared POST helper for cloud embedders (OpenAI, Voyage).
|
|
2
|
+
// Same envelope: Bearer auth, JSON body shaped by caller, JSON response
|
|
3
|
+
// extracted by caller, dim validated against the adapter's expected size.
|
|
4
|
+
export async function cloudEmbed(adapter, text, ErrCls, { provider, body, extract }) {
|
|
5
|
+
if (typeof text !== "string" || text.length === 0) throw new ErrCls("embed: text must be a non-empty string");
|
|
6
|
+
const ctrl = new AbortController();
|
|
7
|
+
const timer = setTimeout(() => ctrl.abort(), adapter.timeoutMs);
|
|
8
|
+
let res;
|
|
9
|
+
try {
|
|
10
|
+
res = await adapter.fetch(adapter.url, {
|
|
11
|
+
method: "POST",
|
|
12
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${adapter.apiKey}` },
|
|
13
|
+
body: JSON.stringify(body),
|
|
14
|
+
signal: ctrl.signal,
|
|
15
|
+
});
|
|
16
|
+
} catch (err) { throw new ErrCls(`${provider} embed request failed: ${err.message}`, { cause: err }); }
|
|
17
|
+
finally { clearTimeout(timer); }
|
|
18
|
+
if (!res.ok) throw new ErrCls(`${provider} embed HTTP ${res.status}`, { status: res.status });
|
|
19
|
+
const json = await res.json();
|
|
20
|
+
const arr = extract(json);
|
|
21
|
+
if (!Array.isArray(arr) || arr.length === 0) throw new ErrCls(`${provider} response missing embedding`);
|
|
22
|
+
if (arr.length !== adapter.dim) throw new ErrCls(`${provider} dim mismatch: got ${arr.length}, expected ${adapter.dim} for model ${adapter.model}`);
|
|
23
|
+
return Float32Array.from(arr);
|
|
24
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// KJC-TSK-0446 — Cohere embed-v3 adapter. Cohere returns
|
|
2
|
+
// { embeddings: { float: [[...]] } } when `embedding_types: ["float"]`
|
|
3
|
+
// is requested, or { embeddings: [[...]] } in legacy v1. We extract the
|
|
4
|
+
// modern shape with a v1 fallback.
|
|
5
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
6
|
+
|
|
7
|
+
const DEFAULTS = { url: "https://api.cohere.com/v2/embed", model: "embed-multilingual-v3.0", dim: 1024, timeoutMs: 30000 };
|
|
8
|
+
|
|
9
|
+
export class CohereEmbedderError extends Error {
|
|
10
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "CohereEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export class CohereEmbedder {
|
|
14
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_COHERE_KEY, model = process.env.KJ_COHERE_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
15
|
+
// KJC architecture invariant: scoped env var (KJ_COHERE_KEY), not the
|
|
16
|
+
// generic COHERE_API_KEY — same rule as OpenAI/Voyage adapters.
|
|
17
|
+
if (!apiKey) throw new CohereEmbedderError("Cohere embedder requires an api_key (config.rag.embedder.api_key or KJ_COHERE_KEY env)");
|
|
18
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
19
|
+
}
|
|
20
|
+
async embed(text) {
|
|
21
|
+
return cloudEmbed(this, text, CohereEmbedderError, {
|
|
22
|
+
provider: "Cohere",
|
|
23
|
+
body: { model: this.model, texts: [text], input_type: "search_document", embedding_types: ["float"] },
|
|
24
|
+
extract: (b) => b?.embeddings?.float?.[0] || b?.embeddings?.[0],
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new CohereEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
28
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// KJC-TSK-0442 / KJC-TSK-0446 — Embedder factory. Selects provider from
|
|
2
|
+
// config.rag.embedder.provider (ollama | openai | voyage | cohere | mistral;
|
|
3
|
+
// default ollama). Each provider has its own dim default; the factory wires
|
|
4
|
+
// it so the caller does not need to know.
|
|
5
|
+
import { OllamaEmbedder } from "../embedder.js";
|
|
6
|
+
import { OpenAIEmbedder } from "./openai.js";
|
|
7
|
+
import { VoyageEmbedder } from "./voyage.js";
|
|
8
|
+
import { CohereEmbedder } from "./cohere.js";
|
|
9
|
+
import { MistralEmbedder } from "./mistral.js";
|
|
10
|
+
import { ONNXEmbedder } from "./onnx.js";
|
|
11
|
+
|
|
12
|
+
const PROVIDERS = {
|
|
13
|
+
ollama: { cls: OllamaEmbedder, dim: 768 },
|
|
14
|
+
openai: { cls: OpenAIEmbedder, dim: 1536 },
|
|
15
|
+
voyage: { cls: VoyageEmbedder, dim: 1024 },
|
|
16
|
+
cohere: { cls: CohereEmbedder, dim: 1024 },
|
|
17
|
+
mistral: { cls: MistralEmbedder, dim: 1024 },
|
|
18
|
+
onnx: { cls: ONNXEmbedder, dim: 384 },
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export function makeEmbedder(config = {}) {
|
|
22
|
+
const cfg = config?.rag?.embedder || {};
|
|
23
|
+
const provider = cfg.provider || "ollama";
|
|
24
|
+
const spec = PROVIDERS[provider];
|
|
25
|
+
if (!spec) throw new Error(`Unknown embedder provider: ${provider}. Supported: ${Object.keys(PROVIDERS).join(", ")}`);
|
|
26
|
+
const dim = cfg.dim || spec.dim;
|
|
27
|
+
return new spec.cls({ url: cfg.url, apiKey: cfg.api_key, model: cfg.model, dim, timeoutMs: cfg.timeout_ms });
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export { PROVIDERS };
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// KJC-TSK-0446 — Mistral AI embeddings adapter. EU-hosted, useful for
|
|
2
|
+
// users with GDPR constraints that prefer not to send chunks to US
|
|
3
|
+
// endpoints. Single model today (`mistral-embed`, 1024 dim).
|
|
4
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
5
|
+
|
|
6
|
+
const DEFAULTS = { url: "https://api.mistral.ai/v1/embeddings", model: "mistral-embed", dim: 1024, timeoutMs: 30000 };
|
|
7
|
+
|
|
8
|
+
export class MistralEmbedderError extends Error {
|
|
9
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "MistralEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export class MistralEmbedder {
|
|
13
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_MISTRAL_KEY, model = process.env.KJ_MISTRAL_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
14
|
+
// KJC architecture invariant: scoped env var (KJ_MISTRAL_KEY).
|
|
15
|
+
if (!apiKey) throw new MistralEmbedderError("Mistral embedder requires an api_key (config.rag.embedder.api_key or KJ_MISTRAL_KEY env)");
|
|
16
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
17
|
+
}
|
|
18
|
+
async embed(text) {
|
|
19
|
+
return cloudEmbed(this, text, MistralEmbedderError, {
|
|
20
|
+
provider: "Mistral",
|
|
21
|
+
body: { model: this.model, input: [text] },
|
|
22
|
+
extract: (b) => b?.data?.[0]?.embedding,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new MistralEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
26
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// KJC-TSK-0447 — ONNX local embedder via @huggingface/transformers
|
|
2
|
+
// (formerly @xenova/transformers). Runs sentence-transformer models
|
|
3
|
+
// directly in Node — no API key, no Docker, no Ollama.
|
|
4
|
+
//
|
|
5
|
+
// First call downloads the model weights to ~/.cache/huggingface/ (~80 MB
|
|
6
|
+
// for the default MiniLM). Subsequent calls reuse the cache, so the steady
|
|
7
|
+
// state is fully offline.
|
|
8
|
+
//
|
|
9
|
+
// Default model: `Xenova/all-MiniLM-L6-v2` (384 dim, ~80 MB, fast).
|
|
10
|
+
// Higher-quality alternative: `Xenova/jina-embeddings-v2-base-en`
|
|
11
|
+
// (768 dim, ~260 MB, slower but better for retrieval).
|
|
12
|
+
|
|
13
|
+
const DEFAULTS = { model: "Xenova/all-MiniLM-L6-v2", dim: 384, pooling: "mean", normalize: true };
|
|
14
|
+
|
|
15
|
+
export class ONNXEmbedderError extends Error {
|
|
16
|
+
constructor(message, { cause } = {}) { super(message); this.name = "ONNXEmbedderError"; if (cause) this.cause = cause; }
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function loadTransformers() {
|
|
20
|
+
// Prefer the modern @huggingface/transformers (post-2024 rebrand);
|
|
21
|
+
// fall back to @xenova/transformers for users still on the legacy package.
|
|
22
|
+
// Both are optional peer deps — Karajan does not ship them by default to
|
|
23
|
+
// keep the install size small (~500 MB with WASM + weights). The user
|
|
24
|
+
// installs whichever they prefer when they opt into provider: onnx.
|
|
25
|
+
/* eslint-disable import-x/no-unresolved */
|
|
26
|
+
try { return await import("@huggingface/transformers"); }
|
|
27
|
+
catch (e1) {
|
|
28
|
+
try { return await import("@xenova/transformers"); }
|
|
29
|
+
catch (e2) {
|
|
30
|
+
throw new ONNXEmbedderError(
|
|
31
|
+
"ONNX embedder requires @huggingface/transformers (or legacy @xenova/transformers). Install with `npm install @huggingface/transformers`.",
|
|
32
|
+
{ cause: e2 },
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/* eslint-enable import-x/no-unresolved */
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export class ONNXEmbedder {
|
|
40
|
+
constructor({ model = process.env.KJ_ONNX_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, pooling = DEFAULTS.pooling, normalize = DEFAULTS.normalize } = {}) {
|
|
41
|
+
Object.assign(this, { model, dim, pooling, normalize, _pipe: null });
|
|
42
|
+
}
|
|
43
|
+
async _ensurePipeline() {
|
|
44
|
+
if (this._pipe) return this._pipe;
|
|
45
|
+
const transformers = await loadTransformers();
|
|
46
|
+
this._pipe = await transformers.pipeline("feature-extraction", this.model);
|
|
47
|
+
return this._pipe;
|
|
48
|
+
}
|
|
49
|
+
async embed(text) {
|
|
50
|
+
if (typeof text !== "string" || text.length === 0) throw new ONNXEmbedderError("embed: text must be a non-empty string");
|
|
51
|
+
const pipe = await this._ensurePipeline();
|
|
52
|
+
const out = await pipe(text, { pooling: this.pooling, normalize: this.normalize });
|
|
53
|
+
const arr = Array.from(out.data || []);
|
|
54
|
+
if (arr.length !== this.dim) throw new ONNXEmbedderError(`ONNX dim mismatch: got ${arr.length}, expected ${this.dim} for model ${this.model}`);
|
|
55
|
+
return Float32Array.from(arr);
|
|
56
|
+
}
|
|
57
|
+
async embedBatch(texts) {
|
|
58
|
+
if (!Array.isArray(texts)) throw new ONNXEmbedderError("embedBatch: texts must be an array");
|
|
59
|
+
const out = [];
|
|
60
|
+
for (const t of texts) out.push(await this.embed(t));
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// KJC-TSK-0442 — OpenAI text-embeddings adapter.
|
|
2
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
3
|
+
|
|
4
|
+
const DEFAULTS = { url: "https://api.openai.com/v1/embeddings", model: "text-embedding-3-small", dim: 1536, timeoutMs: 30000 };
|
|
5
|
+
|
|
6
|
+
export class OpenAIEmbedderError extends Error {
|
|
7
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "OpenAIEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export class OpenAIEmbedder {
|
|
11
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_OPENAI_KEY, model = process.env.KJ_OPENAI_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
12
|
+
// KJC architecture invariant: Karajan does not read provider API keys
|
|
13
|
+
// (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.) — those belong to the CLI
|
|
14
|
+
// agents Karajan spawns. RAG embedders are the one exception: they
|
|
15
|
+
// call the OpenAI endpoint directly, but use a Karajan-scoped env var
|
|
16
|
+
// (KJ_OPENAI_KEY) so the invariant stays clean.
|
|
17
|
+
if (!apiKey) throw new OpenAIEmbedderError("OpenAI embedder requires an api_key (config.rag.embedder.api_key or KJ_OPENAI_KEY env)");
|
|
18
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
19
|
+
}
|
|
20
|
+
async embed(text) { return cloudEmbed(this, text, OpenAIEmbedderError, { provider: "OpenAI", body: { model: this.model, input: text }, extract: (b) => b?.data?.[0]?.embedding }); }
|
|
21
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new OpenAIEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
22
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// KJC-TSK-0442 — Voyage AI embeddings adapter.
|
|
2
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
3
|
+
|
|
4
|
+
const DEFAULTS = { url: "https://api.voyageai.com/v1/embeddings", model: "voyage-code-3", dim: 1024, timeoutMs: 30000 };
|
|
5
|
+
|
|
6
|
+
export class VoyageEmbedderError extends Error {
|
|
7
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "VoyageEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export class VoyageEmbedder {
|
|
11
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_VOYAGE_KEY, model = process.env.KJ_VOYAGE_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
12
|
+
// Same invariant as OpenAIEmbedder: Karajan-scoped env var.
|
|
13
|
+
if (!apiKey) throw new VoyageEmbedderError("Voyage embedder requires an api_key (config.rag.embedder.api_key or KJ_VOYAGE_KEY env)");
|
|
14
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
15
|
+
}
|
|
16
|
+
async embed(text) { return cloudEmbed(this, text, VoyageEmbedderError, { provider: "Voyage", body: { model: this.model, input: [text], input_type: "document" }, extract: (b) => b?.data?.[0]?.embedding }); }
|
|
17
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new VoyageEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
18
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// KJC-TSK-0449 — Cross-encoder rerank stage. Re-scores the top-N hits
|
|
2
|
+
// from the hybrid retriever using a (query, passage) cross-encoder.
|
|
3
|
+
//
|
|
4
|
+
// Cross-encoders are slower than the bi-encoder embedders (they jointly
|
|
5
|
+
// encode query + passage instead of caching the passage), but they are
|
|
6
|
+
// substantially more precise for the final top-K ranking. Karajan calls
|
|
7
|
+
// the reranker only on the post-fusion candidates (≤topK*2), so the
|
|
8
|
+
// added latency is bounded.
|
|
9
|
+
//
|
|
10
|
+
// Model is loaded dynamically via @huggingface/transformers (same dep as
|
|
11
|
+
// the ONNX embedder, KJC-TSK-0447). Default: `Xenova/ms-marco-MiniLM-L-6-v2`,
|
|
12
|
+
// the de-facto standard sentence-transformers reranker.
|
|
13
|
+
|
|
14
|
+
const DEFAULT_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
15
|
+
|
|
16
|
+
export class RerankError extends Error {
|
|
17
|
+
constructor(message, { cause } = {}) { super(message); this.name = "RerankError"; if (cause) this.cause = cause; }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function loadTransformers() {
|
|
21
|
+
/* eslint-disable import-x/no-unresolved */
|
|
22
|
+
try { return await import("@huggingface/transformers"); }
|
|
23
|
+
catch (e1) {
|
|
24
|
+
try { return await import("@xenova/transformers"); }
|
|
25
|
+
catch (e2) {
|
|
26
|
+
throw new RerankError(
|
|
27
|
+
"rerank requires @huggingface/transformers (or legacy @xenova/transformers). Install with `npm install @huggingface/transformers`.",
|
|
28
|
+
{ cause: e2 },
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/* eslint-enable import-x/no-unresolved */
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Lazy singleton — first call downloads weights (~80 MB), every subsequent
|
|
37
|
+
* call reuses the same pipeline instance.
|
|
38
|
+
*/
|
|
39
|
+
let _pipelinePromise = null;
|
|
40
|
+
function getPipeline(modelName) {
|
|
41
|
+
if (!_pipelinePromise) _pipelinePromise = (async () => {
|
|
42
|
+
const t = await loadTransformers();
|
|
43
|
+
// text-classification pipeline returns the cross-encoder relevance score
|
|
44
|
+
// for a `[query, passage]` text pair when applied to a CE model.
|
|
45
|
+
return t.pipeline("text-classification", modelName);
|
|
46
|
+
})();
|
|
47
|
+
return _pipelinePromise;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Reset for tests. */
|
|
51
|
+
export function _resetPipeline() { _pipelinePromise = null; }
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Rerank `hits` against `queryText` using the cross-encoder. Returns a NEW
|
|
55
|
+
* array sorted by descending rerank score (best first), `score` mutated
|
|
56
|
+
* so the existing downstream code (kindBoost, slicing) keeps working.
|
|
57
|
+
*
|
|
58
|
+
* Each input hit is treated as a [query, hit.text] pair. Truncates the
|
|
59
|
+
* passage to `maxChars` to avoid blowing past the CE model's context
|
|
60
|
+
* window — most CE models handle 512 tokens, ~2000 chars is safe.
|
|
61
|
+
*
|
|
62
|
+
* If `pipelineFn` is injected (tests), it bypasses model loading.
|
|
63
|
+
*/
|
|
64
|
+
export async function rerank(queryText, hits, { model = process.env.KJ_RERANK_MODEL || DEFAULT_MODEL, maxChars = 2000, pipelineFn = null } = {}) {
|
|
65
|
+
if (!queryText || typeof queryText !== "string") throw new RerankError("rerank: queryText must be a non-empty string");
|
|
66
|
+
if (!Array.isArray(hits)) throw new RerankError("rerank: hits must be an array");
|
|
67
|
+
if (hits.length === 0) return hits;
|
|
68
|
+
const pipe = pipelineFn || await getPipeline(model);
|
|
69
|
+
const pairs = hits.map((h) => ({ text: queryText, text_pair: String(h.text || "").slice(0, maxChars) }));
|
|
70
|
+
const results = await pipe(pairs);
|
|
71
|
+
return hits
|
|
72
|
+
.map((h, i) => ({ ...h, _rerank: Number(results[i]?.score) || 0, score: -Number(results[i]?.score) || 0 }))
|
|
73
|
+
.sort((a, b) => a.score - b.score);
|
|
74
|
+
}
|
package/src/rag/retriever.js
CHANGED
|
@@ -1,77 +1,75 @@
|
|
|
1
|
-
// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
|
|
2
|
+
import { searchSimilar, searchBM25 } from "./vec-store.js";
|
|
3
|
+
import { parseWhere, buildWhereSql } from "./where-parser.js";
|
|
4
|
+
import { rerank } from "./rerank.js";
|
|
4
5
|
|
|
5
|
-
import { searchSimilar } from "./vec-store.js";
|
|
6
|
-
|
|
7
|
-
// Kind boosts for rerank: a plan chunk is more "intentional" than a
|
|
8
|
-
// raw code chunk, an onboarding brief sits between them. Tunable via
|
|
9
|
-
// the third argument; the defaults are conservative — they break ties
|
|
10
|
-
// between equidistant chunks but won't reorder by big distance gaps.
|
|
11
6
|
const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
|
|
12
7
|
|
|
13
|
-
// KJC-TSK-0440 — asymmetric source/test boost.
|
|
14
|
-
// systematic bias: queries in natural language ("how does X work") return
|
|
15
|
-
// tests/X.test.js ahead of src/X.js because tests carry more descriptive
|
|
16
|
-
// prose. When the query itself does NOT mention test-flavoured terms,
|
|
17
|
-
// give code chunks +0.05 — but only chunks whose source path is NOT a
|
|
18
|
-
// test file. Tests keep the baseline boost.
|
|
8
|
+
// KJC-TSK-0440 — asymmetric source/test boost.
|
|
19
9
|
const SOURCE_BOOST_NON_TEST = 0.05;
|
|
20
10
|
const TEST_TERMS_RE = /\b(test|tests|spec|specs|expect|describe|it\(|jest|vitest|mocha)\b/i;
|
|
21
11
|
const TEST_PATH_RE = /[\\/](tests?|specs?|__tests__)[\\/]|\.test\.[jt]sx?$|\.spec\.[jt]sx?$/i;
|
|
22
12
|
|
|
23
|
-
function shouldBoostSources(queryText) {
|
|
24
|
-
|
|
25
|
-
}
|
|
13
|
+
function shouldBoostSources(queryText) { return !TEST_TERMS_RE.test(queryText); }
|
|
14
|
+
function isTestPath(source) { return TEST_PATH_RE.test(source || ""); }
|
|
26
15
|
|
|
27
|
-
|
|
28
|
-
|
|
16
|
+
// KJC-TSK-0443 — fuse semantic + keyword hits into a unified candidate set.
|
|
17
|
+
// For 'semantic'/'keyword' modes we surface that side. For 'hybrid' (default)
|
|
18
|
+
// we min-max normalise both scores to [0,1] (lower=better) and linear-combine
|
|
19
|
+
// via alpha * semantic + (1-alpha) * keyword. Result written back to
|
|
20
|
+
// `distance` so the kind+source boost pipeline keeps working unchanged.
|
|
21
|
+
function fuseHits(semantic, keyword, alpha, mode) {
|
|
22
|
+
if (mode === "semantic") return semantic;
|
|
23
|
+
if (mode === "keyword") return keyword.map((h) => ({ ...h, distance: h.bm25 }));
|
|
24
|
+
const byId = new Map();
|
|
25
|
+
for (const h of semantic) byId.set(h.id, { ...h, _sem: h.distance });
|
|
26
|
+
for (const h of keyword) {
|
|
27
|
+
const prev = byId.get(h.id);
|
|
28
|
+
if (prev) prev._kw = h.bm25;
|
|
29
|
+
else byId.set(h.id, { ...h, _kw: h.bm25, distance: h.bm25 });
|
|
30
|
+
}
|
|
31
|
+
const list = [...byId.values()];
|
|
32
|
+
const norm = (vals) => {
|
|
33
|
+
const xs = vals.filter((v) => Number.isFinite(v));
|
|
34
|
+
if (xs.length === 0) return () => 0.5;
|
|
35
|
+
const min = Math.min(...xs); const max = Math.max(...xs); const span = max - min || 1;
|
|
36
|
+
return (v) => Number.isFinite(v) ? (v - min) / span : 1;
|
|
37
|
+
};
|
|
38
|
+
const nSem = norm(list.map((h) => h._sem));
|
|
39
|
+
const nKw = norm(list.map((h) => h._kw));
|
|
40
|
+
for (const h of list) h.distance = alpha * nSem(h._sem) + (1 - alpha) * nKw(h._kw);
|
|
41
|
+
return list;
|
|
29
42
|
}
|
|
30
43
|
|
|
31
|
-
|
|
32
|
-
* Run a natural-language query against the indexed RAG corpus. Returns
|
|
33
|
-
* the top-K chunks ranked by adjusted distance (lower = closer).
|
|
34
|
-
*
|
|
35
|
-
* query(db, embedder, "how did I handle auth in module X?",
|
|
36
|
-
* { topK: 5, scope: 'plans' })
|
|
37
|
-
*
|
|
38
|
-
* @param {Database} db — opened via openVecStore()
|
|
39
|
-
* @param {OllamaEmbedder} embedder
|
|
40
|
-
* @param {string} text — the query
|
|
41
|
-
* @param {Object} [opts]
|
|
42
|
-
* @param {number} [opts.topK=5]
|
|
43
|
-
* @param {'plans'|'code'|'onboarding'|'all'} [opts.scope='all']
|
|
44
|
-
* @param {Object} [opts.kindBoost]
|
|
45
|
-
*/
|
|
46
|
-
export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null } = {}) {
|
|
44
|
+
export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null } = {}) {
|
|
47
45
|
if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
|
|
48
|
-
const queryEmbedding = await embedder.embed(text);
|
|
49
|
-
// Over-fetch (topK * 2) so the rerank step has room to reshuffle
|
|
50
|
-
// without dropping a relevant chunk past the cut. Clamped at 50 to
|
|
51
|
-
// avoid loading the entire store on tiny topK calls.
|
|
52
46
|
const fetchK = Math.min(50, topK * 2);
|
|
53
47
|
const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
|
|
54
|
-
// KJC-TSK-
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
const
|
|
48
|
+
// KJC-TSK-0448 — metadata filter. `where` is a string like "symbol=Foo AND
|
|
49
|
+
// hu_id=HU-003"; we parse once and reuse the SQL fragment across both
|
|
50
|
+
// semantic and keyword searches.
|
|
51
|
+
const parsed = parseWhere(where);
|
|
52
|
+
if (!parsed.ok) throw new Error(`query: ${parsed.error}`);
|
|
53
|
+
const { sql: whereSql, params: whereParams } = buildWhereSql(parsed.clauses);
|
|
54
|
+
const opts = { kind: scopeKind, project, whereSql, whereParams };
|
|
55
|
+
const wantSemantic = mode !== "keyword";
|
|
56
|
+
const wantKeyword = mode !== "semantic";
|
|
57
|
+
const semanticHits = wantSemantic ? searchSimilar(db, await embedder.embed(text), fetchK, opts) : [];
|
|
58
|
+
const keywordHits = wantKeyword ? searchBM25(db, text, fetchK, opts) : [];
|
|
59
|
+
const raw = fuseHits(semanticHits, keywordHits, alpha, mode);
|
|
58
60
|
if (raw.length === 0) return [];
|
|
59
|
-
// Adjust each chunk's distance by its kind boost. The vec store returns
|
|
60
|
-
// cosine distance (smaller = closer); subtracting the boost makes
|
|
61
|
-
// higher-priority kinds appear "closer" without altering the source
|
|
62
|
-
// truth in the DB.
|
|
63
|
-
// KJC-TSK-0440 — apply the source-vs-test asymmetric boost only when
|
|
64
|
-
// the query itself is not test-flavoured. Test-flavoured queries keep
|
|
65
|
-
// the symmetric baseline so e.g. "vitest mock setup" still surfaces
|
|
66
|
-
// test files.
|
|
67
61
|
const boostSources = shouldBoostSources(text);
|
|
68
62
|
const reranked = raw.map((r) => {
|
|
69
63
|
const kindB = kindBoost[r.kind] || 0;
|
|
70
64
|
const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
|
|
71
65
|
return { ...r, score: r.distance - kindB - sourceB };
|
|
72
66
|
}).sort((a, b) => a.score - b.score).slice(0, topK);
|
|
67
|
+
// KJC-TSK-0449 — optional cross-encoder rerank. When `rerankOpts` is set,
|
|
68
|
+
// we re-score the topK survivors with a (query, passage) cross-encoder;
|
|
69
|
+
// the kind+source boost has already been applied, so the rerank acts as
|
|
70
|
+
// a finer-grained quality lever on top.
|
|
71
|
+
if (rerankOpts) return await rerank(text, reranked, rerankOpts);
|
|
73
72
|
return reranked;
|
|
74
73
|
}
|
|
75
74
|
|
|
76
|
-
|
|
77
|
-
export { shouldBoostSources, isTestPath };
|
|
75
|
+
export { shouldBoostSources, isTestPath, fuseHits };
|
package/src/rag/vec-store.js
CHANGED
|
@@ -54,6 +54,29 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
|
|
|
54
54
|
db.exec("CREATE INDEX IF NOT EXISTS chunks_by_project ON chunks(project_slug);");
|
|
55
55
|
}
|
|
56
56
|
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
|
|
57
|
+
// KJC-TSK-0443 — FTS5 keyword index for BM25 hybrid scoring. content='chunks'
|
|
58
|
+
// makes it a contentless mirror linked by id (no double storage); triggers
|
|
59
|
+
// sync the FTS5 rows on insert/update/delete. Rebuilds idempotently when
|
|
60
|
+
// the FTS5 table exists but is empty (e.g. created post-migration on a
|
|
61
|
+
// DB with pre-v2.28 chunks).
|
|
62
|
+
db.exec(`
|
|
63
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, content='chunks', content_rowid='id');
|
|
64
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
|
|
65
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
|
|
66
|
+
END;
|
|
67
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
68
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
69
|
+
END;
|
|
70
|
+
CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
|
|
71
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
72
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
|
|
73
|
+
END;
|
|
74
|
+
`);
|
|
75
|
+
const ftsCount = db.prepare("SELECT COUNT(*) AS n FROM chunks_fts").get().n;
|
|
76
|
+
const chunksCount = db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
77
|
+
if (ftsCount === 0 && chunksCount > 0) {
|
|
78
|
+
db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');");
|
|
79
|
+
}
|
|
57
80
|
return db;
|
|
58
81
|
}
|
|
59
82
|
|
|
@@ -83,7 +106,7 @@ export function insertChunk(db, { source, kind, text, metadata = null, embedding
|
|
|
83
106
|
* text, metadata, distance }]` for the topK best matches; lower
|
|
84
107
|
* distance = closer. Returns `[]` when the store is empty.
|
|
85
108
|
*/
|
|
86
|
-
export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null } = {}) {
|
|
109
|
+
export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null, whereSql = "", whereParams = [] } = {}) {
|
|
87
110
|
const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
|
|
88
111
|
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
89
112
|
const projectClause = project ? "AND c.project_slug = ?" : "";
|
|
@@ -91,12 +114,13 @@ export function searchSimilar(db, embedding, topK = 5, { kind = null, project =
|
|
|
91
114
|
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, v.distance
|
|
92
115
|
FROM vec_chunks v
|
|
93
116
|
JOIN chunks c ON c.id = v.rowid
|
|
94
|
-
WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}
|
|
117
|
+
WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}${whereSql}
|
|
95
118
|
ORDER BY v.distance
|
|
96
119
|
`;
|
|
97
120
|
const params = [Buffer.from(buf.buffer), topK];
|
|
98
121
|
if (kind) params.push(kind);
|
|
99
122
|
if (project) params.push(project);
|
|
123
|
+
if (whereParams.length) params.push(...whereParams);
|
|
100
124
|
const rows = db.prepare(sql).all(...params);
|
|
101
125
|
return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
|
|
102
126
|
}
|
|
@@ -133,3 +157,31 @@ export function countChunks(db, { kind = null } = {}) {
|
|
|
133
157
|
if (kind) return db.prepare("SELECT COUNT(*) AS n FROM chunks WHERE kind = ?").get(kind).n;
|
|
134
158
|
return db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
135
159
|
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* KJC-TSK-0443 — BM25 keyword search over chunks_fts. Returns
|
|
163
|
+
* [{ id, source, kind, text, metadata, project_slug, bm25 }] ordered by
|
|
164
|
+
* relevance (lower bm25 = better in SQLite FTS5). `query` is sanitized for
|
|
165
|
+
* the FTS5 grammar: special chars are stripped, words split on whitespace.
|
|
166
|
+
*/
|
|
167
|
+
export function searchBM25(db, queryText, topK = 10, { kind = null, project = null, whereSql = "", whereParams = [] } = {}) {
|
|
168
|
+
const cleaned = String(queryText || "").replace(/["'()]/g, " ").split(/\s+/).filter(Boolean).join(" OR ");
|
|
169
|
+
if (!cleaned) return [];
|
|
170
|
+
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
171
|
+
const projectClause = project ? "AND c.project_slug = ?" : "";
|
|
172
|
+
const sql = `
|
|
173
|
+
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, bm25(chunks_fts) AS bm25
|
|
174
|
+
FROM chunks_fts JOIN chunks c ON c.id = chunks_fts.rowid
|
|
175
|
+
WHERE chunks_fts MATCH ? ${kindClause} ${projectClause}${whereSql}
|
|
176
|
+
ORDER BY bm25 LIMIT ?
|
|
177
|
+
`;
|
|
178
|
+
const params = [cleaned];
|
|
179
|
+
if (kind) params.push(kind);
|
|
180
|
+
if (project) params.push(project);
|
|
181
|
+
if (whereParams.length) params.push(...whereParams);
|
|
182
|
+
params.push(topK);
|
|
183
|
+
try {
|
|
184
|
+
const rows = db.prepare(sql).all(...params);
|
|
185
|
+
return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
|
|
186
|
+
} catch { return []; }
|
|
187
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// KJC-TSK-0441 (RAG v2.28.0 PR1) — chokidar watcher. Live re-index of plans
|
|
2
|
+
// + onboarding + (opt) sources, debounced. PID file arbitrates a single daemon.
|
|
3
|
+
import { writeFileSync, readFileSync, existsSync, unlinkSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import chokidar from "chokidar";
|
|
6
|
+
import { openVecStore, deleteChunksBySource, projectSlug } from "./vec-store.js";
|
|
7
|
+
import { makeEmbedder } from "./embedders/factory.js";
|
|
8
|
+
import { indexFile } from "./indexer.js";
|
|
9
|
+
import { getKarajanHome } from "../utils/paths.js";
|
|
10
|
+
|
|
11
|
+
const PIDFILE = () => join(getKarajanHome(), "watcher.pid");
|
|
12
|
+
const DEFAULT_DEBOUNCE_MS = 1000;
|
|
13
|
+
const SKIP_SEGMENTS = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
|
|
14
|
+
const SOURCE_RE = /\.(js|mjs|cjs|ts|tsx|jsx)$/;
|
|
15
|
+
|
|
16
|
+
function isWatchable(path) {
|
|
17
|
+
if (path.includes("/.karajan/onboarding/")) return /\.md$/.test(path);
|
|
18
|
+
if (path.includes("/.karajan/plans/")) return /\.json$/.test(path);
|
|
19
|
+
if (path.split("/").some((seg) => SKIP_SEGMENTS.has(seg))) return false;
|
|
20
|
+
return SOURCE_RE.test(path);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function startWatcher({ projectDir, config, logger = console, debounceMs = DEFAULT_DEBOUNCE_MS, withSources = false } = {}) {
|
|
24
|
+
if (!projectDir) throw new Error("startWatcher: projectDir is required");
|
|
25
|
+
const slug = projectSlug(projectDir);
|
|
26
|
+
const dim = config?.rag?.embedder?.dim || 768;
|
|
27
|
+
const db = openVecStore({ dim });
|
|
28
|
+
const embedder = makeEmbedder(config);
|
|
29
|
+
const paths = [join(getKarajanHome(), "onboarding"), join(getKarajanHome(), "plans")];
|
|
30
|
+
if (withSources) paths.push(projectDir);
|
|
31
|
+
const watcher = chokidar.watch(paths, { ignoreInitial: true, persistent: true });
|
|
32
|
+
const pending = new Map();
|
|
33
|
+
const flush = async (p) => {
|
|
34
|
+
pending.delete(p);
|
|
35
|
+
if (!existsSync(p)) { deleteChunksBySource(db, p); logger.info?.(`[rag-watcher] removed → ${p}`); return; }
|
|
36
|
+
if (!isWatchable(p)) return;
|
|
37
|
+
try { const r = await indexFile(p, { db, embedder, logger, project: slug }); logger.info?.(`[rag-watcher] reindexed ${p} → ${r.indexed} chunk(s)`); }
|
|
38
|
+
catch (err) { logger.warn?.(`[rag-watcher] reindex failed for ${p}: ${err.message}`); }
|
|
39
|
+
};
|
|
40
|
+
const schedule = (p) => { if (pending.has(p)) clearTimeout(pending.get(p)); pending.set(p, setTimeout(() => flush(p), debounceMs)); };
|
|
41
|
+
watcher.on("add", schedule).on("change", schedule).on("unlink", schedule).on("error", (err) => logger.warn?.(`[rag-watcher] ${err.message}`));
|
|
42
|
+
return async function stop() {
|
|
43
|
+
for (const t of pending.values()) clearTimeout(t);
|
|
44
|
+
pending.clear();
|
|
45
|
+
await watcher.close();
|
|
46
|
+
try { db.close(); } catch { /* already closed */ }
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function writePidFile(pid = process.pid) { writeFileSync(PIDFILE(), String(pid), "utf8"); }
|
|
51
|
+
export function clearPidFile() { if (existsSync(PIDFILE())) unlinkSync(PIDFILE()); }
|
|
52
|
+
export function readPidFile() {
|
|
53
|
+
if (!existsSync(PIDFILE())) return null;
|
|
54
|
+
const pid = Number(readFileSync(PIDFILE(), "utf8").trim());
|
|
55
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
56
|
+
}
|
|
57
|
+
export function isPidAlive(pid) {
|
|
58
|
+
if (!pid) return false;
|
|
59
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
60
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// KJC-TSK-0448 — minimal --where filter parser for metadata-aware RAG
|
|
2
|
+
// queries. Grammar (intentionally small):
|
|
3
|
+
//
|
|
4
|
+
// clause := pair ( "AND" pair )*
|
|
5
|
+
// pair := IDENT "=" VALUE
|
|
6
|
+
// IDENT := [A-Za-z_][A-Za-z0-9_.-]*
|
|
7
|
+
// VALUE := unquoted token | "quoted string"
|
|
8
|
+
//
|
|
9
|
+
// Examples:
|
|
10
|
+
// symbol=loadConfig
|
|
11
|
+
// hu_id=HU-003 AND kind=plan
|
|
12
|
+
// "headingPath.0"=Architecture
|
|
13
|
+
//
|
|
14
|
+
// Returns { ok: true, clauses: [["symbol","loadConfig"], ...] } or
|
|
15
|
+
// { ok: false, error: "...explanation..." }. The retriever turns each
|
|
16
|
+
// clause into a `json_extract(c.metadata, '$.<key>') = ?` SQL fragment;
|
|
17
|
+
// the `kind` key is special-cased to filter the column directly.
|
|
18
|
+
|
|
19
|
+
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_.-]*$/;
|
|
20
|
+
|
|
21
|
+
export function parseWhere(input) {
|
|
22
|
+
if (input == null) return { ok: true, clauses: [] };
|
|
23
|
+
if (typeof input !== "string") return { ok: false, error: "where: input must be a string" };
|
|
24
|
+
const s = input.trim();
|
|
25
|
+
if (!s) return { ok: true, clauses: [] };
|
|
26
|
+
const parts = s.split(/\s+AND\s+/i);
|
|
27
|
+
const clauses = [];
|
|
28
|
+
for (const p of parts) {
|
|
29
|
+
const eq = p.indexOf("=");
|
|
30
|
+
if (eq <= 0) return { ok: false, error: `where: bad pair "${p}" — expected key=value` };
|
|
31
|
+
const key = p.slice(0, eq).trim().replace(/^"|"$/g, "");
|
|
32
|
+
let val = p.slice(eq + 1).trim();
|
|
33
|
+
if (!IDENT_RE.test(key)) return { ok: false, error: `where: invalid key "${key}"` };
|
|
34
|
+
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) val = val.slice(1, -1);
|
|
35
|
+
if (val.length === 0) return { ok: false, error: `where: empty value for key "${key}"` };
|
|
36
|
+
clauses.push([key, val]);
|
|
37
|
+
}
|
|
38
|
+
return { ok: true, clauses };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Build a `(sqlFragment, params)` pair to append to a chunks-table query.
|
|
43
|
+
* `kind` filters the column directly; everything else goes through json_extract.
|
|
44
|
+
*/
|
|
45
|
+
export function buildWhereSql(clauses) {
|
|
46
|
+
if (!clauses?.length) return { sql: "", params: [] };
|
|
47
|
+
const fragments = [];
|
|
48
|
+
const params = [];
|
|
49
|
+
for (const [k, v] of clauses) {
|
|
50
|
+
if (k === "kind") { fragments.push("c.kind = ?"); params.push(v); }
|
|
51
|
+
else { fragments.push(`json_extract(c.metadata, '$.${k}') = ?`); params.push(v); }
|
|
52
|
+
}
|
|
53
|
+
return { sql: ` AND ${fragments.join(" AND ")}`, params };
|
|
54
|
+
}
|
|
@@ -39,23 +39,21 @@ export function parseCooldown(message) {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
// 5. Claude Code session / weekly limit: "resets 10:10pm" / "resets 4am"
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
const ampmMatch = /reset(?:s|ting)?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*([ap]m)\b/i.exec(
|
|
48
|
-
message
|
|
49
|
-
);
|
|
42
|
+
// 5. Claude Code session / weekly limit: "resets 10:10pm (Europe/Madrid)" / "resets 4am"
|
|
43
|
+
// KJC-BUG-0064: prefer the parenthesised IANA TZ when present so the
|
|
44
|
+
// same stderr resolves to the same instant regardless of where kj runs
|
|
45
|
+
// (CI in TZ=UTC was misclassifying Madrid 10:10pm as already past).
|
|
46
|
+
// No TZ literal → fall back to the host's local TZ.
|
|
47
|
+
const ampmMatch = /reset(?:s|ting)?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*([ap]m)\b/i.exec(message);
|
|
50
48
|
if (ampmMatch) {
|
|
51
49
|
let hour = Number.parseInt(ampmMatch[1], 10) % 12;
|
|
52
50
|
if (/pm/i.test(ampmMatch[3])) hour += 12;
|
|
53
51
|
const minute = ampmMatch[2] ? Number.parseInt(ampmMatch[2], 10) : 0;
|
|
54
52
|
if (hour < 24 && minute < 60) {
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
return { cooldownUntil: target.toISOString(), cooldownMs: target.getTime() - Date.now() };
|
|
53
|
+
const tzMatch = /\(([A-Za-z]+\/[A-Za-z_/+\-0-9]+)\)/.exec(message);
|
|
54
|
+
const tz = tzMatch ? tzMatch[1] : null;
|
|
55
|
+
const target = tz ? nextOccurrenceInTZ(hour, minute, tz) : nextOccurrenceLocal(hour, minute);
|
|
56
|
+
if (target) return { cooldownUntil: target.toISOString(), cooldownMs: Math.max(0, target.getTime() - Date.now()) };
|
|
59
57
|
}
|
|
60
58
|
}
|
|
61
59
|
|
|
@@ -84,6 +82,50 @@ export function parseCooldown(message) {
|
|
|
84
82
|
return { cooldownUntil: null, cooldownMs: null };
|
|
85
83
|
}
|
|
86
84
|
|
|
85
|
+
// KJC-BUG-0064 — TZ-aware "next occurrence of hour:minute" resolver.
|
|
86
|
+
// Walks today + tomorrow in the given IANA TZ; returns the first instant
|
|
87
|
+
// > now. Uses Intl.DateTimeFormat as the source of truth so it works on
|
|
88
|
+
// any host TZ (CI runs UTC, dev runs Europe/Madrid, etc.).
|
|
89
|
+
function nextOccurrenceInTZ(hour, minute, tz) {
|
|
90
|
+
try {
|
|
91
|
+
const now = new Date();
|
|
92
|
+
for (let dayOffset = 0; dayOffset <= 1; dayOffset++) {
|
|
93
|
+
const probe = new Date(now.getTime() + dayOffset * 86400000);
|
|
94
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
95
|
+
timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit",
|
|
96
|
+
}).formatToParts(probe).reduce((acc, p) => (p.type !== "literal" ? { ...acc, [p.type]: p.value } : acc), {});
|
|
97
|
+
if (!parts.year) return null;
|
|
98
|
+
const candidate = zonedTimeToUTC(Number(parts.year), Number(parts.month), Number(parts.day), hour, minute, tz);
|
|
99
|
+
if (candidate && candidate.getTime() > now.getTime()) return candidate;
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
} catch { return null; }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Converts a wall-clock date + time in `tz` to a UTC Date.
|
|
106
|
+
// Strategy: interpret year/month/day/hour/minute as naive UTC, see what
|
|
107
|
+
// wall-clock that maps to in `tz`, derive the offset (tz - UTC at that
|
|
108
|
+
// moment) and subtract it from the naive value to land on the true UTC.
|
|
109
|
+
// One step suffices except at DST transitions (within ±1h, acceptable
|
|
110
|
+
// for cooldown timing).
|
|
111
|
+
function zonedTimeToUTC(year, month, day, hour, minute, tz) {
|
|
112
|
+
const naiveUTC = Date.UTC(year, month - 1, day, hour, minute);
|
|
113
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
114
|
+
timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit",
|
|
115
|
+
hour: "2-digit", minute: "2-digit", hour12: false,
|
|
116
|
+
}).formatToParts(new Date(naiveUTC)).reduce((acc, p) => (p.type !== "literal" ? { ...acc, [p.type]: p.value } : acc), {});
|
|
117
|
+
const wallInTZ = Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day), Number(parts.hour) % 24, Number(parts.minute));
|
|
118
|
+
const tzOffset = wallInTZ - naiveUTC; // positive when tz is ahead of UTC
|
|
119
|
+
return new Date(naiveUTC - tzOffset);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function nextOccurrenceLocal(hour, minute) {
|
|
123
|
+
const target = new Date();
|
|
124
|
+
target.setHours(hour, minute, 0, 0);
|
|
125
|
+
if (target.getTime() <= Date.now()) target.setDate(target.getDate() + 1);
|
|
126
|
+
return target;
|
|
127
|
+
}
|
|
128
|
+
|
|
87
129
|
const RATE_LIMIT_PATTERNS = [
|
|
88
130
|
// Claude CLI
|
|
89
131
|
{ pattern: /usage limit/i, agent: "claude" },
|