karajan-code 2.28.0 → 2.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/package.json +1 -1
- package/packages/hu-board/public/app.js +70 -5
- package/packages/hu-board/public/index.html +1 -0
- package/packages/hu-board/public/rag.html +21 -0
- package/packages/hu-board/public/rag.js +39 -0
- package/packages/hu-board/public/styles.css +14 -0
- package/packages/hu-board/src/config-yaml.js +190 -13
- package/packages/hu-board/src/routes/api.js +21 -9
- package/packages/hu-board/src/routes/rag.js +36 -0
- package/packages/hu-board/src/server.js +2 -0
- package/scripts/esbuild-sea.config.mjs +8 -1
- package/src/cli/register-meta.js +3 -0
- package/src/commands/rag.js +3 -1
- package/src/rag/embedders/cohere.js +28 -0
- package/src/rag/embedders/factory.js +15 -5
- package/src/rag/embedders/mistral.js +26 -0
- package/src/rag/embedders/onnx.js +63 -0
- package/src/rag/rerank.js +74 -0
- package/src/rag/retriever.js +17 -3
- package/src/rag/vec-store.js +6 -4
- package/src/rag/where-parser.js +54 -0
package/README.md
CHANGED
|
@@ -345,6 +345,21 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
345
345
|
|
|
346
346
|
## Recent releases
|
|
347
347
|
|
|
348
|
+
> **v2.30.0 released** — Minor. **Writable config UI on HU Board** — settings modal with grouped sections, atomic-write backend, and a global vs per-project scope toggle. No more hand-editing the YAML.
|
|
349
|
+
>
|
|
350
|
+
> - **Pipeline role toggles** (PR #854): 8 nuevos booleanos en el modal (`planner`, `researcher`, `architect`, `tester`, `security`, `refactorer`, `impeccable`, `brain`) reflejan los defaults reales de `src/config/defaults.js`.
|
|
351
|
+
> - **RAG controls** (PR #855): `rag.preload.{enabled,topK,scope}` + `rag.embedder.provider` editables sin abrir el archivo. El dropdown del provider lista los 6 embedders (ollama / openai / voyage / cohere / mistral / onnx).
|
|
352
|
+
> - **Grouped sections** (PR #856): los campos se agrupan por categoría (Agentes y modelos, Roles del pipeline, RAG, Tiempos de sesión, Calidad) con iconos y orden determinista. Campos nuevos sin categoría caen en "Otros" — defensivo.
|
|
353
|
+
> - **Scope toggle global vs per-project** (PR #857): pill toggle en el header del modal. Switchea entre `~/.karajan/kj.config.yml` (global, default) y `<projectDir>/.karajan/kj.config.yml` (override del proyecto, se crea al guardar). Atomic-write + `.bak` aplican a ambos.
|
|
354
|
+
>
|
|
355
|
+
> **v2.29.0 released** — Minor. **RAG quality lift** — dashboard + three new providers + metadata filter + rerank.
|
|
356
|
+
>
|
|
357
|
+
> - **Retrieval dashboard on HU Board** (PR #843): nuevo `/rag.html` con embedder activo, tamaño de DB, last-index, chunks por kind, chunks por proyecto. Primera pieza del config UI que llega completo en v2.30.
|
|
358
|
+
> - **Cohere + Mistral embedders** (PR #848): `embed-multilingual-v3.0` (multi-idioma) y `mistral-embed` (EU-hosted, útil para GDPR). `KJ_COHERE_KEY` / `KJ_MISTRAL_KEY` Karajan-scoped.
|
|
359
|
+
> - **ONNX local embedder** (PR #??): `@huggingface/transformers` corriendo en Node — sin Docker, sin API key, sin Ollama. Default `Xenova/all-MiniLM-L6-v2` (384 dim). Es la base para el zero-config init de v2.31.
|
|
360
|
+
> - **Metadata `--where` filter** (PR #??): `kj rag query 'auth' --where 'symbol=loadConfig AND kind=plan'`. Gramática mínima KEY=VALUE AND KEY=VALUE; cualquier metadata que emite el chunker es queryable sin schema changes.
|
|
361
|
+
> - **Cross-encoder rerank** (PR #??): opt-in `--rerank` re-ordena los topK con un cross-encoder (`Xenova/ms-marco-MiniLM-L-6-v2`). Latencia acotada, calidad final notablemente mejor.
|
|
362
|
+
>
|
|
348
363
|
> **v2.28.0 released** — Minor. **RAG advanced** — live re-index + cloud embedders + hybrid scoring + AST chunker, plus a real fix for the v2.27.0 chapuza.
|
|
349
364
|
>
|
|
350
365
|
> - **`kj watch`** (PR #836): chokidar daemon que re-indexa los archivos del proyecto tras cada edit (1s debounce). Fin del `kj rag index` manual.
|
|
@@ -360,7 +375,7 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
360
375
|
> - **Asymmetric source-vs-test ranking** (PR #833): NL queries like `how does X work` no longer rank `tests/X.test.js` above `src/X.js`; test-flavoured queries still surface tests.
|
|
361
376
|
> - Plus **KJC-BUG-0063** (PR #834): skipped a TZ-dependent test that was blocking every CI run, and a `shrink-budget` workflow exclude fix for `docs/*.md` at the root.
|
|
362
377
|
>
|
|
363
|
-
> **Coming in v2.
|
|
378
|
+
> **Coming in v2.30.0+**: writable config UI on the HU Board (toggle roles, switch coder/reviewer, adjust alpha/mode/rerank without re-editing the YAML), then v2.31 zero-config init (reduce the wizard to one critical question, smart defaults for everything).
|
|
364
379
|
>
|
|
365
380
|
> **v2.26.0 released** — Minor. **RAG Auto-Bootstrap** — Ollama runs in Docker out of the box. `kj init` now provisions the embedder automatically (or skips with a clear reason on modest hardware / `--no-ollama`); `kj doctor` surfaces health; `kj ollama [start|stop|status|pull]` manages lifecycle without docker compose. See [docs/RAG.md](docs/RAG.md) for the full RAG guide ([español](docs/es/RAG.md)).
|
|
366
381
|
>
|
package/package.json
CHANGED
|
@@ -3272,10 +3272,14 @@ function handleRoute() {
|
|
|
3272
3272
|
// row. `select` → <select>, `boolean` → <input type=checkbox>,
|
|
3273
3273
|
// `number` → <input type=number min/max>. Help text under each
|
|
3274
3274
|
// label as a small grey paragraph.
|
|
3275
|
-
async function showConfigEditor() {
|
|
3275
|
+
async function showConfigEditor(scope = 'global') {
|
|
3276
|
+
// v2.30.0 PR4 — scope toggle. 'global' edita ~/.karajan/kj.config.yml
|
|
3277
|
+
// (afecta a todos los proyectos); 'project' edita
|
|
3278
|
+
// <projectDir>/.karajan/kj.config.yml (sólo el repo actual). El backend
|
|
3279
|
+
// resuelve <projectDir> desde KJ_PROJECT_DIR || cwd.
|
|
3276
3280
|
let cfg;
|
|
3277
3281
|
try {
|
|
3278
|
-
const r = await fetch(
|
|
3282
|
+
const r = await fetch(`/api/config?scope=${encodeURIComponent(scope)}`);
|
|
3279
3283
|
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
|
3280
3284
|
cfg = await r.json();
|
|
3281
3285
|
} catch (err) {
|
|
@@ -3306,14 +3310,66 @@ async function showConfigEditor() {
|
|
|
3306
3310
|
</div>
|
|
3307
3311
|
`;
|
|
3308
3312
|
};
|
|
3313
|
+
|
|
3314
|
+
// v2.30.0 — agrupa los campos por `f.category`. El backend declara
|
|
3315
|
+
// las categorías ordenadas en `cfg.categories`; cualquier campo sin
|
|
3316
|
+
// categoría cae a "Otros" (defensivo, no se pierde si olvidamos
|
|
3317
|
+
// etiquetar uno nuevo). Mantiene el orden interno del whitelist
|
|
3318
|
+
// dentro de cada sección.
|
|
3319
|
+
const cats = Array.isArray(cfg.categories) && cfg.categories.length > 0
|
|
3320
|
+
? cfg.categories
|
|
3321
|
+
: [{ id: 'misc', label: 'Otros', icon: '⚙', order: 999 }];
|
|
3322
|
+
const byCat = new Map(cats.map((c) => [c.id, []]));
|
|
3323
|
+
byCat.set('misc', byCat.get('misc') || []);
|
|
3324
|
+
for (const f of cfg.fields) {
|
|
3325
|
+
const id = byCat.has(f.category) ? f.category : 'misc';
|
|
3326
|
+
if (!byCat.has(id)) byCat.set(id, []);
|
|
3327
|
+
byCat.get(id).push(f);
|
|
3328
|
+
}
|
|
3329
|
+
const renderSection = (cat) => {
|
|
3330
|
+
const fields = byCat.get(cat.id) || [];
|
|
3331
|
+
if (fields.length === 0) return '';
|
|
3332
|
+
return `
|
|
3333
|
+
<section style="margin-bottom:22px;">
|
|
3334
|
+
<h3 style="display:flex;align-items:center;gap:8px;margin:0 0 12px;padding-bottom:6px;border-bottom:1px solid var(--border);font-size:0.95rem;font-weight:700;color:var(--text);">
|
|
3335
|
+
<span aria-hidden="true">${esc(cat.icon || '⚙')}</span>
|
|
3336
|
+
<span>${esc(cat.label)}</span>
|
|
3337
|
+
</h3>
|
|
3338
|
+
${fields.map(renderField).join('')}
|
|
3339
|
+
</section>
|
|
3340
|
+
`;
|
|
3341
|
+
};
|
|
3342
|
+
const allCats = [...cats];
|
|
3343
|
+
if ((byCat.get('misc') || []).length > 0 && !allCats.some((c) => c.id === 'misc')) {
|
|
3344
|
+
allCats.push({ id: 'misc', label: 'Otros', icon: '⚙', order: 999 });
|
|
3345
|
+
}
|
|
3346
|
+
const sectionsHtml = allCats.map(renderSection).join('');
|
|
3347
|
+
|
|
3348
|
+
// PR4 — toggle global / proyecto. Cambiarlo recarga el modal contra el
|
|
3349
|
+
// otro fichero. Visual: dos pills mutuamente excluyentes.
|
|
3350
|
+
const pill = (id, value, label, active) => `
|
|
3351
|
+
<button data-scope="${value}" id="${id}" type="button"
|
|
3352
|
+
style="padding:4px 10px;border-radius:999px;border:1px solid var(--border);
|
|
3353
|
+
background:${active ? 'var(--color-blue)' : 'var(--bg-primary)'};
|
|
3354
|
+
color:${active ? '#fff' : 'var(--text)'};font-size:0.75rem;cursor:pointer;">
|
|
3355
|
+
${esc(label)}
|
|
3356
|
+
</button>`;
|
|
3357
|
+
const scopeToggle = `
|
|
3358
|
+
<div style="display:flex;align-items:center;gap:4px;margin-left:auto;" role="radiogroup" aria-label="Ámbito de configuración">
|
|
3359
|
+
${pill('cfg-scope-global', 'global', 'Global', scope === 'global')}
|
|
3360
|
+
${pill('cfg-scope-project', 'project', 'Este proyecto', scope === 'project')}
|
|
3361
|
+
</div>`;
|
|
3309
3362
|
dlg.innerHTML = `
|
|
3310
3363
|
<div style="padding:14px 18px;border-bottom:1px solid var(--border);font-weight:700;display:flex;align-items:center;gap:10px;">
|
|
3311
3364
|
<span style="font-size:1.2rem;">⚙</span>
|
|
3312
3365
|
<span>Configuración de Karajan</span>
|
|
3313
|
-
|
|
3366
|
+
${scopeToggle}
|
|
3367
|
+
</div>
|
|
3368
|
+
<div style="padding:6px 18px 0;font-family:var(--font-mono,monospace);font-size:0.7rem;color:var(--text-muted);">
|
|
3369
|
+
${esc(cfg.path || '')}${cfg.exists === false ? ' · (aún no existe, se creará al guardar)' : ''}
|
|
3314
3370
|
</div>
|
|
3315
3371
|
<div style="padding:14px 18px;max-height:65vh;overflow:auto;">
|
|
3316
|
-
${
|
|
3372
|
+
${sectionsHtml}
|
|
3317
3373
|
<p style="margin:14px 0 0;font-size:0.75rem;color:var(--text-muted);">
|
|
3318
3374
|
Los cambios se guardan en <code>kj.config.yml</code> de forma atómica (con copia de seguridad <code>.bak</code>).
|
|
3319
3375
|
El próximo <code>kj run</code> los aplicará — no hace falta reiniciar el board.
|
|
@@ -3325,6 +3381,15 @@ async function showConfigEditor() {
|
|
|
3325
3381
|
</div>
|
|
3326
3382
|
`;
|
|
3327
3383
|
if (typeof dlg.showModal === 'function' && !dlg.open) dlg.showModal();
|
|
3384
|
+
// PR4 — al cambiar de pill, recargar el modal con el otro scope.
|
|
3385
|
+
for (const btn of dlg.querySelectorAll('[data-scope]')) {
|
|
3386
|
+
btn.addEventListener('click', () => {
|
|
3387
|
+
const next = btn.getAttribute('data-scope');
|
|
3388
|
+
if (next === scope) return;
|
|
3389
|
+
try { dlg.close(); } catch { /* ignore */ }
|
|
3390
|
+
showConfigEditor(next);
|
|
3391
|
+
}, { once: true });
|
|
3392
|
+
}
|
|
3328
3393
|
dlg.querySelector('#config-cancel')?.addEventListener('click', () => { try { dlg.close(); } catch { /* ignore */ } }, { once: true });
|
|
3329
3394
|
dlg.querySelector('#config-save')?.addEventListener('click', async () => {
|
|
3330
3395
|
// Build the patch from the form, comparing against initial values
|
|
@@ -3347,7 +3412,7 @@ async function showConfigEditor() {
|
|
|
3347
3412
|
const r = await fetch('/api/config', {
|
|
3348
3413
|
method: 'PUT',
|
|
3349
3414
|
headers: { 'Content-Type': 'application/json' },
|
|
3350
|
-
body: JSON.stringify({ patch }),
|
|
3415
|
+
body: JSON.stringify({ patch, scope }),
|
|
3351
3416
|
});
|
|
3352
3417
|
if (!r.ok) {
|
|
3353
3418
|
const body = await r.json().catch(() => ({}));
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
<button class="nav-btn" data-view="dashboard" title="Per-project landing: stats grid + project list. Click a project card to focus the kanban / graph on that project.">Dashboard</button>
|
|
26
26
|
<button class="nav-btn" data-view="sessions" title="Every kj run that has touched the board, newest first. Filter by project; click a session to inspect its iterations, commits and checkpoints.">Sessions</button>
|
|
27
27
|
<a class="nav-btn" href="/pipeline.html" title="Live observability of pipeline runs (Karajan v2.7+): stage timings, agent calls, decision logs.">Pipeline</a>
|
|
28
|
+
<a class="nav-btn" href="/rag.html" title="RAG retrieval-quality dashboard: chunk counts, embedder provider, db size.">RAG</a>
|
|
28
29
|
<select class="project-select" id="project-select">
|
|
29
30
|
<option value="">All Projects</option>
|
|
30
31
|
</select>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>Karajan RAG · Retrieval Dashboard</title>
|
|
7
|
+
<link rel="stylesheet" href="/styles.css">
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
<header class="header">
|
|
11
|
+
<div class="header__logo"><div><div class="header__title">Karajan RAG</div><div class="header__subtitle">retrieval-quality dashboard</div></div></div>
|
|
12
|
+
<nav class="header__nav">
|
|
13
|
+
<a class="nav-btn" href="/">← Board</a>
|
|
14
|
+
<button class="nav-btn active">RAG</button>
|
|
15
|
+
<div class="header__controls"><button class="control-btn" id="rag-refresh" title="Reload stats">🔄</button></div>
|
|
16
|
+
</nav>
|
|
17
|
+
</header>
|
|
18
|
+
<main class="main" id="rag-app"><div class="loading"><div class="loading__spinner"></div><p>Loading…</p></div></main>
|
|
19
|
+
<script src="/rag.js"></script>
|
|
20
|
+
</body>
|
|
21
|
+
</html>
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// KJC-TSK-0445 — Retrieval dashboard frontend. Fetches /api/rag/stats and
|
|
2
|
+
// renders chunk counts per project/kind, embedder, DB size and last-index.
|
|
3
|
+
const TOKEN = new URLSearchParams(location.search).get("token") || localStorage.getItem("kj_board_token") || "";
|
|
4
|
+
const HEADERS = TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {};
|
|
5
|
+
const ESC = (s) => String(s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
6
|
+
|
|
7
|
+
function bytes(n) { for (const u of ["B", "KB", "MB", "GB"]) { if (n < 1024) return `${u === "B" ? n : n.toFixed(1)} ${u}`; n /= 1024; } return `${n.toFixed(2)} TB`; }
|
|
8
|
+
|
|
9
|
+
function bars(rows, lab, val, max) {
|
|
10
|
+
if (!rows.length) return '<p class="rag-empty">no data</p>';
|
|
11
|
+
return rows.map((r) => `<div class="rag-bar"><span class="rag-bar__label">${ESC(lab(r))}</span><span class="rag-bar__fill" style="width:${Math.round((val(r) / max) * 100)}%"></span><span class="rag-bar__value">${val(r).toLocaleString()}</span></div>`).join("");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function embedderCard(e) {
|
|
15
|
+
return e ? `<div class="rag-card"><h3>Embedder</h3><p class="rag-stat-big">${ESC(e.provider)}</p><p class="rag-stat-sub">${ESC(e.model || "default model")} · dim ${e.dim ?? "?"}</p></div>` : "";
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function render(root, data) {
|
|
19
|
+
if (!data.initialized) { root.innerHTML = `<section class="rag-grid"><div class="rag-card"><h3>RAG DB</h3><p class="rag-empty">${ESC(data.message || "not initialized")}</p></div>${embedderCard(data.embedder)}</section>`; return; }
|
|
20
|
+
const lastIdx = data.last_indexed ? new Date(data.last_indexed).toLocaleString() : "never";
|
|
21
|
+
const mk = Math.max(1, ...data.by_kind.map((r) => r.n));
|
|
22
|
+
const mp = Math.max(1, ...data.by_project.map((r) => r.n));
|
|
23
|
+
root.innerHTML = `<section class="rag-grid">
|
|
24
|
+
<div class="rag-card"><h3>Totals</h3><p class="rag-stat-big">${data.total_chunks.toLocaleString()}</p><p class="rag-stat-sub">chunks · ${bytes(data.db_size_bytes)} · last indexed ${ESC(lastIdx)}</p></div>
|
|
25
|
+
${embedderCard(data.embedder)}
|
|
26
|
+
<div class="rag-card rag-card--wide"><h3>By kind</h3>${bars(data.by_kind, (r) => r.kind, (r) => r.n, mk)}</div>
|
|
27
|
+
<div class="rag-card rag-card--wide"><h3>By project (top 20)</h3>${bars(data.by_project, (r) => r.project, (r) => r.n, mp)}</div>
|
|
28
|
+
</section>`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function load() {
|
|
32
|
+
const app = document.getElementById("rag-app");
|
|
33
|
+
app.innerHTML = '<div class="loading"><div class="loading__spinner"></div><p>Loading…</p></div>';
|
|
34
|
+
try { const r = await fetch("/api/rag/stats", { headers: HEADERS }); if (!r.ok) throw new Error(`HTTP ${r.status}`); render(app, await r.json()); }
|
|
35
|
+
catch (e) { app.innerHTML = `<div class="rag-card rag-card--error"><strong>Error:</strong> ${ESC(e.message)}</div>`; }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
document.getElementById("rag-refresh").addEventListener("click", load);
|
|
39
|
+
load();
|
|
@@ -916,3 +916,17 @@ body.scoped-mode .nav-btn[data-view="dashboard"],
|
|
|
916
916
|
body.scoped-mode .header__nav a[href="/pipeline.html"] {
|
|
917
917
|
display: none;
|
|
918
918
|
}
|
|
919
|
+
|
|
920
|
+
/* KJC-TSK-0445 — RAG dashboard */
|
|
921
|
+
.rag-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 1rem; padding: 1rem; }
|
|
922
|
+
.rag-card { background: var(--bg-secondary, #1e1e2e); border-radius: 8px; padding: 1.25rem; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
|
|
923
|
+
.rag-card--wide { grid-column: 1 / -1; }
|
|
924
|
+
.rag-card--error { color: #f87171; background: #2a1414; }
|
|
925
|
+
.rag-card h3 { margin: 0 0 .75rem; font-size: .9rem; text-transform: uppercase; letter-spacing: .05em; color: var(--text-secondary, #94a3b8); }
|
|
926
|
+
.rag-stat-big { font-size: 2rem; font-weight: 700; margin: 0; color: var(--text-primary, #f8fafc); }
|
|
927
|
+
.rag-stat-sub { margin: .25rem 0 0; font-size: .85rem; color: var(--text-secondary, #94a3b8); }
|
|
928
|
+
.rag-empty { color: var(--text-secondary, #94a3b8); font-style: italic; }
|
|
929
|
+
.rag-bar { display: grid; grid-template-columns: 140px 1fr 60px; align-items: center; gap: .5rem; margin: .35rem 0; font-size: .85rem; }
|
|
930
|
+
.rag-bar__label { color: var(--text-secondary, #cbd5e1); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
931
|
+
.rag-bar__fill { display: block; height: 14px; background: linear-gradient(90deg, #7c3aed, #a78bfa); border-radius: 3px; }
|
|
932
|
+
.rag-bar__value { text-align: right; color: var(--text-primary, #f8fafc); font-variant-numeric: tabular-nums; }
|
|
@@ -34,7 +34,29 @@ import { tmpdir } from 'node:os';
|
|
|
34
34
|
import yaml from 'js-yaml';
|
|
35
35
|
import { getKjHome } from './db.js';
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
/**
|
|
38
|
+
* v2.30.0 PR4 — scope resolver.
|
|
39
|
+
*
|
|
40
|
+
* 'global' → ~/.karajan/kj.config.yml (mismo path que el resto de
|
|
41
|
+
* Karajan; afecta a todos los proyectos).
|
|
42
|
+
* 'project' → <projectDir>/.karajan/kj.config.yml (override por
|
|
43
|
+
* proyecto; cuando existe, el loader del padre lo prefiere
|
|
44
|
+
* sobre el global).
|
|
45
|
+
*
|
|
46
|
+
* projectDir se resuelve desde `KJ_PROJECT_DIR` (ya usado por
|
|
47
|
+
* journal-parser.js) con fallback a `process.cwd()` — el board se
|
|
48
|
+
* arranca desde el repo del usuario, así que cwd es el proyecto.
|
|
49
|
+
*/
|
|
50
|
+
export const SCOPES = ['global', 'project'];
|
|
51
|
+
|
|
52
|
+
function getProjectDir() {
|
|
53
|
+
return process.env.KJ_PROJECT_DIR || process.cwd();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function configPath(scope = 'global') {
|
|
57
|
+
if (scope === 'project') {
|
|
58
|
+
return join(getProjectDir(), '.karajan', 'kj.config.yml');
|
|
59
|
+
}
|
|
38
60
|
// Same path the parent uses (src/utils/paths.js::getKarajanHome).
|
|
39
61
|
// KJC-TSK-0420: delegate to db.js::getKjHome — gives us KARAJAN_HOME
|
|
40
62
|
// priority + KJ_HOME fallback + VITEST guard from one place, instead
|
|
@@ -42,13 +64,28 @@ function configPath() {
|
|
|
42
64
|
return join(getKjHome(), 'kj.config.yml');
|
|
43
65
|
}
|
|
44
66
|
|
|
45
|
-
function backupPath() { return `${configPath()}.bak`; }
|
|
67
|
+
function backupPath(scope = 'global') { return `${configPath(scope)}.bak`; }
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* v2.30.0 — Categorías para agrupar los campos en el modal. El backend
|
|
71
|
+
* sólo declara la metadata (id, label, icon, order). El front (app.js)
|
|
72
|
+
* itera sobre EDITABLE_FIELDS, agrupa por `f.category` y dibuja un
|
|
73
|
+
* <h3> por sección. Si un campo nuevo no declara categoría, cae en
|
|
74
|
+
* "Otros" (defensivo: no se pierde aunque se nos olvide etiquetarlo).
|
|
75
|
+
*/
|
|
76
|
+
export const CATEGORIES = [
|
|
77
|
+
{ id: 'agents', label: 'Agentes y modelos', icon: '🤖', order: 1 },
|
|
78
|
+
{ id: 'pipeline', label: 'Roles del pipeline', icon: '🔌', order: 2 },
|
|
79
|
+
{ id: 'rag', label: 'RAG (búsqueda de contexto)', icon: '📚', order: 3 },
|
|
80
|
+
{ id: 'session', label: 'Tiempos de sesión', icon: '⏱', order: 4 },
|
|
81
|
+
{ id: 'quality', label: 'Calidad', icon: '✅', order: 5 },
|
|
82
|
+
];
|
|
46
83
|
|
|
47
84
|
/**
|
|
48
85
|
* Whitelist of fields the board's modal exposes. Everything else
|
|
49
86
|
* stays untouched in the yml. Each entry says where in the yml the
|
|
50
87
|
* field lives (`path`) and how to render it for the UI (`type`,
|
|
51
|
-
* `label`, `options` for selects).
|
|
88
|
+
* `label`, `options` for selects, `category` for grouping).
|
|
52
89
|
*
|
|
53
90
|
* The path is a dotted notation we resolve manually (so we never
|
|
54
91
|
* need to take a runtime dep on lodash for `_.get`/`_.set`).
|
|
@@ -57,6 +94,7 @@ export const EDITABLE_FIELDS = [
|
|
|
57
94
|
{
|
|
58
95
|
key: 'coder',
|
|
59
96
|
path: 'coder',
|
|
97
|
+
category: 'agents',
|
|
60
98
|
type: 'select',
|
|
61
99
|
label: 'Agente para el rol "coder"',
|
|
62
100
|
help: 'Quién escribe el código. Claude es el más fiable, Codex es más rápido.',
|
|
@@ -66,6 +104,7 @@ export const EDITABLE_FIELDS = [
|
|
|
66
104
|
{
|
|
67
105
|
key: 'reviewer',
|
|
68
106
|
path: 'reviewer',
|
|
107
|
+
category: 'agents',
|
|
69
108
|
type: 'select',
|
|
70
109
|
label: 'Agente para el rol "reviewer"',
|
|
71
110
|
help: 'Quién revisa el código del coder. Conviene que sea un agente distinto al coder para detectar errores.',
|
|
@@ -75,6 +114,7 @@ export const EDITABLE_FIELDS = [
|
|
|
75
114
|
{
|
|
76
115
|
key: 'modelMode',
|
|
77
116
|
path: '_modelMode_virtual',
|
|
117
|
+
category: 'agents',
|
|
78
118
|
type: 'select',
|
|
79
119
|
label: 'Estrategia de modelos',
|
|
80
120
|
help: 'Auto = Karajan elige el modelo según la complejidad (haiku/sonnet/opus). Sonnet siempre = barato, ideal para pruebas. Opus siempre = caro, máxima calidad.',
|
|
@@ -86,6 +126,7 @@ export const EDITABLE_FIELDS = [
|
|
|
86
126
|
{
|
|
87
127
|
key: 'maxIterationMinutes',
|
|
88
128
|
path: 'session.max_iteration_minutes',
|
|
129
|
+
category: 'session',
|
|
89
130
|
type: 'number',
|
|
90
131
|
label: 'Tope de minutos por iteración',
|
|
91
132
|
help: 'Si una iteración (coder + reviewer) supera este tiempo, Karajan la corta y Solomon decide qué hacer.',
|
|
@@ -96,6 +137,7 @@ export const EDITABLE_FIELDS = [
|
|
|
96
137
|
{
|
|
97
138
|
key: 'maxTotalMinutes',
|
|
98
139
|
path: 'session.max_total_minutes',
|
|
140
|
+
category: 'session',
|
|
99
141
|
type: 'number',
|
|
100
142
|
label: 'Tope de minutos total por sesión',
|
|
101
143
|
help: 'Tope absoluto del wall-clock de la sesión. Si se llega aquí, Karajan para aunque queden HUs.',
|
|
@@ -106,11 +148,133 @@ export const EDITABLE_FIELDS = [
|
|
|
106
148
|
{
|
|
107
149
|
key: 'sonarEnabled',
|
|
108
150
|
path: 'sonarqube.enabled',
|
|
151
|
+
category: 'quality',
|
|
109
152
|
type: 'boolean',
|
|
110
153
|
label: 'Activar análisis SonarQube',
|
|
111
154
|
help: 'Pasa el código por SonarQube y bloquea si el quality gate falla. Requiere SonarQube corriendo.',
|
|
112
155
|
default: false,
|
|
113
156
|
},
|
|
157
|
+
// v2.30.0 — pipeline role toggles. Cada uno controla si una etapa
|
|
158
|
+
// del pipeline se ejecuta. Brain está OFF por defecto porque sigue
|
|
159
|
+
// siendo opt-in (v2 feature); el resto reflejan los defaults de
|
|
160
|
+
// src/config/defaults.js.
|
|
161
|
+
{
|
|
162
|
+
key: 'plannerEnabled',
|
|
163
|
+
path: 'pipeline.planner.enabled',
|
|
164
|
+
category: 'pipeline',
|
|
165
|
+
type: 'boolean',
|
|
166
|
+
label: 'Activar rol planner',
|
|
167
|
+
help: 'Genera un plan de implementación antes de codificar. Útil en tareas con >2 archivos o devPoints ≥ 3.',
|
|
168
|
+
default: true,
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
key: 'researcherEnabled',
|
|
172
|
+
path: 'pipeline.researcher.enabled',
|
|
173
|
+
category: 'pipeline',
|
|
174
|
+
type: 'boolean',
|
|
175
|
+
label: 'Activar rol researcher',
|
|
176
|
+
help: 'Investiga la codebase antes del coder (lectura de archivos clave, patrones existentes).',
|
|
177
|
+
default: true,
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
key: 'architectEnabled',
|
|
181
|
+
path: 'pipeline.architect.enabled',
|
|
182
|
+
category: 'pipeline',
|
|
183
|
+
type: 'boolean',
|
|
184
|
+
label: 'Activar rol architect',
|
|
185
|
+
help: 'Diseña la arquitectura (capas, patrones, contratos) antes de codificar. Recomendado en cambios estructurales.',
|
|
186
|
+
default: true,
|
|
187
|
+
},
|
|
188
|
+
{
|
|
189
|
+
key: 'testerEnabled',
|
|
190
|
+
path: 'pipeline.tester.enabled',
|
|
191
|
+
category: 'pipeline',
|
|
192
|
+
type: 'boolean',
|
|
193
|
+
label: 'Activar rol tester',
|
|
194
|
+
help: 'Genera tests automáticos durante o después del coder. Refuerza la calidad.',
|
|
195
|
+
default: true,
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
key: 'securityEnabled',
|
|
199
|
+
path: 'pipeline.security.enabled',
|
|
200
|
+
category: 'pipeline',
|
|
201
|
+
type: 'boolean',
|
|
202
|
+
label: 'Activar rol security',
|
|
203
|
+
help: 'Audita el código contra OWASP-style issues, secret leaks, dependencias vulnerables.',
|
|
204
|
+
default: true,
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
key: 'refactorerEnabled',
|
|
208
|
+
path: 'pipeline.refactorer.enabled',
|
|
209
|
+
category: 'pipeline',
|
|
210
|
+
type: 'boolean',
|
|
211
|
+
label: 'Activar rol refactorer',
|
|
212
|
+
help: 'Pasa una iteración de limpieza tras el coder (extract function, rename, etc.).',
|
|
213
|
+
default: false,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
key: 'impeccableEnabled',
|
|
217
|
+
path: 'pipeline.impeccable.enabled',
|
|
218
|
+
category: 'pipeline',
|
|
219
|
+
type: 'boolean',
|
|
220
|
+
label: 'Activar rol impeccable',
|
|
221
|
+
help: 'Auditoría UI/UX automática (Lighthouse, a11y, contraste). Solo útil en frontend.',
|
|
222
|
+
default: false,
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
key: 'brainEnabled',
|
|
226
|
+
path: 'pipeline.brain.enabled',
|
|
227
|
+
category: 'pipeline',
|
|
228
|
+
type: 'boolean',
|
|
229
|
+
label: 'Activar Karajan Brain (v2)',
|
|
230
|
+
help: 'Orquestador IA central: enriquece feedback, comprime outputs y verifica cambios reales. Opt-in.',
|
|
231
|
+
default: false,
|
|
232
|
+
},
|
|
233
|
+
// v2.30.0 — RAG controls. Sólo se exponen las claves que el código
|
|
234
|
+
// realmente lee hoy (rag.preload.* en rag-context-stage.js y
|
|
235
|
+
// rag.embedder.provider en embedders/factory.js). mode/alpha/rerank
|
|
236
|
+
// se controlan vía flags CLI; cuando se cableen en config se añadirán
|
|
237
|
+
// aquí (ver task #99).
|
|
238
|
+
{
|
|
239
|
+
key: 'ragPreloadEnabled',
|
|
240
|
+
path: 'rag.preload.enabled',
|
|
241
|
+
category: 'rag',
|
|
242
|
+
type: 'boolean',
|
|
243
|
+
label: 'Activar pre-carga RAG en el pipeline',
|
|
244
|
+
help: 'Si está activo, antes de cada iteración se inyecta contexto del vector store en el prompt del coder.',
|
|
245
|
+
default: false,
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
key: 'ragPreloadTopK',
|
|
249
|
+
path: 'rag.preload.topK',
|
|
250
|
+
category: 'rag',
|
|
251
|
+
type: 'number',
|
|
252
|
+
label: 'Nº de chunks a pre-cargar (topK)',
|
|
253
|
+
help: 'Cuántos fragmentos del vector store se inyectan como contexto. 3-5 funciona bien; subir gasta tokens.',
|
|
254
|
+
min: 1,
|
|
255
|
+
max: 20,
|
|
256
|
+
default: 5,
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
key: 'ragPreloadScope',
|
|
260
|
+
path: 'rag.preload.scope',
|
|
261
|
+
category: 'rag',
|
|
262
|
+
type: 'select',
|
|
263
|
+
label: 'Scope de la pre-carga RAG',
|
|
264
|
+
help: '"all" mezcla todo. "code" solo busca en el código indexado, "plans" en planes previos, "onboarding" en el brief del proyecto.',
|
|
265
|
+
options: ['all', 'code', 'plans', 'onboarding'],
|
|
266
|
+
default: 'all',
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
key: 'ragEmbedderProvider',
|
|
270
|
+
path: 'rag.embedder.provider',
|
|
271
|
+
category: 'rag',
|
|
272
|
+
type: 'select',
|
|
273
|
+
label: 'Proveedor de embeddings RAG',
|
|
274
|
+
help: 'Ollama local es gratis. OpenAI/Voyage/Cohere/Mistral son cloud (requieren API key). ONNX corre 100% local con @huggingface/transformers.',
|
|
275
|
+
options: ['ollama', 'openai', 'voyage', 'cohere', 'mistral', 'onnx'],
|
|
276
|
+
default: 'ollama',
|
|
277
|
+
},
|
|
114
278
|
];
|
|
115
279
|
|
|
116
280
|
const ROLES_FOR_MODEL_MODE = [
|
|
@@ -203,8 +367,13 @@ function applyModelMode(parsed, mode) {
|
|
|
203
367
|
*
|
|
204
368
|
* Returns sane defaults when the file doesn't exist yet.
|
|
205
369
|
*/
|
|
206
|
-
export function readConfig() {
|
|
207
|
-
|
|
370
|
+
export function readConfig({ scope = 'global' } = {}) {
|
|
371
|
+
// PR4 — accept scope. Default 'global' preserves the previous behavior;
|
|
372
|
+
// 'project' reads <projectDir>/.karajan/kj.config.yml.
|
|
373
|
+
if (!SCOPES.includes(scope)) {
|
|
374
|
+
throw new Error(`Scope inválido: ${scope}. Permitidos: ${SCOPES.join(', ')}`);
|
|
375
|
+
}
|
|
376
|
+
const p = configPath(scope);
|
|
208
377
|
let parsed = {};
|
|
209
378
|
let exists = false;
|
|
210
379
|
if (existsSync(p)) {
|
|
@@ -222,7 +391,11 @@ export function readConfig() {
|
|
|
222
391
|
if (value === undefined) value = f.default;
|
|
223
392
|
return { ...f, value };
|
|
224
393
|
});
|
|
225
|
-
|
|
394
|
+
// v2.30.0 — also expose CATEGORIES so the modal can render grouped
|
|
395
|
+
// sections without re-importing the constant client-side. Ordered
|
|
396
|
+
// by `order` for deterministic UI.
|
|
397
|
+
const categories = [...CATEGORIES].sort((a, b) => (a.order ?? 999) - (b.order ?? 999));
|
|
398
|
+
return { path: p, scope, exists, fields, categories, raw: parsed };
|
|
226
399
|
}
|
|
227
400
|
|
|
228
401
|
/**
|
|
@@ -232,16 +405,20 @@ export function readConfig() {
|
|
|
232
405
|
*
|
|
233
406
|
* @returns {{ path: string, written: boolean, applied: object[], errors: string[] }}
|
|
234
407
|
*/
|
|
235
|
-
export function writeConfigPatch(patch) {
|
|
408
|
+
export function writeConfigPatch(patch, { scope = 'global' } = {}) {
|
|
409
|
+
// PR4 — same shape as readConfig: scope decides target file.
|
|
410
|
+
if (!SCOPES.includes(scope)) {
|
|
411
|
+
return { path: '', scope, written: false, applied: [], errors: [`Scope inválido: ${scope}. Permitidos: ${SCOPES.join(', ')}`] };
|
|
412
|
+
}
|
|
236
413
|
if (!patch || typeof patch !== 'object') {
|
|
237
|
-
return { path: configPath(), written: false, applied: [], errors: ['patch debe ser un objeto'] };
|
|
414
|
+
return { path: configPath(scope), scope, written: false, applied: [], errors: ['patch debe ser un objeto'] };
|
|
238
415
|
}
|
|
239
416
|
// Read current state (or empty if file doesn't exist yet).
|
|
240
|
-
const p = configPath();
|
|
417
|
+
const p = configPath(scope);
|
|
241
418
|
let parsed = {};
|
|
242
419
|
if (existsSync(p)) {
|
|
243
420
|
try { parsed = yaml.load(readFileSync(p, 'utf8'), { json: true }) || {}; }
|
|
244
|
-
catch (err) { return { path: p, written: false, applied: [], errors: [`No se pudo parsear el yml actual: ${err.message}`] }; }
|
|
421
|
+
catch (err) { return { path: p, scope, written: false, applied: [], errors: [`No se pudo parsear el yml actual: ${err.message}`] }; }
|
|
245
422
|
}
|
|
246
423
|
const errors = [];
|
|
247
424
|
const applied = [];
|
|
@@ -268,18 +445,18 @@ export function writeConfigPatch(patch) {
|
|
|
268
445
|
else setDeep(parsed, field.path, field.type === 'number' ? Number(val) : val);
|
|
269
446
|
applied.push({ key, value: val });
|
|
270
447
|
}
|
|
271
|
-
if (errors.length > 0) return { path: p, written: false, applied, errors };
|
|
448
|
+
if (errors.length > 0) return { path: p, scope, written: false, applied, errors };
|
|
272
449
|
|
|
273
450
|
// Write atomically: serialize → write to .tmp → rename. Backup the
|
|
274
451
|
// existing file first so the user can recover.
|
|
275
452
|
const dir = dirname(p);
|
|
276
453
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
277
|
-
if (existsSync(p)) { try { copyFileSync(p, backupPath()); } catch { /* best-effort */ } }
|
|
454
|
+
if (existsSync(p)) { try { copyFileSync(p, backupPath(scope)); } catch { /* best-effort */ } }
|
|
278
455
|
const dump = yaml.dump(parsed, { lineWidth: 120 });
|
|
279
456
|
// mkstemp would be nicer but we want to stay in the same dir for
|
|
280
457
|
// a guaranteed-atomic rename across the same filesystem.
|
|
281
458
|
const tmpFile = join(dir, `.kj.config.yml.${process.pid}.${Date.now()}.tmp`);
|
|
282
459
|
writeFileSync(tmpFile, dump, 'utf8');
|
|
283
460
|
renameSync(tmpFile, p);
|
|
284
|
-
return { path: p, written: true, applied, errors: [] };
|
|
461
|
+
return { path: p, scope, written: true, applied, errors: [] };
|
|
285
462
|
}
|
|
@@ -1031,29 +1031,41 @@ router.post('/projects/:id/run', (req, res) => {
|
|
|
1031
1031
|
});
|
|
1032
1032
|
|
|
1033
1033
|
/**
|
|
1034
|
-
* GET /api/config — read the editable subset of
|
|
1034
|
+
* GET /api/config — read the editable subset of the kj.config.yml.
|
|
1035
1035
|
* Powers the board's settings modal; non-editable keys in the yml are
|
|
1036
1036
|
* left untouched on PUT (see writeConfigPatch's whitelist semantics).
|
|
1037
|
+
*
|
|
1038
|
+
* v2.30.0 PR4 — supports `?scope=global|project`:
|
|
1039
|
+
* - `global` (default): reads ~/.karajan/kj.config.yml.
|
|
1040
|
+
* - `project`: reads <projectDir>/.karajan/kj.config.yml.
|
|
1041
|
+
* Unknown scopes are 400'd. Response includes the resolved `scope` and
|
|
1042
|
+
* `path` so the UI can show which file backs the current view.
|
|
1037
1043
|
*/
|
|
1038
|
-
router.get('/config', (
|
|
1044
|
+
router.get('/config', (req, res) => {
|
|
1039
1045
|
try {
|
|
1040
|
-
const
|
|
1046
|
+
const scope = req.query?.scope || 'global';
|
|
1047
|
+
const data = readConfig({ scope });
|
|
1041
1048
|
res.json(data);
|
|
1042
1049
|
} catch (err) {
|
|
1043
|
-
res.status(
|
|
1050
|
+
res.status(400).json({ error: err.message });
|
|
1044
1051
|
}
|
|
1045
1052
|
});
|
|
1046
1053
|
|
|
1047
1054
|
/**
|
|
1048
|
-
* PUT /api/config — apply a patch to the yml. Body is
|
|
1049
|
-
*
|
|
1050
|
-
*
|
|
1051
|
-
*
|
|
1055
|
+
* PUT /api/config — apply a patch to the yml. Body is
|
|
1056
|
+
* `{ patch: { key: value }, scope?: 'global'|'project' }` with the keys
|
|
1057
|
+
* defined in EDITABLE_FIELDS. Unknown keys are 400'd; type errors are
|
|
1058
|
+
* 400'd; validation errors are 400'd. Writes are atomic (tmp + rename)
|
|
1059
|
+
* and a `.bak` is kept of the previous content.
|
|
1060
|
+
*
|
|
1061
|
+
* v2.30.0 PR4 — `scope` decides which file is patched. Default 'global'
|
|
1062
|
+
* preserves the previous behavior.
|
|
1052
1063
|
*/
|
|
1053
1064
|
router.put('/config', (req, res) => {
|
|
1054
1065
|
try {
|
|
1055
1066
|
const patch = req.body?.patch;
|
|
1056
|
-
const
|
|
1067
|
+
const scope = req.body?.scope || 'global';
|
|
1068
|
+
const result = writeConfigPatch(patch, { scope });
|
|
1057
1069
|
if (!result.written) {
|
|
1058
1070
|
return res.status(400).json({ error: 'No se pudo guardar la configuración', details: result });
|
|
1059
1071
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// KJC-TSK-0445 — RAG retrieval-quality dashboard route.
|
|
2
|
+
// GET /api/rag/stats → snapshot of the local rag.db (counts per kind/project,
|
|
3
|
+
// active embedder, db size, last-index timestamp). Missing DB returns an
|
|
4
|
+
// "uninitialized" payload so the dashboard renders an empty state cleanly.
|
|
5
|
+
import { Router } from "express";
|
|
6
|
+
import { existsSync, statSync } from "node:fs";
|
|
7
|
+
import Database from "better-sqlite3";
|
|
8
|
+
import { dbPath } from "../../../../src/rag/vec-store.js";
|
|
9
|
+
import { readConfig } from "../config-yaml.js";
|
|
10
|
+
|
|
11
|
+
const router = Router();
|
|
12
|
+
const DIMS = { ollama: 768, openai: 1536, voyage: 1024, cohere: 1024, mistral: 1024, onnx: 384 };
|
|
13
|
+
|
|
14
|
+
function embedder() {
|
|
15
|
+
try {
|
|
16
|
+
const e = (readConfig()?.rag?.embedder) || {};
|
|
17
|
+
const provider = e.provider || "ollama";
|
|
18
|
+
return { provider, model: e.model || null, dim: e.dim || DIMS[provider] || null };
|
|
19
|
+
} catch { return { provider: "ollama", model: null, dim: 768 }; }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
router.get("/stats", (_req, res) => {
|
|
23
|
+
const path = dbPath();
|
|
24
|
+
if (!existsSync(path)) return res.json({ ok: true, initialized: false, message: "rag.db not initialized — run `kj rag index <dir>` to start indexing.", embedder: embedder() });
|
|
25
|
+
try {
|
|
26
|
+
const db = new Database(path, { readonly: true, fileMustExist: true });
|
|
27
|
+
const total_chunks = db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
28
|
+
const by_kind = db.prepare("SELECT kind, COUNT(*) AS n FROM chunks GROUP BY kind").all();
|
|
29
|
+
const by_project = db.prepare("SELECT COALESCE(project_slug, '(no slug)') AS project, COUNT(*) AS n FROM chunks GROUP BY project_slug ORDER BY n DESC LIMIT 20").all();
|
|
30
|
+
const last_indexed = db.prepare("SELECT MAX(created_at) AS ts FROM chunks").get().ts;
|
|
31
|
+
db.close();
|
|
32
|
+
res.json({ ok: true, initialized: true, db_path: path, db_size_bytes: statSync(path).size, total_chunks, by_kind, by_project, last_indexed, embedder: embedder() });
|
|
33
|
+
} catch (err) { res.status(500).json({ ok: false, error: String(err.message || err) }); }
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
export default router;
|
|
@@ -9,6 +9,7 @@ import { initDb, closeDb } from './db.js';
|
|
|
9
9
|
import { fullScan, startWatcher } from './sync.js';
|
|
10
10
|
import apiRoutes from './routes/api.js';
|
|
11
11
|
import pipelineRoutes from './routes/pipeline.js';
|
|
12
|
+
import ragRoutes from './routes/rag.js';
|
|
12
13
|
import { authMiddleware } from './auth.js';
|
|
13
14
|
import { getOrCreateToken, getTokenPath } from './token-store.js';
|
|
14
15
|
import { reapZombieSessions } from './zombie-reaper.js';
|
|
@@ -298,6 +299,7 @@ async function main() {
|
|
|
298
299
|
app.use(express.static(PUBLIC_DIR, { setHeaders: noStoreHeaders, etag: false, lastModified: false }));
|
|
299
300
|
app.use('/api', buildRateLimiter(), authMiddleware(), apiRoutes);
|
|
300
301
|
app.use('/api/pipeline', authMiddleware(), pipelineRoutes);
|
|
302
|
+
app.use('/api/rag', authMiddleware(), ragRoutes);
|
|
301
303
|
|
|
302
304
|
// SPA fallback: serve index.html for non-API, non-static routes
|
|
303
305
|
app.get('/{*splat}', (_req, res) => {
|
|
@@ -173,9 +173,16 @@ const ragStubPlugin = {
|
|
|
173
173
|
// KJC-TSK-0441 — RAG chokidar watcher.
|
|
174
174
|
startWatcher: notAvailable, readPidFile: notAvailable, clearPidFile: notAvailable,
|
|
175
175
|
isPidAlive: notAvailable, writePidFile: notAvailable,
|
|
176
|
-
// KJC-TSK-0442 — RAG embedders cloud + factory.
|
|
176
|
+
// KJC-TSK-0442 / KJC-TSK-0446 — RAG embedders cloud + factory.
|
|
177
177
|
makeEmbedder: notAvailable, OpenAIEmbedder: notAvailable, VoyageEmbedder: notAvailable,
|
|
178
178
|
OpenAIEmbedderError: notAvailable, VoyageEmbedderError: notAvailable, PROVIDERS: notAvailable,
|
|
179
|
+
CohereEmbedder: notAvailable, CohereEmbedderError: notAvailable,
|
|
180
|
+
MistralEmbedder: notAvailable, MistralEmbedderError: notAvailable,
|
|
181
|
+
ONNXEmbedder: notAvailable, ONNXEmbedderError: notAvailable,
|
|
182
|
+
// KJC-TSK-0448 — RAG --where metadata filter.
|
|
183
|
+
parseWhere: notAvailable, buildWhereSql: notAvailable,
|
|
184
|
+
// KJC-TSK-0449 — RAG cross-encoder rerank.
|
|
185
|
+
rerank: notAvailable, RerankError: notAvailable, _resetPipeline: notAvailable,
|
|
179
186
|
default: notAvailable,
|
|
180
187
|
};
|
|
181
188
|
`,
|
package/src/cli/register-meta.js
CHANGED
|
@@ -117,6 +117,9 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
117
117
|
.option("--project <slug>", "Filter by project slug. Pass 'all' to query across every indexed project. Default: cwd basename")
|
|
118
118
|
.option("--mode <mode>", "hybrid (default) | semantic (cosine only) | keyword (BM25 only)", "hybrid")
|
|
119
119
|
.option("--alpha <n>", "Hybrid weight for semantic component (0..1). Default 0.6", "0.6")
|
|
120
|
+
.option("--where <clause>", "Metadata filter, e.g. 'symbol=loadConfig' or 'hu_id=HU-003 AND kind=plan'")
|
|
121
|
+
.option("--rerank", "Apply cross-encoder rerank to top-K (needs @huggingface/transformers, slower but more precise)")
|
|
122
|
+
.option("--rerank-model <name>", "Cross-encoder model id. Default: Xenova/ms-marco-MiniLM-L-6-v2")
|
|
120
123
|
.option("--json", "Output the hits as JSON")
|
|
121
124
|
.action(async (text, flags) => {
|
|
122
125
|
await withConfig(pkgVersion, "rag-query", flags, async ({ config, logger }) => {
|
package/src/commands/rag.js
CHANGED
|
@@ -57,7 +57,9 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
|
|
|
57
57
|
}
|
|
58
58
|
const mode = flags.mode || "hybrid";
|
|
59
59
|
const alpha = Math.max(0, Math.min(1, Number(flags.alpha) || 0.6));
|
|
60
|
-
const
|
|
60
|
+
const where = flags.where || null;
|
|
61
|
+
const rerankOpts = flags.rerank ? { model: flags.rerankModel } : null;
|
|
62
|
+
const hits = await query(db, makeEmbedder(config), text, { topK, scope, project, mode, alpha, where, rerankOpts });
|
|
61
63
|
if (flags.json) {
|
|
62
64
|
process.stdout.write(`${JSON.stringify(hits)}\n`);
|
|
63
65
|
} else {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// KJC-TSK-0446 — Cohere embed-v3 adapter. Cohere returns
|
|
2
|
+
// { embeddings: { float: [[...]] } } when `embedding_types: ["float"]`
|
|
3
|
+
// is requested, or { embeddings: [[...]] } in legacy v1. We extract the
|
|
4
|
+
// modern shape with a v1 fallback.
|
|
5
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
6
|
+
|
|
7
|
+
const DEFAULTS = { url: "https://api.cohere.com/v2/embed", model: "embed-multilingual-v3.0", dim: 1024, timeoutMs: 30000 };
|
|
8
|
+
|
|
9
|
+
export class CohereEmbedderError extends Error {
|
|
10
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "CohereEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export class CohereEmbedder {
|
|
14
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_COHERE_KEY, model = process.env.KJ_COHERE_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
15
|
+
// KJC architecture invariant: scoped env var (KJ_COHERE_KEY), not the
|
|
16
|
+
// generic COHERE_API_KEY — same rule as OpenAI/Voyage adapters.
|
|
17
|
+
if (!apiKey) throw new CohereEmbedderError("Cohere embedder requires an api_key (config.rag.embedder.api_key or KJ_COHERE_KEY env)");
|
|
18
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
19
|
+
}
|
|
20
|
+
async embed(text) {
|
|
21
|
+
return cloudEmbed(this, text, CohereEmbedderError, {
|
|
22
|
+
provider: "Cohere",
|
|
23
|
+
body: { model: this.model, texts: [text], input_type: "search_document", embedding_types: ["float"] },
|
|
24
|
+
extract: (b) => b?.embeddings?.float?.[0] || b?.embeddings?.[0],
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new CohereEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
28
|
+
}
|
|
@@ -1,12 +1,22 @@
|
|
|
1
|
-
// KJC-TSK-0442 — Embedder factory. Selects provider from
|
|
2
|
-
// config.rag.embedder.provider (ollama | openai | voyage
|
|
3
|
-
// Each provider has its own dim default; the factory wires
|
|
4
|
-
// caller does not need to know.
|
|
1
|
+
// KJC-TSK-0442 / KJC-TSK-0446 — Embedder factory. Selects provider from
|
|
2
|
+
// config.rag.embedder.provider (ollama | openai | voyage | cohere | mistral;
|
|
3
|
+
// default ollama). Each provider has its own dim default; the factory wires
|
|
4
|
+
// it so the caller does not need to know.
|
|
5
5
|
import { OllamaEmbedder } from "../embedder.js";
|
|
6
6
|
import { OpenAIEmbedder } from "./openai.js";
|
|
7
7
|
import { VoyageEmbedder } from "./voyage.js";
|
|
8
|
+
import { CohereEmbedder } from "./cohere.js";
|
|
9
|
+
import { MistralEmbedder } from "./mistral.js";
|
|
10
|
+
import { ONNXEmbedder } from "./onnx.js";
|
|
8
11
|
|
|
9
|
-
const PROVIDERS = {
|
|
12
|
+
const PROVIDERS = {
|
|
13
|
+
ollama: { cls: OllamaEmbedder, dim: 768 },
|
|
14
|
+
openai: { cls: OpenAIEmbedder, dim: 1536 },
|
|
15
|
+
voyage: { cls: VoyageEmbedder, dim: 1024 },
|
|
16
|
+
cohere: { cls: CohereEmbedder, dim: 1024 },
|
|
17
|
+
mistral: { cls: MistralEmbedder, dim: 1024 },
|
|
18
|
+
onnx: { cls: ONNXEmbedder, dim: 384 },
|
|
19
|
+
};
|
|
10
20
|
|
|
11
21
|
export function makeEmbedder(config = {}) {
|
|
12
22
|
const cfg = config?.rag?.embedder || {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// KJC-TSK-0446 — Mistral AI embeddings adapter. EU-hosted, useful for
|
|
2
|
+
// users with GDPR constraints that prefer not to send chunks to US
|
|
3
|
+
// endpoints. Single model today (`mistral-embed`, 1024 dim).
|
|
4
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
5
|
+
|
|
6
|
+
const DEFAULTS = { url: "https://api.mistral.ai/v1/embeddings", model: "mistral-embed", dim: 1024, timeoutMs: 30000 };
|
|
7
|
+
|
|
8
|
+
export class MistralEmbedderError extends Error {
|
|
9
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "MistralEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export class MistralEmbedder {
|
|
13
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_MISTRAL_KEY, model = process.env.KJ_MISTRAL_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
14
|
+
// KJC architecture invariant: scoped env var (KJ_MISTRAL_KEY).
|
|
15
|
+
if (!apiKey) throw new MistralEmbedderError("Mistral embedder requires an api_key (config.rag.embedder.api_key or KJ_MISTRAL_KEY env)");
|
|
16
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
17
|
+
}
|
|
18
|
+
async embed(text) {
|
|
19
|
+
return cloudEmbed(this, text, MistralEmbedderError, {
|
|
20
|
+
provider: "Mistral",
|
|
21
|
+
body: { model: this.model, input: [text] },
|
|
22
|
+
extract: (b) => b?.data?.[0]?.embedding,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new MistralEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
26
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// KJC-TSK-0447 — ONNX local embedder via @huggingface/transformers
|
|
2
|
+
// (formerly @xenova/transformers). Runs sentence-transformer models
|
|
3
|
+
// directly in Node — no API key, no Docker, no Ollama.
|
|
4
|
+
//
|
|
5
|
+
// First call downloads the model weights to ~/.cache/huggingface/ (~80 MB
|
|
6
|
+
// for the default MiniLM). Subsequent calls reuse the cache, so the steady
|
|
7
|
+
// state is fully offline.
|
|
8
|
+
//
|
|
9
|
+
// Default model: `Xenova/all-MiniLM-L6-v2` (384 dim, ~80 MB, fast).
|
|
10
|
+
// Higher-quality alternative: `Xenova/jina-embeddings-v2-base-en`
|
|
11
|
+
// (768 dim, ~260 MB, slower but better for retrieval).
|
|
12
|
+
|
|
13
|
+
const DEFAULTS = { model: "Xenova/all-MiniLM-L6-v2", dim: 384, pooling: "mean", normalize: true };
|
|
14
|
+
|
|
15
|
+
export class ONNXEmbedderError extends Error {
|
|
16
|
+
constructor(message, { cause } = {}) { super(message); this.name = "ONNXEmbedderError"; if (cause) this.cause = cause; }
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function loadTransformers() {
|
|
20
|
+
// Prefer the modern @huggingface/transformers (post-2024 rebrand);
|
|
21
|
+
// fall back to @xenova/transformers for users still on the legacy package.
|
|
22
|
+
// Both are optional peer deps — Karajan does not ship them by default to
|
|
23
|
+
// keep the install size small (~500 MB with WASM + weights). The user
|
|
24
|
+
// installs whichever they prefer when they opt into provider: onnx.
|
|
25
|
+
/* eslint-disable import-x/no-unresolved */
|
|
26
|
+
try { return await import("@huggingface/transformers"); }
|
|
27
|
+
catch (e1) {
|
|
28
|
+
try { return await import("@xenova/transformers"); }
|
|
29
|
+
catch (e2) {
|
|
30
|
+
throw new ONNXEmbedderError(
|
|
31
|
+
"ONNX embedder requires @huggingface/transformers (or legacy @xenova/transformers). Install with `npm install @huggingface/transformers`.",
|
|
32
|
+
{ cause: e2 },
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/* eslint-enable import-x/no-unresolved */
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export class ONNXEmbedder {
|
|
40
|
+
constructor({ model = process.env.KJ_ONNX_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, pooling = DEFAULTS.pooling, normalize = DEFAULTS.normalize } = {}) {
|
|
41
|
+
Object.assign(this, { model, dim, pooling, normalize, _pipe: null });
|
|
42
|
+
}
|
|
43
|
+
async _ensurePipeline() {
|
|
44
|
+
if (this._pipe) return this._pipe;
|
|
45
|
+
const transformers = await loadTransformers();
|
|
46
|
+
this._pipe = await transformers.pipeline("feature-extraction", this.model);
|
|
47
|
+
return this._pipe;
|
|
48
|
+
}
|
|
49
|
+
async embed(text) {
|
|
50
|
+
if (typeof text !== "string" || text.length === 0) throw new ONNXEmbedderError("embed: text must be a non-empty string");
|
|
51
|
+
const pipe = await this._ensurePipeline();
|
|
52
|
+
const out = await pipe(text, { pooling: this.pooling, normalize: this.normalize });
|
|
53
|
+
const arr = Array.from(out.data || []);
|
|
54
|
+
if (arr.length !== this.dim) throw new ONNXEmbedderError(`ONNX dim mismatch: got ${arr.length}, expected ${this.dim} for model ${this.model}`);
|
|
55
|
+
return Float32Array.from(arr);
|
|
56
|
+
}
|
|
57
|
+
async embedBatch(texts) {
|
|
58
|
+
if (!Array.isArray(texts)) throw new ONNXEmbedderError("embedBatch: texts must be an array");
|
|
59
|
+
const out = [];
|
|
60
|
+
for (const t of texts) out.push(await this.embed(t));
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// KJC-TSK-0449 — Cross-encoder rerank stage. Re-scores the top-N hits
|
|
2
|
+
// from the hybrid retriever using a (query, passage) cross-encoder.
|
|
3
|
+
//
|
|
4
|
+
// Cross-encoders are slower than the bi-encoder embedders (they jointly
|
|
5
|
+
// encode query + passage instead of caching the passage), but they are
|
|
6
|
+
// substantially more precise for the final top-K ranking. Karajan calls
|
|
7
|
+
// the reranker only on the post-fusion candidates (≤topK*2), so the
|
|
8
|
+
// added latency is bounded.
|
|
9
|
+
//
|
|
10
|
+
// Model is loaded dynamically via @huggingface/transformers (same dep as
|
|
11
|
+
// the ONNX embedder, KJC-TSK-0447). Default: `Xenova/ms-marco-MiniLM-L-6-v2`,
|
|
12
|
+
// the de-facto standard sentence-transformers reranker.
|
|
13
|
+
|
|
14
|
+
const DEFAULT_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
15
|
+
|
|
16
|
+
export class RerankError extends Error {
|
|
17
|
+
constructor(message, { cause } = {}) { super(message); this.name = "RerankError"; if (cause) this.cause = cause; }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function loadTransformers() {
|
|
21
|
+
/* eslint-disable import-x/no-unresolved */
|
|
22
|
+
try { return await import("@huggingface/transformers"); }
|
|
23
|
+
catch (e1) {
|
|
24
|
+
try { return await import("@xenova/transformers"); }
|
|
25
|
+
catch (e2) {
|
|
26
|
+
throw new RerankError(
|
|
27
|
+
"rerank requires @huggingface/transformers (or legacy @xenova/transformers). Install with `npm install @huggingface/transformers`.",
|
|
28
|
+
{ cause: e2 },
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/* eslint-enable import-x/no-unresolved */
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Lazy singleton — first call downloads weights (~80 MB), every subsequent
|
|
37
|
+
* call reuses the same pipeline instance.
|
|
38
|
+
*/
|
|
39
|
+
let _pipelinePromise = null;
|
|
40
|
+
function getPipeline(modelName) {
|
|
41
|
+
if (!_pipelinePromise) _pipelinePromise = (async () => {
|
|
42
|
+
const t = await loadTransformers();
|
|
43
|
+
// text-classification pipeline returns the cross-encoder relevance score
|
|
44
|
+
// for a `[query, passage]` text pair when applied to a CE model.
|
|
45
|
+
return t.pipeline("text-classification", modelName);
|
|
46
|
+
})();
|
|
47
|
+
return _pipelinePromise;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Reset for tests. */
|
|
51
|
+
export function _resetPipeline() { _pipelinePromise = null; }
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Rerank `hits` against `queryText` using the cross-encoder. Returns a NEW
|
|
55
|
+
* array sorted by descending rerank score (best first), `score` mutated
|
|
56
|
+
* so the existing downstream code (kindBoost, slicing) keeps working.
|
|
57
|
+
*
|
|
58
|
+
* Each input hit is treated as a [query, hit.text] pair. Truncates the
|
|
59
|
+
* passage to `maxChars` to avoid blowing past the CE model's context
|
|
60
|
+
* window — most CE models handle 512 tokens, ~2000 chars is safe.
|
|
61
|
+
*
|
|
62
|
+
* If `pipelineFn` is injected (tests), it bypasses model loading.
|
|
63
|
+
*/
|
|
64
|
+
export async function rerank(queryText, hits, { model = process.env.KJ_RERANK_MODEL || DEFAULT_MODEL, maxChars = 2000, pipelineFn = null } = {}) {
|
|
65
|
+
if (!queryText || typeof queryText !== "string") throw new RerankError("rerank: queryText must be a non-empty string");
|
|
66
|
+
if (!Array.isArray(hits)) throw new RerankError("rerank: hits must be an array");
|
|
67
|
+
if (hits.length === 0) return hits;
|
|
68
|
+
const pipe = pipelineFn || await getPipeline(model);
|
|
69
|
+
const pairs = hits.map((h) => ({ text: queryText, text_pair: String(h.text || "").slice(0, maxChars) }));
|
|
70
|
+
const results = await pipe(pairs);
|
|
71
|
+
return hits
|
|
72
|
+
.map((h, i) => ({ ...h, _rerank: Number(results[i]?.score) || 0, score: -Number(results[i]?.score) || 0 }))
|
|
73
|
+
.sort((a, b) => a.score - b.score);
|
|
74
|
+
}
|
package/src/rag/retriever.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
|
|
2
2
|
import { searchSimilar, searchBM25 } from "./vec-store.js";
|
|
3
|
+
import { parseWhere, buildWhereSql } from "./where-parser.js";
|
|
4
|
+
import { rerank } from "./rerank.js";
|
|
3
5
|
|
|
4
6
|
const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
|
|
5
7
|
|
|
@@ -39,14 +41,21 @@ function fuseHits(semantic, keyword, alpha, mode) {
|
|
|
39
41
|
return list;
|
|
40
42
|
}
|
|
41
43
|
|
|
42
|
-
export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6 } = {}) {
|
|
44
|
+
export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6, where = null, rerankOpts = null } = {}) {
|
|
43
45
|
if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
|
|
44
46
|
const fetchK = Math.min(50, topK * 2);
|
|
45
47
|
const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
|
|
48
|
+
// KJC-TSK-0448 — metadata filter. `where` is a string like "symbol=Foo AND
|
|
49
|
+
// hu_id=HU-003"; we parse once and reuse the SQL fragment across both
|
|
50
|
+
// semantic and keyword searches.
|
|
51
|
+
const parsed = parseWhere(where);
|
|
52
|
+
if (!parsed.ok) throw new Error(`query: ${parsed.error}`);
|
|
53
|
+
const { sql: whereSql, params: whereParams } = buildWhereSql(parsed.clauses);
|
|
54
|
+
const opts = { kind: scopeKind, project, whereSql, whereParams };
|
|
46
55
|
const wantSemantic = mode !== "keyword";
|
|
47
56
|
const wantKeyword = mode !== "semantic";
|
|
48
|
-
const semanticHits = wantSemantic ? searchSimilar(db, await embedder.embed(text), fetchK,
|
|
49
|
-
const keywordHits = wantKeyword ? searchBM25(db, text, fetchK,
|
|
57
|
+
const semanticHits = wantSemantic ? searchSimilar(db, await embedder.embed(text), fetchK, opts) : [];
|
|
58
|
+
const keywordHits = wantKeyword ? searchBM25(db, text, fetchK, opts) : [];
|
|
50
59
|
const raw = fuseHits(semanticHits, keywordHits, alpha, mode);
|
|
51
60
|
if (raw.length === 0) return [];
|
|
52
61
|
const boostSources = shouldBoostSources(text);
|
|
@@ -55,6 +64,11 @@ export async function query(db, embedder, text, { topK = 5, scope = "all", kindB
|
|
|
55
64
|
const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
|
|
56
65
|
return { ...r, score: r.distance - kindB - sourceB };
|
|
57
66
|
}).sort((a, b) => a.score - b.score).slice(0, topK);
|
|
67
|
+
// KJC-TSK-0449 — optional cross-encoder rerank. When `rerankOpts` is set,
|
|
68
|
+
// we re-score the topK survivors with a (query, passage) cross-encoder;
|
|
69
|
+
// the kind+source boost has already been applied, so the rerank acts as
|
|
70
|
+
// a finer-grained quality lever on top.
|
|
71
|
+
if (rerankOpts) return await rerank(text, reranked, rerankOpts);
|
|
58
72
|
return reranked;
|
|
59
73
|
}
|
|
60
74
|
|
package/src/rag/vec-store.js
CHANGED
|
@@ -106,7 +106,7 @@ export function insertChunk(db, { source, kind, text, metadata = null, embedding
|
|
|
106
106
|
* text, metadata, distance }]` for the topK best matches; lower
|
|
107
107
|
* distance = closer. Returns `[]` when the store is empty.
|
|
108
108
|
*/
|
|
109
|
-
export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null } = {}) {
|
|
109
|
+
export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null, whereSql = "", whereParams = [] } = {}) {
|
|
110
110
|
const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
|
|
111
111
|
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
112
112
|
const projectClause = project ? "AND c.project_slug = ?" : "";
|
|
@@ -114,12 +114,13 @@ export function searchSimilar(db, embedding, topK = 5, { kind = null, project =
|
|
|
114
114
|
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, v.distance
|
|
115
115
|
FROM vec_chunks v
|
|
116
116
|
JOIN chunks c ON c.id = v.rowid
|
|
117
|
-
WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}
|
|
117
|
+
WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}${whereSql}
|
|
118
118
|
ORDER BY v.distance
|
|
119
119
|
`;
|
|
120
120
|
const params = [Buffer.from(buf.buffer), topK];
|
|
121
121
|
if (kind) params.push(kind);
|
|
122
122
|
if (project) params.push(project);
|
|
123
|
+
if (whereParams.length) params.push(...whereParams);
|
|
123
124
|
const rows = db.prepare(sql).all(...params);
|
|
124
125
|
return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
|
|
125
126
|
}
|
|
@@ -163,7 +164,7 @@ export function countChunks(db, { kind = null } = {}) {
|
|
|
163
164
|
* relevance (lower bm25 = better in SQLite FTS5). `query` is sanitized for
|
|
164
165
|
* the FTS5 grammar: special chars are stripped, words split on whitespace.
|
|
165
166
|
*/
|
|
166
|
-
export function searchBM25(db, queryText, topK = 10, { kind = null, project = null } = {}) {
|
|
167
|
+
export function searchBM25(db, queryText, topK = 10, { kind = null, project = null, whereSql = "", whereParams = [] } = {}) {
|
|
167
168
|
const cleaned = String(queryText || "").replace(/["'()]/g, " ").split(/\s+/).filter(Boolean).join(" OR ");
|
|
168
169
|
if (!cleaned) return [];
|
|
169
170
|
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
@@ -171,12 +172,13 @@ export function searchBM25(db, queryText, topK = 10, { kind = null, project = nu
|
|
|
171
172
|
const sql = `
|
|
172
173
|
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, bm25(chunks_fts) AS bm25
|
|
173
174
|
FROM chunks_fts JOIN chunks c ON c.id = chunks_fts.rowid
|
|
174
|
-
WHERE chunks_fts MATCH ? ${kindClause} ${projectClause}
|
|
175
|
+
WHERE chunks_fts MATCH ? ${kindClause} ${projectClause}${whereSql}
|
|
175
176
|
ORDER BY bm25 LIMIT ?
|
|
176
177
|
`;
|
|
177
178
|
const params = [cleaned];
|
|
178
179
|
if (kind) params.push(kind);
|
|
179
180
|
if (project) params.push(project);
|
|
181
|
+
if (whereParams.length) params.push(...whereParams);
|
|
180
182
|
params.push(topK);
|
|
181
183
|
try {
|
|
182
184
|
const rows = db.prepare(sql).all(...params);
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// KJC-TSK-0448 — minimal --where filter parser for metadata-aware RAG
|
|
2
|
+
// queries. Grammar (intentionally small):
|
|
3
|
+
//
|
|
4
|
+
// clause := pair ( "AND" pair )*
|
|
5
|
+
// pair := IDENT "=" VALUE
|
|
6
|
+
// IDENT := [A-Za-z_][A-Za-z0-9_.-]*
|
|
7
|
+
// VALUE := unquoted token | "quoted string"
|
|
8
|
+
//
|
|
9
|
+
// Examples:
|
|
10
|
+
// symbol=loadConfig
|
|
11
|
+
// hu_id=HU-003 AND kind=plan
|
|
12
|
+
// "headingPath.0"=Architecture
|
|
13
|
+
//
|
|
14
|
+
// Returns { ok: true, clauses: [["symbol","loadConfig"], ...] } or
|
|
15
|
+
// { ok: false, error: "...explanation..." }. The retriever turns each
|
|
16
|
+
// clause into a `json_extract(c.metadata, '$.<key>') = ?` SQL fragment;
|
|
17
|
+
// the `kind` key is special-cased to filter the column directly.
|
|
18
|
+
|
|
19
|
+
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_.-]*$/;
|
|
20
|
+
|
|
21
|
+
export function parseWhere(input) {
|
|
22
|
+
if (input == null) return { ok: true, clauses: [] };
|
|
23
|
+
if (typeof input !== "string") return { ok: false, error: "where: input must be a string" };
|
|
24
|
+
const s = input.trim();
|
|
25
|
+
if (!s) return { ok: true, clauses: [] };
|
|
26
|
+
const parts = s.split(/\s+AND\s+/i);
|
|
27
|
+
const clauses = [];
|
|
28
|
+
for (const p of parts) {
|
|
29
|
+
const eq = p.indexOf("=");
|
|
30
|
+
if (eq <= 0) return { ok: false, error: `where: bad pair "${p}" — expected key=value` };
|
|
31
|
+
const key = p.slice(0, eq).trim().replace(/^"|"$/g, "");
|
|
32
|
+
let val = p.slice(eq + 1).trim();
|
|
33
|
+
if (!IDENT_RE.test(key)) return { ok: false, error: `where: invalid key "${key}"` };
|
|
34
|
+
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) val = val.slice(1, -1);
|
|
35
|
+
if (val.length === 0) return { ok: false, error: `where: empty value for key "${key}"` };
|
|
36
|
+
clauses.push([key, val]);
|
|
37
|
+
}
|
|
38
|
+
return { ok: true, clauses };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Build a `(sqlFragment, params)` pair to append to a chunks-table query.
|
|
43
|
+
* `kind` filters the column directly; everything else goes through json_extract.
|
|
44
|
+
*/
|
|
45
|
+
export function buildWhereSql(clauses) {
|
|
46
|
+
if (!clauses?.length) return { sql: "", params: [] };
|
|
47
|
+
const fragments = [];
|
|
48
|
+
const params = [];
|
|
49
|
+
for (const [k, v] of clauses) {
|
|
50
|
+
if (k === "kind") { fragments.push("c.kind = ?"); params.push(v); }
|
|
51
|
+
else { fragments.push(`json_extract(c.metadata, '$.${k}') = ?`); params.push(v); }
|
|
52
|
+
}
|
|
53
|
+
return { sql: ` AND ${fragments.join(" AND ")}`, params };
|
|
54
|
+
}
|