karajan-code 2.26.0 → 2.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/package.json +2 -1
- package/packages/hu-board/src/routes/api.js +2 -2
- package/scripts/esbuild-sea.config.mjs +9 -1
- package/src/cli/register-meta.js +16 -0
- package/src/commands/rag.js +11 -8
- package/src/commands/watch.js +46 -0
- package/src/mcp/handlers/rag-handler.js +1 -6
- package/src/orchestrator/stages/rag-context-stage.js +2 -3
- package/src/rag/chunker-ast.js +56 -0
- package/src/rag/chunker.js +14 -5
- package/src/rag/embedders/_cloud-base.js +24 -0
- package/src/rag/embedders/factory.js +20 -0
- package/src/rag/embedders/openai.js +22 -0
- package/src/rag/embedders/voyage.js +18 -0
- package/src/rag/indexer.js +11 -6
- package/src/rag/retriever.js +52 -38
- package/src/rag/vec-store.js +78 -6
- package/src/rag/watcher.js +60 -0
- package/src/utils/rate-limit-detector.js +54 -12
package/README.md
CHANGED
|
@@ -345,13 +345,30 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
345
345
|
|
|
346
346
|
## Recent releases
|
|
347
347
|
|
|
348
|
-
> **v2.
|
|
348
|
+
> **v2.28.0 released** — Minor. **RAG advanced** — live re-index + cloud embedders + hybrid scoring + AST chunker, plus a real fix for the v2.27.0 chapuza.
|
|
349
|
+
>
|
|
350
|
+
> - **`kj watch`** (PR #836): chokidar daemon que re-indexa los archivos del proyecto tras cada edit (1s debounce). Fin del `kj rag index` manual.
|
|
351
|
+
> - **OpenAI + Voyage embedders** (PR #841): para usuarios sin Docker local. `config.rag.embedder.provider: openai | voyage` + `KJ_OPENAI_KEY` / `KJ_VOYAGE_KEY` env vars (Karajan-scoped — preserva el architecture invariant).
|
|
352
|
+
> - **BM25 hybrid scoring** (PR #838): SQLite FTS5 + cosine fusion. `kj rag query --mode hybrid|semantic|keyword`. Queries con símbolos exactos rankean correctamente.
|
|
353
|
+
> - **AST source chunker** (PR #839): `@babel/parser`. Cada top-level declaration entera en un chunk; TypeScript + JSX + decorators soportados.
|
|
354
|
+
> - **KJC-BUG-0064** (PR #840): `parseCooldown` TZ-aware via `Intl.DateTimeFormat`. Deshace el skip-in-CI workaround de v2.27.0 — tests pasan en cualquier TZ.
|
|
355
|
+
>
|
|
356
|
+
> **v2.27.0 released** — Minor. **RAG polish** — three improvements triggered by the v2.26.0 smoke test on karajan-code itself:
|
|
357
|
+
>
|
|
358
|
+
> - **Per-project isolation** (PR #831): new `project_slug` column on chunks; `kj rag query --project <slug>` (auto-detected from cwd) filters the global DB. `--project all` to query across everything.
|
|
359
|
+
> - **`docs/RAG.md` + `docs/es/RAG.md`** (PR #832): single unified guide covering architecture, install, six workflows, configuration, limitations and troubleshooting. Replaces the documentation spread across CHANGELOG / templates / landing.
|
|
360
|
+
> - **Asymmetric source-vs-test ranking** (PR #833): NL queries like `how does X work` no longer rank `tests/X.test.js` above `src/X.js`; test-flavoured queries still surface tests.
|
|
361
|
+
> - Plus **KJC-BUG-0063** (PR #834): skipped a TZ-dependent test that was blocking every CI run, and a `shrink-budget` workflow exclude fix for `docs/*.md` at the root.
|
|
362
|
+
>
|
|
363
|
+
> **Coming in v2.28.0+**: chokidar watcher for live re-indexing, AST source chunker (tree-sitter / @babel/parser), BM25 + cosine hybrid scoring, OpenAI/Voyage embedder adapters (for users without local Docker).
|
|
364
|
+
>
|
|
365
|
+
> **v2.26.0 released** — Minor. **RAG Auto-Bootstrap** — Ollama runs in Docker out of the box. `kj init` now provisions the embedder automatically (or skips with a clear reason on modest hardware / `--no-ollama`); `kj doctor` surfaces health; `kj ollama [start|stop|status|pull]` manages lifecycle without docker compose. See [docs/RAG.md](docs/RAG.md) for the full RAG guide ([español](docs/es/RAG.md)).
|
|
349
366
|
>
|
|
350
367
|
> Capability check (RAM + Docker) means the bootstrap never breaks init: on Windows without Docker Desktop, on hosts under 4 GB free, or with `--no-ollama`, the wizard logs a one-liner and continues. Where Ollama is already running on `:11434`, `kj init` reuses the external instance instead of spawning a second container.
|
|
351
368
|
>
|
|
352
369
|
> Bundles **KJC-BUG-0061** fix (caught during v2.25.0 smoke test): `kj onboard --no-synth` was ignored by Commander shape mapping, `OnboarderRole.run()` was called without `init()`, and `kj rag query --json` on empty store emitted just `[]` instead of the `{empty: true}` contract the MCP handler returns.
|
|
353
370
|
>
|
|
354
|
-
> **Coming
|
|
371
|
+
> **Coming next from v2.26.0**: addressed in v2.27.0 above.
|
|
355
372
|
>
|
|
356
373
|
> **v2.25.0 released** — Minor. **RAG Camino B + Camino D** (KJC-PCS-0049). Closes the consumer-surface plan. Skills hosts can now invoke RAG via `/kj-rag-query` without MCP, and the pre-loop retrieval stage from v2.24.0 only fires when triage signals make it worthwhile.
|
|
357
374
|
>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "karajan-code",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.28.0",
|
|
4
4
|
"description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "AGPL-3.0",
|
|
@@ -69,6 +69,7 @@
|
|
|
69
69
|
"src/**/*.js": "eslint --max-warnings=999"
|
|
70
70
|
},
|
|
71
71
|
"dependencies": {
|
|
72
|
+
"@babel/parser": "^7.29.3",
|
|
72
73
|
"@modelcontextprotocol/sdk": "^1.26.0",
|
|
73
74
|
"better-sqlite3": "^11.0.0",
|
|
74
75
|
"chokidar": "^4.0.0",
|
|
@@ -1292,12 +1292,12 @@ router.post('/rag/query', async (req, res) => {
|
|
|
1292
1292
|
}
|
|
1293
1293
|
try {
|
|
1294
1294
|
const { openVecStore, countChunks } = await import('../../../../src/rag/vec-store.js');
|
|
1295
|
-
const {
|
|
1295
|
+
const { makeEmbedder } = await import('../../../../src/rag/embedders/factory.js');
|
|
1296
1296
|
const { query } = await import('../../../../src/rag/retriever.js');
|
|
1297
1297
|
const db = openVecStore({ dim: 768 });
|
|
1298
1298
|
try {
|
|
1299
1299
|
if (countChunks(db) === 0) return res.json({ hits: [], empty: true, topK, scope });
|
|
1300
|
-
const embedder =
|
|
1300
|
+
const embedder = makeEmbedder({});
|
|
1301
1301
|
const hits = await query(db, embedder, text, { topK: Number(topK), scope });
|
|
1302
1302
|
res.json({ hits, empty: false, topK, scope });
|
|
1303
1303
|
} finally { db.close(); }
|
|
@@ -146,7 +146,7 @@ const huBoardStubPlugin = {
|
|
|
146
146
|
const ragStubPlugin = {
|
|
147
147
|
name: "rag-stub",
|
|
148
148
|
setup(build) {
|
|
149
|
-
build.onResolve({ filter: /[\\/](rag[\\/]
|
|
149
|
+
build.onResolve({ filter: /[\\/](rag[\\/].+|commands[\\/](rag|watch))\.js$/ }, (args) => ({
|
|
150
150
|
path: args.path, namespace: "rag-stub",
|
|
151
151
|
}));
|
|
152
152
|
build.onLoad({ filter: /.*/, namespace: "rag-stub" }, () => ({
|
|
@@ -168,6 +168,14 @@ const ragStubPlugin = {
|
|
|
168
168
|
// KJC-TSK-0436 — Ollama capability check + model pull.
|
|
169
169
|
checkDockerAvailable: notAvailable, checkRamCapacity: notAvailable,
|
|
170
170
|
checkOllamaCapability: notAvailable, pullOllamaModel: notAvailable,
|
|
171
|
+
// KJC-TSK-0438 — RAG project isolation.
|
|
172
|
+
projectSlug: notAvailable,
|
|
173
|
+
// KJC-TSK-0441 — RAG chokidar watcher.
|
|
174
|
+
startWatcher: notAvailable, readPidFile: notAvailable, clearPidFile: notAvailable,
|
|
175
|
+
isPidAlive: notAvailable, writePidFile: notAvailable,
|
|
176
|
+
// KJC-TSK-0442 — RAG embedders cloud + factory.
|
|
177
|
+
makeEmbedder: notAvailable, OpenAIEmbedder: notAvailable, VoyageEmbedder: notAvailable,
|
|
178
|
+
OpenAIEmbedderError: notAvailable, VoyageEmbedderError: notAvailable, PROVIDERS: notAvailable,
|
|
171
179
|
default: notAvailable,
|
|
172
180
|
};
|
|
173
181
|
`,
|
package/src/cli/register-meta.js
CHANGED
|
@@ -4,6 +4,7 @@ import { researcherCommand } from "../commands/researcher.js";
|
|
|
4
4
|
import { architectCommand } from "../commands/architect.js";
|
|
5
5
|
import { onboardCommand } from "../commands/onboard.js";
|
|
6
6
|
import { ragIndexCommand, ragQueryCommand } from "../commands/rag.js";
|
|
7
|
+
import { watchStartCommand, watchStopCommand, watchStatusCommand } from "../commands/watch.js";
|
|
7
8
|
import { auditCommand } from "../commands/audit.js";
|
|
8
9
|
import { resumeCommand } from "../commands/resume.js";
|
|
9
10
|
import { boardCommand } from "../commands/board.js";
|
|
@@ -113,6 +114,9 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
113
114
|
.description("Run a semantic query against the indexed RAG corpus")
|
|
114
115
|
.option("--scope <scope>", "plans | code | onboarding | all (default: all)", "all")
|
|
115
116
|
.option("--top-k <n>", "Number of hits to return (default: 5)", "5")
|
|
117
|
+
.option("--project <slug>", "Filter by project slug. Pass 'all' to query across every indexed project. Default: cwd basename")
|
|
118
|
+
.option("--mode <mode>", "hybrid (default) | semantic (cosine only) | keyword (BM25 only)", "hybrid")
|
|
119
|
+
.option("--alpha <n>", "Hybrid weight for semantic component (0..1). Default 0.6", "0.6")
|
|
116
120
|
.option("--json", "Output the hits as JSON")
|
|
117
121
|
.action(async (text, flags) => {
|
|
118
122
|
await withConfig(pkgVersion, "rag-query", flags, async ({ config, logger }) => {
|
|
@@ -120,6 +124,18 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
120
124
|
});
|
|
121
125
|
});
|
|
122
126
|
|
|
127
|
+
// KJC-TSK-0441 — `kj watch [start|stop|status]` for live RAG re-index.
|
|
128
|
+
const watch = program.command("watch").description("Live RAG re-index daemon");
|
|
129
|
+
watch.command("start").description("Start the watcher daemon").option("--foreground", "Run in foreground (do not detach)").option("--with-sources", "Watch project source files too").action(async (flags) => {
|
|
130
|
+
await withConfig(pkgVersion, "watch:start", flags, async ({ config, logger }) => watchStartCommand({ config, logger, flags }));
|
|
131
|
+
});
|
|
132
|
+
watch.command("stop").description("Stop the watcher daemon").action(async () => {
|
|
133
|
+
await withConfig(pkgVersion, "watch:stop", {}, async ({ logger }) => watchStopCommand({ logger }));
|
|
134
|
+
});
|
|
135
|
+
watch.command("status").description("Report watcher daemon state").action(async () => {
|
|
136
|
+
await withConfig(pkgVersion, "watch:status", {}, async ({ logger }) => watchStatusCommand({ logger }));
|
|
137
|
+
});
|
|
138
|
+
|
|
123
139
|
program
|
|
124
140
|
.command("audit")
|
|
125
141
|
.description("Analyze codebase health (read-only)")
|
package/src/commands/rag.js
CHANGED
|
@@ -2,17 +2,12 @@
|
|
|
2
2
|
// kj rag index [--project <slug>] [--with-sources]
|
|
3
3
|
// kj rag query <text> [--scope plans|code|onboarding|all] [--top-k N] [--json]
|
|
4
4
|
// Closes the v2.22.0 RAG MVP end-to-end from the terminal.
|
|
5
|
-
import { openVecStore, countChunks } from "../rag/vec-store.js";
|
|
6
|
-
import {
|
|
5
|
+
import { openVecStore, countChunks, projectSlug } from "../rag/vec-store.js";
|
|
6
|
+
import { makeEmbedder } from "../rag/embedders/factory.js";
|
|
7
7
|
import { indexProject } from "../rag/indexer.js";
|
|
8
8
|
import { query } from "../rag/retriever.js";
|
|
9
9
|
import { getKarajanHome } from "../utils/paths.js";
|
|
10
10
|
|
|
11
|
-
function makeEmbedder(config) {
|
|
12
|
-
const cfg = config?.rag?.embedder || {};
|
|
13
|
-
return new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim: cfg.dim });
|
|
14
|
-
}
|
|
15
|
-
|
|
16
11
|
function openDb(config) {
|
|
17
12
|
return openVecStore({ dim: config?.rag?.embedder?.dim || 768 });
|
|
18
13
|
}
|
|
@@ -43,6 +38,12 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
|
|
|
43
38
|
try {
|
|
44
39
|
const topK = Math.max(1, Number(flags.topK) || 5);
|
|
45
40
|
const scope = flags.scope || "all";
|
|
41
|
+
// KJC-TSK-0438 — project isolation. Auto-detect slug from projectDir
|
|
42
|
+
// (basename normalised); `--project all` disables the filter, `--project
|
|
43
|
+
// <slug>` overrides. Pre-v2.27 chunks with NULL slug are only visible
|
|
44
|
+
// when no filter is in effect.
|
|
45
|
+
const detected = projectSlug(config?.projectDir || process.cwd());
|
|
46
|
+
const project = flags.project === "all" ? null : (flags.project || detected || null);
|
|
46
47
|
// KJC-BUG-0061 follow-up: align the CLI `--json` shape with the MCP
|
|
47
48
|
// handler. The MCP tool responds `{ hits: [], empty: true, topK, scope }`
|
|
48
49
|
// so agents (and the `/kj-rag-query` skill from Camino B) have a
|
|
@@ -54,7 +55,9 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
|
|
|
54
55
|
if (flags.json) process.stdout.write(`${JSON.stringify({ hits: [], empty: true, topK, scope })}\n`);
|
|
55
56
|
return [];
|
|
56
57
|
}
|
|
57
|
-
const
|
|
58
|
+
const mode = flags.mode || "hybrid";
|
|
59
|
+
const alpha = Math.max(0, Math.min(1, Number(flags.alpha) || 0.6));
|
|
60
|
+
const hits = await query(db, makeEmbedder(config), text, { topK, scope, project, mode, alpha });
|
|
58
61
|
if (flags.json) {
|
|
59
62
|
process.stdout.write(`${JSON.stringify(hits)}\n`);
|
|
60
63
|
} else {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// KJC-TSK-0441 — `kj watch [start|stop|status]`. Thin wrappers.
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { startWatcher, readPidFile, clearPidFile, isPidAlive, writePidFile } from "../rag/watcher.js";
|
|
6
|
+
|
|
7
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
|
|
9
|
+
export async function watchStartCommand({ config, logger, flags = {} }) {
|
|
10
|
+
const projectDir = config?.projectDir || process.cwd();
|
|
11
|
+
const existing = readPidFile();
|
|
12
|
+
if (existing && isPidAlive(existing)) { logger.info(`Watcher already running (pid ${existing}).`); return { pid: existing, started: false }; }
|
|
13
|
+
if (existing) clearPidFile();
|
|
14
|
+
if (flags.foreground) {
|
|
15
|
+
writePidFile();
|
|
16
|
+
const stop = startWatcher({ projectDir, config, logger, withSources: !!flags.withSources });
|
|
17
|
+
const shutdown = async () => { await stop(); clearPidFile(); process.exit(0); };
|
|
18
|
+
process.on("SIGINT", shutdown).on("SIGTERM", shutdown);
|
|
19
|
+
logger.info(`Watcher started in foreground (pid ${process.pid}). Ctrl+C to stop.`);
|
|
20
|
+
return { pid: process.pid, started: true };
|
|
21
|
+
}
|
|
22
|
+
const args = [join(HERE, "..", "cli.js"), "watch", "start", "--foreground", ...(flags.withSources ? ["--with-sources"] : [])];
|
|
23
|
+
const child = spawn(process.execPath, args, { detached: true, stdio: "ignore", env: { ...process.env, KJ_WATCHER_DAEMON: "1" } });
|
|
24
|
+
child.unref();
|
|
25
|
+
logger.info(`Watcher started in background (pid ${child.pid}).`);
|
|
26
|
+
return { pid: child.pid, started: true };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function watchStopCommand({ logger }) {
|
|
30
|
+
const pid = readPidFile();
|
|
31
|
+
if (!pid) { logger.info("No watcher running."); return { stopped: false }; }
|
|
32
|
+
if (!isPidAlive(pid)) { clearPidFile(); logger.info(`Stale PID file removed (process ${pid} was not alive).`); return { stopped: false, stale: true }; }
|
|
33
|
+
try { process.kill(pid, "SIGTERM"); } catch (err) { logger.warn(`Could not signal pid ${pid}: ${err.message}`); }
|
|
34
|
+
clearPidFile();
|
|
35
|
+
logger.info(`Watcher (pid ${pid}) stopped.`);
|
|
36
|
+
return { stopped: true, pid };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function watchStatusCommand({ logger }) {
|
|
40
|
+
const pid = readPidFile();
|
|
41
|
+
const alive = pid && isPidAlive(pid);
|
|
42
|
+
if (alive) logger.info(`Watcher running (pid ${pid}).`);
|
|
43
|
+
else if (pid) logger.warn(`Stale PID file (process ${pid} not alive). Run \`kj watch stop\` to clean up.`);
|
|
44
|
+
else logger.info("Watcher is NOT running.");
|
|
45
|
+
return { pid, alive };
|
|
46
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// from src/commands/* are forbidden by the layer-boundaries test
|
|
3
3
|
// (MCP and CLI are peer layers); we hit the pure rag/* modules instead.
|
|
4
4
|
import { countChunks, openVecStore } from "../../rag/vec-store.js";
|
|
5
|
-
import {
|
|
5
|
+
import { makeEmbedder } from "../../rag/embedders/factory.js";
|
|
6
6
|
import { indexProject } from "../../rag/indexer.js";
|
|
7
7
|
import { query } from "../../rag/retriever.js";
|
|
8
8
|
import { getKarajanHome } from "../../utils/paths.js";
|
|
@@ -12,11 +12,6 @@ const silentLogger = {
|
|
|
12
12
|
info: () => {}, warn: () => {}, error: () => {}, debug: () => {}, setContext: () => {},
|
|
13
13
|
};
|
|
14
14
|
|
|
15
|
-
function makeEmbedder(config) {
|
|
16
|
-
const cfg = config?.rag?.embedder || {};
|
|
17
|
-
return new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim: cfg.dim });
|
|
18
|
-
}
|
|
19
|
-
|
|
20
15
|
export async function handleRagQuery(args, server) {
|
|
21
16
|
const text = args?.text;
|
|
22
17
|
if (typeof text !== "string" || text.length === 0) {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// never sees an exception from this stage.
|
|
7
7
|
import { emitProgress, makeEvent } from "../../utils/events.js";
|
|
8
8
|
import { openVecStore, countChunks } from "../../rag/vec-store.js";
|
|
9
|
-
import {
|
|
9
|
+
import { makeEmbedder } from "../../rag/embedders/factory.js";
|
|
10
10
|
import { query } from "../../rag/retriever.js";
|
|
11
11
|
|
|
12
12
|
const DEFAULT_TOP_K = 5;
|
|
@@ -35,8 +35,7 @@ export async function runRagContextStage({ config, logger, emitter, eventBase, t
|
|
|
35
35
|
}
|
|
36
36
|
const topK = preload.topK || DEFAULT_TOP_K;
|
|
37
37
|
const scope = preload.scope || DEFAULT_SCOPE;
|
|
38
|
-
const
|
|
39
|
-
const embedder = new OllamaEmbedder({ url: cfg.url, model: cfg.model, dim });
|
|
38
|
+
const embedder = makeEmbedder(config);
|
|
40
39
|
const hits = await query(db, embedder, task, { topK, scope });
|
|
41
40
|
if (hits.length === 0) return { skipped: true, reason: "no-hits" };
|
|
42
41
|
const lines = hits.map((h) => {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// KJC-TSK-0444 — AST-aware source chunker. Replaces the regex-based
|
|
2
|
+
// chunkSource for JS/TS/JSX files. Each top-level declaration (function,
|
|
3
|
+
// class, var/const, export named/default) becomes one chunk whose `text`
|
|
4
|
+
// is the literal source slice [node.start, node.end]. Comments
|
|
5
|
+
// immediately preceding a declaration are folded in (JSDoc + leading
|
|
6
|
+
// line comments) so the chunk reads like a doc-aware snippet.
|
|
7
|
+
//
|
|
8
|
+
// Falls back to the regex chunker if @babel/parser cannot parse the
|
|
9
|
+
// file (errorRecovery: true catches most cases; a hard syntax error
|
|
10
|
+
// returns null so the caller can fall back).
|
|
11
|
+
import { parse } from "@babel/parser";
|
|
12
|
+
|
|
13
|
+
const PLUGINS = ["typescript", "jsx", "decorators-legacy", "classProperties", "topLevelAwait"];
|
|
14
|
+
|
|
15
|
+
export function chunkSourceAST(text, { path = "<src>", limit = 800, overlap = 100 } = {}) {
|
|
16
|
+
let ast;
|
|
17
|
+
try { ast = parse(text, { sourceType: "module", errorRecovery: true, plugins: PLUGINS }); }
|
|
18
|
+
catch { return null; }
|
|
19
|
+
const body = ast?.program?.body;
|
|
20
|
+
if (!Array.isArray(body) || body.length === 0) return null;
|
|
21
|
+
const out = [];
|
|
22
|
+
for (const node of body) {
|
|
23
|
+
const sym = extractSymbol(node);
|
|
24
|
+
const start = leadingCommentsStart(node);
|
|
25
|
+
const slice = text.slice(start, node.end ?? start);
|
|
26
|
+
if (!slice.trim()) continue;
|
|
27
|
+
if (slice.length <= limit) out.push({ text: slice, metadata: { source: path, kind: "code", symbol: sym } });
|
|
28
|
+
else for (const piece of windowText(slice, limit, overlap)) out.push({ text: piece, metadata: { source: path, kind: "code", symbol: sym } });
|
|
29
|
+
}
|
|
30
|
+
return out.length === 0 ? null : out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function extractSymbol(node) {
|
|
34
|
+
if (node.type === "ExportNamedDeclaration" && node.declaration) return extractSymbol(node.declaration);
|
|
35
|
+
if (node.type === "ExportDefaultDeclaration") return "default";
|
|
36
|
+
if (node.id?.name) return node.id.name;
|
|
37
|
+
if (node.declarations?.[0]?.id?.name) return node.declarations[0].id.name;
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function leadingCommentsStart(node) {
|
|
42
|
+
const lc = node.leadingComments;
|
|
43
|
+
if (Array.isArray(lc) && lc.length > 0) return lc[0].start ?? node.start ?? 0;
|
|
44
|
+
return node.start ?? 0;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function windowText(text, limit, overlap) {
|
|
48
|
+
if (text.length <= limit) return [text];
|
|
49
|
+
const step = Math.max(1, limit - overlap);
|
|
50
|
+
const out = [];
|
|
51
|
+
for (let i = 0; i < text.length; i += step) {
|
|
52
|
+
out.push(text.slice(i, i + limit));
|
|
53
|
+
if (i + limit >= text.length) break;
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
package/src/rag/chunker.js
CHANGED
|
@@ -91,13 +91,20 @@ export function chunkPlan(plan, { path = "<plan>", limit = DEFAULT_CHUNK_LIMIT }
|
|
|
91
91
|
|
|
92
92
|
// ---------- source chunker (export-symbol granularity) ------------
|
|
93
93
|
|
|
94
|
+
import { chunkSourceAST } from "./chunker-ast.js";
|
|
95
|
+
|
|
94
96
|
const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|let)\s+(\w+)/gm;
|
|
95
97
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
// KJC-TSK-0444 — try AST chunker first (top-level declarations as whole
|
|
99
|
+
// units, JSDoc folded in); fall back to the export-regex if @babel/parser
|
|
100
|
+
// can't parse the file (returns null).
|
|
101
|
+
export function chunkSource(text, opts = {}) {
|
|
102
|
+
const ast = chunkSourceAST(text, opts);
|
|
103
|
+
if (ast) return ast;
|
|
104
|
+
return chunkSourceRegex(text, opts);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function chunkSourceRegex(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
|
|
101
108
|
const marks = [];
|
|
102
109
|
EXPORT_RE.lastIndex = 0;
|
|
103
110
|
let m;
|
|
@@ -117,3 +124,5 @@ export function chunkSource(text, { path = "<src>", limit = DEFAULT_CHUNK_LIMIT,
|
|
|
117
124
|
}
|
|
118
125
|
return out;
|
|
119
126
|
}
|
|
127
|
+
|
|
128
|
+
export { chunkSourceRegex };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// KJC-TSK-0442 — shared POST helper for cloud embedders (OpenAI, Voyage).
|
|
2
|
+
// Same envelope: Bearer auth, JSON body shaped by caller, JSON response
|
|
3
|
+
// extracted by caller, dim validated against the adapter's expected size.
|
|
4
|
+
export async function cloudEmbed(adapter, text, ErrCls, { provider, body, extract }) {
|
|
5
|
+
if (typeof text !== "string" || text.length === 0) throw new ErrCls("embed: text must be a non-empty string");
|
|
6
|
+
const ctrl = new AbortController();
|
|
7
|
+
const timer = setTimeout(() => ctrl.abort(), adapter.timeoutMs);
|
|
8
|
+
let res;
|
|
9
|
+
try {
|
|
10
|
+
res = await adapter.fetch(adapter.url, {
|
|
11
|
+
method: "POST",
|
|
12
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${adapter.apiKey}` },
|
|
13
|
+
body: JSON.stringify(body),
|
|
14
|
+
signal: ctrl.signal,
|
|
15
|
+
});
|
|
16
|
+
} catch (err) { throw new ErrCls(`${provider} embed request failed: ${err.message}`, { cause: err }); }
|
|
17
|
+
finally { clearTimeout(timer); }
|
|
18
|
+
if (!res.ok) throw new ErrCls(`${provider} embed HTTP ${res.status}`, { status: res.status });
|
|
19
|
+
const json = await res.json();
|
|
20
|
+
const arr = extract(json);
|
|
21
|
+
if (!Array.isArray(arr) || arr.length === 0) throw new ErrCls(`${provider} response missing embedding`);
|
|
22
|
+
if (arr.length !== adapter.dim) throw new ErrCls(`${provider} dim mismatch: got ${arr.length}, expected ${adapter.dim} for model ${adapter.model}`);
|
|
23
|
+
return Float32Array.from(arr);
|
|
24
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// KJC-TSK-0442 — Embedder factory. Selects provider from
|
|
2
|
+
// config.rag.embedder.provider (ollama | openai | voyage; default ollama).
|
|
3
|
+
// Each provider has its own dim default; the factory wires it so the
|
|
4
|
+
// caller does not need to know.
|
|
5
|
+
import { OllamaEmbedder } from "../embedder.js";
|
|
6
|
+
import { OpenAIEmbedder } from "./openai.js";
|
|
7
|
+
import { VoyageEmbedder } from "./voyage.js";
|
|
8
|
+
|
|
9
|
+
const PROVIDERS = { ollama: { cls: OllamaEmbedder, dim: 768 }, openai: { cls: OpenAIEmbedder, dim: 1536 }, voyage: { cls: VoyageEmbedder, dim: 1024 } };
|
|
10
|
+
|
|
11
|
+
export function makeEmbedder(config = {}) {
|
|
12
|
+
const cfg = config?.rag?.embedder || {};
|
|
13
|
+
const provider = cfg.provider || "ollama";
|
|
14
|
+
const spec = PROVIDERS[provider];
|
|
15
|
+
if (!spec) throw new Error(`Unknown embedder provider: ${provider}. Supported: ${Object.keys(PROVIDERS).join(", ")}`);
|
|
16
|
+
const dim = cfg.dim || spec.dim;
|
|
17
|
+
return new spec.cls({ url: cfg.url, apiKey: cfg.api_key, model: cfg.model, dim, timeoutMs: cfg.timeout_ms });
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export { PROVIDERS };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// KJC-TSK-0442 — OpenAI text-embeddings adapter.
|
|
2
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
3
|
+
|
|
4
|
+
const DEFAULTS = { url: "https://api.openai.com/v1/embeddings", model: "text-embedding-3-small", dim: 1536, timeoutMs: 30000 };
|
|
5
|
+
|
|
6
|
+
export class OpenAIEmbedderError extends Error {
|
|
7
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "OpenAIEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export class OpenAIEmbedder {
|
|
11
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_OPENAI_KEY, model = process.env.KJ_OPENAI_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
12
|
+
// KJC architecture invariant: Karajan does not read provider API keys
|
|
13
|
+
// (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.) — those belong to the CLI
|
|
14
|
+
// agents Karajan spawns. RAG embedders are the one exception: they
|
|
15
|
+
// call the OpenAI endpoint directly, but use a Karajan-scoped env var
|
|
16
|
+
// (KJ_OPENAI_KEY) so the invariant stays clean.
|
|
17
|
+
if (!apiKey) throw new OpenAIEmbedderError("OpenAI embedder requires an api_key (config.rag.embedder.api_key or KJ_OPENAI_KEY env)");
|
|
18
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
19
|
+
}
|
|
20
|
+
async embed(text) { return cloudEmbed(this, text, OpenAIEmbedderError, { provider: "OpenAI", body: { model: this.model, input: text }, extract: (b) => b?.data?.[0]?.embedding }); }
|
|
21
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new OpenAIEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
22
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// KJC-TSK-0442 — Voyage AI embeddings adapter.
|
|
2
|
+
import { cloudEmbed } from "./_cloud-base.js";
|
|
3
|
+
|
|
4
|
+
const DEFAULTS = { url: "https://api.voyageai.com/v1/embeddings", model: "voyage-code-3", dim: 1024, timeoutMs: 30000 };
|
|
5
|
+
|
|
6
|
+
export class VoyageEmbedderError extends Error {
|
|
7
|
+
constructor(message, { cause, status } = {}) { super(message); this.name = "VoyageEmbedderError"; if (cause) this.cause = cause; if (status != null) this.status = status; }
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export class VoyageEmbedder {
|
|
11
|
+
constructor({ url = DEFAULTS.url, apiKey = process.env.KJ_VOYAGE_KEY, model = process.env.KJ_VOYAGE_EMBED_MODEL || DEFAULTS.model, dim = DEFAULTS.dim, timeoutMs = DEFAULTS.timeoutMs, fetchFn = globalThis.fetch } = {}) {
|
|
12
|
+
// Same invariant as OpenAIEmbedder: Karajan-scoped env var.
|
|
13
|
+
if (!apiKey) throw new VoyageEmbedderError("Voyage embedder requires an api_key (config.rag.embedder.api_key or KJ_VOYAGE_KEY env)");
|
|
14
|
+
Object.assign(this, { url, apiKey, model, dim, timeoutMs, fetch: fetchFn });
|
|
15
|
+
}
|
|
16
|
+
async embed(text) { return cloudEmbed(this, text, VoyageEmbedderError, { provider: "Voyage", body: { model: this.model, input: [text], input_type: "document" }, extract: (b) => b?.data?.[0]?.embedding }); }
|
|
17
|
+
async embedBatch(texts) { if (!Array.isArray(texts)) throw new VoyageEmbedderError("embedBatch: texts must be an array"); const out = []; for (const t of texts) out.push(await this.embed(t)); return out; }
|
|
18
|
+
}
|
package/src/rag/indexer.js
CHANGED
|
@@ -41,7 +41,7 @@ function chunksFor(path, kind) {
|
|
|
41
41
|
* fresh batch is written, so a second call against the same file
|
|
42
42
|
* leaves the store with exactly the latest chunks (no duplicates).
|
|
43
43
|
*/
|
|
44
|
-
export async function indexFile(path, { db, embedder, logger = console } = {}) {
|
|
44
|
+
export async function indexFile(path, { db, embedder, logger = console, project = null } = {}) {
|
|
45
45
|
if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
|
|
46
46
|
const kind = detectKind(path);
|
|
47
47
|
if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
|
|
@@ -53,7 +53,7 @@ export async function indexFile(path, { db, embedder, logger = console } = {}) {
|
|
|
53
53
|
for (const ch of chunks) {
|
|
54
54
|
try {
|
|
55
55
|
const embedding = await embedder.embed(ch.text);
|
|
56
|
-
insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding });
|
|
56
|
+
insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project });
|
|
57
57
|
indexed += 1;
|
|
58
58
|
} catch (err) {
|
|
59
59
|
failed += 1;
|
|
@@ -95,20 +95,25 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
|
|
|
95
95
|
if (existsSync(planRoot)) {
|
|
96
96
|
const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
|
|
97
97
|
for (const p of plans) {
|
|
98
|
-
const r = await indexFile(p, { db, embedder, logger });
|
|
98
|
+
const r = await indexFile(p, { db, embedder, logger, project: slug });
|
|
99
99
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
100
100
|
}
|
|
101
101
|
}
|
|
102
102
|
const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
|
|
103
103
|
if (existsSync(onboarding)) {
|
|
104
|
-
const r = await indexFile(onboarding, { db, embedder, logger });
|
|
104
|
+
const r = await indexFile(onboarding, { db, embedder, logger, project: slug });
|
|
105
105
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
106
106
|
}
|
|
107
107
|
if (withSources) {
|
|
108
|
-
|
|
108
|
+
// KJC-BUG-0062: `.kj/worktrees/HU-*/` contains git worktrees Karajan spawns
|
|
109
|
+
// per HU during `kj run`; they hold complete copies of `src/` that
|
|
110
|
+
// duplicate every chunk and contaminate retrieval scores. `_diet` is the
|
|
111
|
+
// tests/_diet/ sandbox used by the test-diet audit harness — never user
|
|
112
|
+
// code. Skip both.
|
|
113
|
+
const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
|
|
109
114
|
const sources = await listFiles(projectDir, (p) => /\.(js|mjs|cjs|ts|tsx|jsx)$/.test(p) && !p.split("/").some((seg) => SKIP.has(seg)));
|
|
110
115
|
for (const s of sources) {
|
|
111
|
-
const r = await indexFile(s, { db, embedder, logger });
|
|
116
|
+
const r = await indexFile(s, { db, embedder, logger, project: slug });
|
|
112
117
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
113
118
|
}
|
|
114
119
|
}
|
package/src/rag/retriever.js
CHANGED
|
@@ -1,47 +1,61 @@
|
|
|
1
|
-
// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
|
|
2
|
-
|
|
3
|
-
// The CLI (Step 6) and the future MCP tool are the only consumers.
|
|
1
|
+
// KJC-PCS-0049 Step 5 — Retriever for the RAG pipeline.
|
|
2
|
+
import { searchSimilar, searchBM25 } from "./vec-store.js";
|
|
4
3
|
|
|
5
|
-
import { searchSimilar } from "./vec-store.js";
|
|
6
|
-
|
|
7
|
-
// Kind boosts for rerank: a plan chunk is more "intentional" than a
|
|
8
|
-
// raw code chunk, an onboarding brief sits between them. Tunable via
|
|
9
|
-
// the third argument; the defaults are conservative — they break ties
|
|
10
|
-
// between equidistant chunks but won't reorder by big distance gaps.
|
|
11
4
|
const DEFAULT_KIND_BOOST = { plan: 0.05, onboarding: 0.03, code: 0 };
|
|
12
5
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
*
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
6
|
+
// KJC-TSK-0440 — asymmetric source/test boost.
|
|
7
|
+
const SOURCE_BOOST_NON_TEST = 0.05;
|
|
8
|
+
const TEST_TERMS_RE = /\b(test|tests|spec|specs|expect|describe|it\(|jest|vitest|mocha)\b/i;
|
|
9
|
+
const TEST_PATH_RE = /[\\/](tests?|specs?|__tests__)[\\/]|\.test\.[jt]sx?$|\.spec\.[jt]sx?$/i;
|
|
10
|
+
|
|
11
|
+
function shouldBoostSources(queryText) { return !TEST_TERMS_RE.test(queryText); }
|
|
12
|
+
function isTestPath(source) { return TEST_PATH_RE.test(source || ""); }
|
|
13
|
+
|
|
14
|
+
// KJC-TSK-0443 — fuse semantic + keyword hits into a unified candidate set.
|
|
15
|
+
// For 'semantic'/'keyword' modes we surface that side. For 'hybrid' (default)
|
|
16
|
+
// we min-max normalise both scores to [0,1] (lower=better) and linear-combine
|
|
17
|
+
// via alpha * semantic + (1-alpha) * keyword. Result written back to
|
|
18
|
+
// `distance` so the kind+source boost pipeline keeps working unchanged.
|
|
19
|
+
function fuseHits(semantic, keyword, alpha, mode) {
|
|
20
|
+
if (mode === "semantic") return semantic;
|
|
21
|
+
if (mode === "keyword") return keyword.map((h) => ({ ...h, distance: h.bm25 }));
|
|
22
|
+
const byId = new Map();
|
|
23
|
+
for (const h of semantic) byId.set(h.id, { ...h, _sem: h.distance });
|
|
24
|
+
for (const h of keyword) {
|
|
25
|
+
const prev = byId.get(h.id);
|
|
26
|
+
if (prev) prev._kw = h.bm25;
|
|
27
|
+
else byId.set(h.id, { ...h, _kw: h.bm25, distance: h.bm25 });
|
|
28
|
+
}
|
|
29
|
+
const list = [...byId.values()];
|
|
30
|
+
const norm = (vals) => {
|
|
31
|
+
const xs = vals.filter((v) => Number.isFinite(v));
|
|
32
|
+
if (xs.length === 0) return () => 0.5;
|
|
33
|
+
const min = Math.min(...xs); const max = Math.max(...xs); const span = max - min || 1;
|
|
34
|
+
return (v) => Number.isFinite(v) ? (v - min) / span : 1;
|
|
35
|
+
};
|
|
36
|
+
const nSem = norm(list.map((h) => h._sem));
|
|
37
|
+
const nKw = norm(list.map((h) => h._kw));
|
|
38
|
+
for (const h of list) h.distance = alpha * nSem(h._sem) + (1 - alpha) * nKw(h._kw);
|
|
39
|
+
return list;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function query(db, embedder, text, { topK = 5, scope = "all", kindBoost = DEFAULT_KIND_BOOST, project = null, mode = "hybrid", alpha = 0.6 } = {}) {
|
|
29
43
|
if (!text || typeof text !== "string") throw new Error("query: text must be a non-empty string");
|
|
30
|
-
const queryEmbedding = await embedder.embed(text);
|
|
31
|
-
// Over-fetch (topK * 2) so the rerank step has room to reshuffle
|
|
32
|
-
// without dropping a relevant chunk past the cut. Clamped at 50 to
|
|
33
|
-
// avoid loading the entire store on tiny topK calls.
|
|
34
44
|
const fetchK = Math.min(50, topK * 2);
|
|
35
45
|
const scopeKind = scope === "plans" ? "plan" : scope === "code" ? "code" : scope === "onboarding" ? "onboarding" : null;
|
|
36
|
-
const
|
|
46
|
+
const wantSemantic = mode !== "keyword";
|
|
47
|
+
const wantKeyword = mode !== "semantic";
|
|
48
|
+
const semanticHits = wantSemantic ? searchSimilar(db, await embedder.embed(text), fetchK, { kind: scopeKind, project }) : [];
|
|
49
|
+
const keywordHits = wantKeyword ? searchBM25(db, text, fetchK, { kind: scopeKind, project }) : [];
|
|
50
|
+
const raw = fuseHits(semanticHits, keywordHits, alpha, mode);
|
|
37
51
|
if (raw.length === 0) return [];
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
score: r.distance - (kindBoost[r.kind] || 0),
|
|
45
|
-
})).sort((a, b) => a.score - b.score).slice(0, topK);
|
|
52
|
+
const boostSources = shouldBoostSources(text);
|
|
53
|
+
const reranked = raw.map((r) => {
|
|
54
|
+
const kindB = kindBoost[r.kind] || 0;
|
|
55
|
+
const sourceB = (boostSources && r.kind === "code" && !isTestPath(r.source)) ? SOURCE_BOOST_NON_TEST : 0;
|
|
56
|
+
return { ...r, score: r.distance - kindB - sourceB };
|
|
57
|
+
}).sort((a, b) => a.score - b.score).slice(0, topK);
|
|
46
58
|
return reranked;
|
|
47
59
|
}
|
|
60
|
+
|
|
61
|
+
export { shouldBoostSources, isTestPath, fuseHits };
|
package/src/rag/vec-store.js
CHANGED
|
@@ -45,7 +45,38 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
|
|
|
45
45
|
CREATE INDEX IF NOT EXISTS chunks_by_source ON chunks(source);
|
|
46
46
|
CREATE INDEX IF NOT EXISTS chunks_by_kind ON chunks(kind);
|
|
47
47
|
`);
|
|
48
|
+
// KJC-TSK-0438 — Project isolation. project_slug = basename of projectDir
|
|
49
|
+
// normalized; chunks indexed before the migration carry NULL and remain
|
|
50
|
+
// queryable when `--project all` (or no slug detected at query time).
|
|
51
|
+
const cols = db.prepare("PRAGMA table_info(chunks)").all().map((c) => c.name);
|
|
52
|
+
if (!cols.includes("project_slug")) {
|
|
53
|
+
db.exec("ALTER TABLE chunks ADD COLUMN project_slug TEXT;");
|
|
54
|
+
db.exec("CREATE INDEX IF NOT EXISTS chunks_by_project ON chunks(project_slug);");
|
|
55
|
+
}
|
|
48
56
|
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[${dim}]);`);
|
|
57
|
+
// KJC-TSK-0443 — FTS5 keyword index for BM25 hybrid scoring. content='chunks'
|
|
58
|
+
// makes it a contentless mirror linked by id (no double storage); triggers
|
|
59
|
+
// sync the FTS5 rows on insert/update/delete. Rebuilds idempotently when
|
|
60
|
+
// the FTS5 table exists but is empty (e.g. created post-migration on a
|
|
61
|
+
// DB with pre-v2.28 chunks).
|
|
62
|
+
db.exec(`
|
|
63
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, content='chunks', content_rowid='id');
|
|
64
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
|
|
65
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
|
|
66
|
+
END;
|
|
67
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
68
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
69
|
+
END;
|
|
70
|
+
CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
|
|
71
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
72
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
|
|
73
|
+
END;
|
|
74
|
+
`);
|
|
75
|
+
const ftsCount = db.prepare("SELECT COUNT(*) AS n FROM chunks_fts").get().n;
|
|
76
|
+
const chunksCount = db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
77
|
+
if (ftsCount === 0 && chunksCount > 0) {
|
|
78
|
+
db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');");
|
|
79
|
+
}
|
|
49
80
|
return db;
|
|
50
81
|
}
|
|
51
82
|
|
|
@@ -54,13 +85,13 @@ export function openVecStore({ dim = DEFAULT_DIM, path = dbPath() } = {}) {
|
|
|
54
85
|
* because the virtual table rowid is set explicitly to the freshly
|
|
55
86
|
* minted chunks.id. Returns that id.
|
|
56
87
|
*/
|
|
57
|
-
export function insertChunk(db, { source, kind, text, metadata = null, embedding }) {
|
|
88
|
+
export function insertChunk(db, { source, kind, text, metadata = null, embedding, project = null }) {
|
|
58
89
|
if (!Array.isArray(embedding) && !ArrayBuffer.isView(embedding)) {
|
|
59
90
|
throw new Error("insertChunk: embedding must be an array or typed array of floats");
|
|
60
91
|
}
|
|
61
92
|
const meta = metadata == null ? null : (typeof metadata === "string" ? metadata : JSON.stringify(metadata));
|
|
62
|
-
const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata) VALUES (?, ?, ?, ?)");
|
|
63
|
-
const info = ins.run(source, kind, text, meta);
|
|
93
|
+
const ins = db.prepare("INSERT INTO chunks (source, kind, text, metadata, project_slug) VALUES (?, ?, ?, ?, ?)");
|
|
94
|
+
const info = ins.run(source, kind, text, meta, project);
|
|
64
95
|
// sqlite-vec's vec0 virtual table requires the rowid to arrive as a BigInt
|
|
65
96
|
// even for values that comfortably fit in a Number. Better-sqlite3 returns
|
|
66
97
|
// lastInsertRowid as BigInt only when safeIntegers mode is on; force it.
|
|
@@ -75,22 +106,36 @@ export function insertChunk(db, { source, kind, text, metadata = null, embedding
|
|
|
75
106
|
* text, metadata, distance }]` for the topK best matches; lower
|
|
76
107
|
* distance = closer. Returns `[]` when the store is empty.
|
|
77
108
|
*/
|
|
78
|
-
export function searchSimilar(db, embedding, topK = 5, { kind = null } = {}) {
|
|
109
|
+
export function searchSimilar(db, embedding, topK = 5, { kind = null, project = null } = {}) {
|
|
79
110
|
const buf = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
|
|
80
111
|
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
112
|
+
const projectClause = project ? "AND c.project_slug = ?" : "";
|
|
81
113
|
const sql = `
|
|
82
|
-
SELECT c.id, c.source, c.kind, c.text, c.metadata, v.distance
|
|
114
|
+
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, v.distance
|
|
83
115
|
FROM vec_chunks v
|
|
84
116
|
JOIN chunks c ON c.id = v.rowid
|
|
85
|
-
WHERE v.embedding MATCH ? AND k = ? ${kindClause}
|
|
117
|
+
WHERE v.embedding MATCH ? AND k = ? ${kindClause} ${projectClause}
|
|
86
118
|
ORDER BY v.distance
|
|
87
119
|
`;
|
|
88
120
|
const params = [Buffer.from(buf.buffer), topK];
|
|
89
121
|
if (kind) params.push(kind);
|
|
122
|
+
if (project) params.push(project);
|
|
90
123
|
const rows = db.prepare(sql).all(...params);
|
|
91
124
|
return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
|
|
92
125
|
}
|
|
93
126
|
|
|
127
|
+
/**
|
|
128
|
+
* Normalize a projectDir into a stable slug used for project isolation.
|
|
129
|
+
* Basename → lowercased → non-alphanumeric stripped to dashes.
|
|
130
|
+
* KJC-TSK-0438.
|
|
131
|
+
*/
|
|
132
|
+
export function projectSlug(projectDir) {
|
|
133
|
+
if (!projectDir) return null;
|
|
134
|
+
const base = String(projectDir).replace(/\/+$/, "").split("/").pop();
|
|
135
|
+
if (!base) return null;
|
|
136
|
+
return base.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
137
|
+
}
|
|
138
|
+
|
|
94
139
|
function safeParse(s) { try { return JSON.parse(s); } catch { return s; } }
|
|
95
140
|
|
|
96
141
|
/**
|
|
@@ -111,3 +156,30 @@ export function countChunks(db, { kind = null } = {}) {
|
|
|
111
156
|
if (kind) return db.prepare("SELECT COUNT(*) AS n FROM chunks WHERE kind = ?").get(kind).n;
|
|
112
157
|
return db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n;
|
|
113
158
|
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* KJC-TSK-0443 — BM25 keyword search over chunks_fts. Returns
|
|
162
|
+
* [{ id, source, kind, text, metadata, project_slug, bm25 }] ordered by
|
|
163
|
+
* relevance (lower bm25 = better in SQLite FTS5). `query` is sanitized for
|
|
164
|
+
* the FTS5 grammar: special chars are stripped, words split on whitespace.
|
|
165
|
+
*/
|
|
166
|
+
export function searchBM25(db, queryText, topK = 10, { kind = null, project = null } = {}) {
|
|
167
|
+
const cleaned = String(queryText || "").replace(/["'()]/g, " ").split(/\s+/).filter(Boolean).join(" OR ");
|
|
168
|
+
if (!cleaned) return [];
|
|
169
|
+
const kindClause = kind ? "AND c.kind = ?" : "";
|
|
170
|
+
const projectClause = project ? "AND c.project_slug = ?" : "";
|
|
171
|
+
const sql = `
|
|
172
|
+
SELECT c.id, c.source, c.kind, c.text, c.metadata, c.project_slug, bm25(chunks_fts) AS bm25
|
|
173
|
+
FROM chunks_fts JOIN chunks c ON c.id = chunks_fts.rowid
|
|
174
|
+
WHERE chunks_fts MATCH ? ${kindClause} ${projectClause}
|
|
175
|
+
ORDER BY bm25 LIMIT ?
|
|
176
|
+
`;
|
|
177
|
+
const params = [cleaned];
|
|
178
|
+
if (kind) params.push(kind);
|
|
179
|
+
if (project) params.push(project);
|
|
180
|
+
params.push(topK);
|
|
181
|
+
try {
|
|
182
|
+
const rows = db.prepare(sql).all(...params);
|
|
183
|
+
return rows.map((r) => ({ ...r, metadata: r.metadata ? safeParse(r.metadata) : null }));
|
|
184
|
+
} catch { return []; }
|
|
185
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// KJC-TSK-0441 (RAG v2.28.0 PR1) — chokidar watcher. Live re-index of plans
|
|
2
|
+
// + onboarding + (opt) sources, debounced. PID file arbitrates a single daemon.
|
|
3
|
+
import { writeFileSync, readFileSync, existsSync, unlinkSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import chokidar from "chokidar";
|
|
6
|
+
import { openVecStore, deleteChunksBySource, projectSlug } from "./vec-store.js";
|
|
7
|
+
import { makeEmbedder } from "./embedders/factory.js";
|
|
8
|
+
import { indexFile } from "./indexer.js";
|
|
9
|
+
import { getKarajanHome } from "../utils/paths.js";
|
|
10
|
+
|
|
11
|
+
const PIDFILE = () => join(getKarajanHome(), "watcher.pid");
|
|
12
|
+
const DEFAULT_DEBOUNCE_MS = 1000;
|
|
13
|
+
const SKIP_SEGMENTS = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".next", ".kj", "_diet"]);
|
|
14
|
+
const SOURCE_RE = /\.(js|mjs|cjs|ts|tsx|jsx)$/;
|
|
15
|
+
|
|
16
|
+
function isWatchable(path) {
|
|
17
|
+
if (path.includes("/.karajan/onboarding/")) return /\.md$/.test(path);
|
|
18
|
+
if (path.includes("/.karajan/plans/")) return /\.json$/.test(path);
|
|
19
|
+
if (path.split("/").some((seg) => SKIP_SEGMENTS.has(seg))) return false;
|
|
20
|
+
return SOURCE_RE.test(path);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function startWatcher({ projectDir, config, logger = console, debounceMs = DEFAULT_DEBOUNCE_MS, withSources = false } = {}) {
|
|
24
|
+
if (!projectDir) throw new Error("startWatcher: projectDir is required");
|
|
25
|
+
const slug = projectSlug(projectDir);
|
|
26
|
+
const dim = config?.rag?.embedder?.dim || 768;
|
|
27
|
+
const db = openVecStore({ dim });
|
|
28
|
+
const embedder = makeEmbedder(config);
|
|
29
|
+
const paths = [join(getKarajanHome(), "onboarding"), join(getKarajanHome(), "plans")];
|
|
30
|
+
if (withSources) paths.push(projectDir);
|
|
31
|
+
const watcher = chokidar.watch(paths, { ignoreInitial: true, persistent: true });
|
|
32
|
+
const pending = new Map();
|
|
33
|
+
const flush = async (p) => {
|
|
34
|
+
pending.delete(p);
|
|
35
|
+
if (!existsSync(p)) { deleteChunksBySource(db, p); logger.info?.(`[rag-watcher] removed → ${p}`); return; }
|
|
36
|
+
if (!isWatchable(p)) return;
|
|
37
|
+
try { const r = await indexFile(p, { db, embedder, logger, project: slug }); logger.info?.(`[rag-watcher] reindexed ${p} → ${r.indexed} chunk(s)`); }
|
|
38
|
+
catch (err) { logger.warn?.(`[rag-watcher] reindex failed for ${p}: ${err.message}`); }
|
|
39
|
+
};
|
|
40
|
+
const schedule = (p) => { if (pending.has(p)) clearTimeout(pending.get(p)); pending.set(p, setTimeout(() => flush(p), debounceMs)); };
|
|
41
|
+
watcher.on("add", schedule).on("change", schedule).on("unlink", schedule).on("error", (err) => logger.warn?.(`[rag-watcher] ${err.message}`));
|
|
42
|
+
return async function stop() {
|
|
43
|
+
for (const t of pending.values()) clearTimeout(t);
|
|
44
|
+
pending.clear();
|
|
45
|
+
await watcher.close();
|
|
46
|
+
try { db.close(); } catch { /* already closed */ }
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function writePidFile(pid = process.pid) { writeFileSync(PIDFILE(), String(pid), "utf8"); }
|
|
51
|
+
export function clearPidFile() { if (existsSync(PIDFILE())) unlinkSync(PIDFILE()); }
|
|
52
|
+
export function readPidFile() {
|
|
53
|
+
if (!existsSync(PIDFILE())) return null;
|
|
54
|
+
const pid = Number(readFileSync(PIDFILE(), "utf8").trim());
|
|
55
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
56
|
+
}
|
|
57
|
+
export function isPidAlive(pid) {
|
|
58
|
+
if (!pid) return false;
|
|
59
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
60
|
+
}
|
|
@@ -39,23 +39,21 @@ export function parseCooldown(message) {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
// 5. Claude Code session / weekly limit: "resets 10:10pm" / "resets 4am"
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
const ampmMatch = /reset(?:s|ting)?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*([ap]m)\b/i.exec(
|
|
48
|
-
message
|
|
49
|
-
);
|
|
42
|
+
// 5. Claude Code session / weekly limit: "resets 10:10pm (Europe/Madrid)" / "resets 4am"
|
|
43
|
+
// KJC-BUG-0064: prefer the parenthesised IANA TZ when present so the
|
|
44
|
+
// same stderr resolves to the same instant regardless of where kj runs
|
|
45
|
+
// (CI in TZ=UTC was misclassifying Madrid 10:10pm as already past).
|
|
46
|
+
// No TZ literal → fall back to the host's local TZ.
|
|
47
|
+
const ampmMatch = /reset(?:s|ting)?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*([ap]m)\b/i.exec(message);
|
|
50
48
|
if (ampmMatch) {
|
|
51
49
|
let hour = Number.parseInt(ampmMatch[1], 10) % 12;
|
|
52
50
|
if (/pm/i.test(ampmMatch[3])) hour += 12;
|
|
53
51
|
const minute = ampmMatch[2] ? Number.parseInt(ampmMatch[2], 10) : 0;
|
|
54
52
|
if (hour < 24 && minute < 60) {
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
return { cooldownUntil: target.toISOString(), cooldownMs: target.getTime() - Date.now() };
|
|
53
|
+
const tzMatch = /\(([A-Za-z]+\/[A-Za-z_/+\-0-9]+)\)/.exec(message);
|
|
54
|
+
const tz = tzMatch ? tzMatch[1] : null;
|
|
55
|
+
const target = tz ? nextOccurrenceInTZ(hour, minute, tz) : nextOccurrenceLocal(hour, minute);
|
|
56
|
+
if (target) return { cooldownUntil: target.toISOString(), cooldownMs: Math.max(0, target.getTime() - Date.now()) };
|
|
59
57
|
}
|
|
60
58
|
}
|
|
61
59
|
|
|
@@ -84,6 +82,50 @@ export function parseCooldown(message) {
|
|
|
84
82
|
return { cooldownUntil: null, cooldownMs: null };
|
|
85
83
|
}
|
|
86
84
|
|
|
85
|
+
// KJC-BUG-0064 — TZ-aware "next occurrence of hour:minute" resolver.
|
|
86
|
+
// Walks today + tomorrow in the given IANA TZ; returns the first instant
|
|
87
|
+
// > now. Uses Intl.DateTimeFormat as the source of truth so it works on
|
|
88
|
+
// any host TZ (CI runs UTC, dev runs Europe/Madrid, etc.).
|
|
89
|
+
function nextOccurrenceInTZ(hour, minute, tz) {
|
|
90
|
+
try {
|
|
91
|
+
const now = new Date();
|
|
92
|
+
for (let dayOffset = 0; dayOffset <= 1; dayOffset++) {
|
|
93
|
+
const probe = new Date(now.getTime() + dayOffset * 86400000);
|
|
94
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
95
|
+
timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit",
|
|
96
|
+
}).formatToParts(probe).reduce((acc, p) => (p.type !== "literal" ? { ...acc, [p.type]: p.value } : acc), {});
|
|
97
|
+
if (!parts.year) return null;
|
|
98
|
+
const candidate = zonedTimeToUTC(Number(parts.year), Number(parts.month), Number(parts.day), hour, minute, tz);
|
|
99
|
+
if (candidate && candidate.getTime() > now.getTime()) return candidate;
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
} catch { return null; }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Converts a wall-clock date + time in `tz` to a UTC Date.
|
|
106
|
+
// Strategy: interpret year/month/day/hour/minute as naive UTC, see what
|
|
107
|
+
// wall-clock that maps to in `tz`, derive the offset (tz - UTC at that
|
|
108
|
+
// moment) and subtract it from the naive value to land on the true UTC.
|
|
109
|
+
// One step suffices except at DST transitions (within ±1h, acceptable
|
|
110
|
+
// for cooldown timing).
|
|
111
|
+
function zonedTimeToUTC(year, month, day, hour, minute, tz) {
|
|
112
|
+
const naiveUTC = Date.UTC(year, month - 1, day, hour, minute);
|
|
113
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
114
|
+
timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit",
|
|
115
|
+
hour: "2-digit", minute: "2-digit", hour12: false,
|
|
116
|
+
}).formatToParts(new Date(naiveUTC)).reduce((acc, p) => (p.type !== "literal" ? { ...acc, [p.type]: p.value } : acc), {});
|
|
117
|
+
const wallInTZ = Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day), Number(parts.hour) % 24, Number(parts.minute));
|
|
118
|
+
const tzOffset = wallInTZ - naiveUTC; // positive when tz is ahead of UTC
|
|
119
|
+
return new Date(naiveUTC - tzOffset);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function nextOccurrenceLocal(hour, minute) {
|
|
123
|
+
const target = new Date();
|
|
124
|
+
target.setHours(hour, minute, 0, 0);
|
|
125
|
+
if (target.getTime() <= Date.now()) target.setDate(target.getDate() + 1);
|
|
126
|
+
return target;
|
|
127
|
+
}
|
|
128
|
+
|
|
87
129
|
const RATE_LIMIT_PATTERNS = [
|
|
88
130
|
// Claude CLI
|
|
89
131
|
{ pattern: /usage limit/i, agent: "claude" },
|