karajan-code 4.62.1 → 4.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/packages/hu-board/src/plan-mutations.js +21 -0
- package/src/cli/register-meta.js +15 -2
- package/src/commands/hu.js +110 -13
- package/src/commands/rag.js +34 -8
- package/src/hu/github-board.js +20 -3
- package/src/hu/team-board-cache.js +84 -0
- package/src/hu/team-board-mirror.js +26 -0
- package/src/lang/chunk-go.js +2 -27
- package/src/lang/chunk-java.js +7 -32
- package/src/lang/chunk-python.js +2 -27
- package/src/lang/chunk-rust.js +2 -27
- package/src/lang/chunk-span.js +49 -0
- package/src/plan/plan-store.js +4 -1
- package/src/rag/auto-update.js +55 -7
- package/src/rag/bootstrap-index.js +86 -0
- package/src/rag/chunker-ast.js +5 -12
- package/src/rag/chunker.js +2 -14
- package/src/rag/embed-header.js +35 -0
- package/src/rag/embed-schema.js +28 -0
- package/src/rag/eval.js +39 -3
- package/src/rag/indexer.js +36 -14
- package/src/rag/worktree-files.js +26 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "karajan-code",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.63.0",
|
|
4
4
|
"description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "AGPL-3.0",
|
|
@@ -115,7 +115,7 @@
|
|
|
115
115
|
"express-rate-limit": "^8.5.0",
|
|
116
116
|
"helmet": "^8.1.0",
|
|
117
117
|
"js-yaml": "^4.2.0",
|
|
118
|
-
"karajan-core": "^1.
|
|
118
|
+
"karajan-core": "^1.6.0",
|
|
119
119
|
"karajan-rag": "^1.2.0",
|
|
120
120
|
"knip": "^6.15.0",
|
|
121
121
|
"madge": "^8.0.0",
|
|
@@ -86,6 +86,13 @@ function readPlan(filePath) {
|
|
|
86
86
|
return JSON.parse(raw);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
+
// KJC-TSK-1166 (ADR 0025): a plan that mirrors the shared board is a view. The truth is
|
|
90
|
+
// in the issues; the board shows it and refuses to edit it, so the two never diverge.
|
|
91
|
+
function mirrorGuard(plan) {
|
|
92
|
+
if (!plan?.readonly) return null;
|
|
93
|
+
return { ok: false, error: `this plan mirrors the shared board (${plan.readonly}) and is read-only here: change it with kj hu move` };
|
|
94
|
+
}
|
|
95
|
+
|
|
89
96
|
function writePlan(filePath, plan) {
|
|
90
97
|
writeJsonAtomicSync(filePath, plan);
|
|
91
98
|
}
|
|
@@ -110,6 +117,8 @@ export function setHuStatus({ planId, huId, status, projectId }) {
|
|
|
110
117
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
111
118
|
|
|
112
119
|
const plan = readPlan(filePath);
|
|
120
|
+
const mirrored = mirrorGuard(plan);
|
|
121
|
+
if (mirrored) return mirrored;
|
|
113
122
|
if (!Array.isArray(plan.hus)) return { ok: false, error: 'plan has no hus[]' };
|
|
114
123
|
|
|
115
124
|
const existed = updateHuStatus(plan, huId, status);
|
|
@@ -153,6 +162,8 @@ export function setHuFields({ planId, huId, patch, projectId }) {
|
|
|
153
162
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
154
163
|
|
|
155
164
|
const plan = readPlan(filePath);
|
|
165
|
+
const mirrored = mirrorGuard(plan);
|
|
166
|
+
if (mirrored) return mirrored;
|
|
156
167
|
if (!Array.isArray(plan.hus)) return { ok: false, error: 'plan has no hus[]' };
|
|
157
168
|
|
|
158
169
|
// KJC-TSK-0401: si el patch toca blocked_by, validar antes de
|
|
@@ -188,6 +199,8 @@ export function setHuFailResult({ planId, huId, blocker, projectId }) {
|
|
|
188
199
|
const filePath = findPlanFilePath(planId, projectId);
|
|
189
200
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
190
201
|
const plan = readPlan(filePath);
|
|
202
|
+
const mirrored = mirrorGuard(plan);
|
|
203
|
+
if (mirrored) return mirrored;
|
|
191
204
|
if (!Array.isArray(plan.hus)) return { ok: false, error: 'plan has no hus[]' };
|
|
192
205
|
const hu = plan.hus.find(h => h.id === huId);
|
|
193
206
|
if (!hu) return { ok: false, error: `hu not found: ${huId}` };
|
|
@@ -220,6 +233,8 @@ export async function revertHuFromSnapshot({ planId, huId, projectId }) {
|
|
|
220
233
|
const filePath = findPlanFilePath(planId, projectId);
|
|
221
234
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
222
235
|
const plan = readPlan(filePath);
|
|
236
|
+
const mirrored = mirrorGuard(plan);
|
|
237
|
+
if (mirrored) return mirrored;
|
|
223
238
|
const hu = plan.hus?.find(h => h.id === huId);
|
|
224
239
|
if (!hu) return { ok: false, error: `hu not found: ${huId}` };
|
|
225
240
|
const snapshotSha = hu.outcome?.snapshot_sha;
|
|
@@ -287,6 +302,8 @@ export function renameProject({ projectId, name }) {
|
|
|
287
302
|
const filePath = join(projDir, f);
|
|
288
303
|
try {
|
|
289
304
|
const plan = readPlan(filePath);
|
|
305
|
+
// A mirror of the shared board keeps its own name; the rename goes on with the rest.
|
|
306
|
+
if (plan.readonly) continue;
|
|
290
307
|
plan.name = trimmed;
|
|
291
308
|
plan.updatedAt = new Date().toISOString();
|
|
292
309
|
writePlan(filePath, plan);
|
|
@@ -316,6 +333,8 @@ export function markPlanReady({ planId, projectId }) {
|
|
|
316
333
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
317
334
|
|
|
318
335
|
const plan = readPlan(filePath);
|
|
336
|
+
const mirrored = mirrorGuard(plan);
|
|
337
|
+
if (mirrored) return mirrored;
|
|
319
338
|
if (!Array.isArray(plan.hus) || plan.hus.length === 0) {
|
|
320
339
|
return { ok: false, error: 'plan has no hus[]' };
|
|
321
340
|
}
|
|
@@ -354,6 +373,8 @@ export function runPlan({ planId, projectId, taskOverride, huIds = null } = {})
|
|
|
354
373
|
if (!filePath) return { ok: false, error: `plan not found: ${planId}` };
|
|
355
374
|
|
|
356
375
|
const plan = readPlan(filePath);
|
|
376
|
+
const mirrored = mirrorGuard(plan);
|
|
377
|
+
if (mirrored) return mirrored;
|
|
357
378
|
if (!Array.isArray(plan.hus) || plan.hus.length === 0) {
|
|
358
379
|
return { ok: false, error: 'plan has no hus[]' };
|
|
359
380
|
}
|
package/src/cli/register-meta.js
CHANGED
|
@@ -353,6 +353,17 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
353
353
|
await huCommand({ config, action: "list", flags });
|
|
354
354
|
});
|
|
355
355
|
});
|
|
356
|
+
// KJC-TSK-1166 (ADR 0025): the HUs created before the shared board, moved to it once.
|
|
357
|
+
hu.command("migrate")
|
|
358
|
+
.description("Move every live local HU (pending or running) to the shared board, once: an issue each, status and old id kept")
|
|
359
|
+
.requiredOption("--to <backend>", "github")
|
|
360
|
+
.option("--dry-run", "Say what would move, touching nothing")
|
|
361
|
+
.option("--json", "Machine-readable output")
|
|
362
|
+
.action(async (flags) => {
|
|
363
|
+
await withConfig(pkgVersion, "hu", flags, async ({ config }) => {
|
|
364
|
+
await huCommand({ config, action: "migrate", flags });
|
|
365
|
+
});
|
|
366
|
+
});
|
|
356
367
|
// KJC-TSK-1163 (ADR 0025): the team board, turned on once per repo.
|
|
357
368
|
hu.command("backend <backend>")
|
|
358
369
|
.description("Turn the repo's HU Board on: github (shared, in GitHub Issues) or local; writes .karajan/board.yml for a PR")
|
|
@@ -784,8 +795,8 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
784
795
|
// first call). Internally we rebind to `flags.mode` so the handler
|
|
785
796
|
// in src/commands/rag.js stays untouched.
|
|
786
797
|
.option("--rag-mode <mode>", "hybrid (default) | semantic (cosine only) | keyword (BM25 only)", "hybrid")
|
|
787
|
-
.option("--alpha <n>", "
|
|
788
|
-
.option("--where <clause>", "Metadata filter
|
|
798
|
+
.option("--alpha <n>", "Deprecated: hybrid fusion is reciprocal rank since 4.63; tune rag.search.rrf_k (default 60) in kj.config.yml", "0.6")
|
|
799
|
+
.option("--where <clause>", "Metadata filter: 'symbol=loadConfig', 'hu_id=HU-003 AND kind=plan', or a prefix with ~= such as 'source~=src/billing' (relative to the project)")
|
|
789
800
|
.option("--library", "Query the distilled engineering canon (pattern cards shipped with kj + ~/.karajan/library, in ~/.karajan/library.db) instead of the project corpus; a project's own .karajan/library cards live in its index and answer normal queries. Cards carry problem signatures, when-NOT-to-apply and canonical citations (KJC-TSK-0697).")
|
|
790
801
|
.option("--rerank", "Apply cross-encoder rerank to top-K (needs @huggingface/transformers, slower but more precise)")
|
|
791
802
|
.option("--rerank-model <name>", "Cross-encoder model id. Default: Xenova/ms-marco-MiniLM-L-6-v2")
|
|
@@ -807,6 +818,8 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
807
818
|
.option("--golden <path>", "Path to golden queries JSON (default: tests/rag/golden-queries.json)")
|
|
808
819
|
.option("--top-k <n>", "Number of hits to retrieve per query (default: 10)", "10")
|
|
809
820
|
.option("--min-recall <n>", "Fail (exit 1) if aggregated recall@5 falls below this threshold")
|
|
821
|
+
.option("--save-baseline <path>", "Write recall@k, MRR and latency to this file as the frozen baseline")
|
|
822
|
+
.option("--baseline <path>", "Fail (exit 1) if recall@k, MRR or p95 latency got worse than that baseline")
|
|
810
823
|
.option("--project <slug>", "Filter by project slug. Pass 'all' to query across every indexed project.")
|
|
811
824
|
.option("--json", "Emit the full report as JSON")
|
|
812
825
|
.action(async (flags) => {
|
package/src/commands/hu.js
CHANGED
|
@@ -13,7 +13,9 @@ import { readIdentity } from "../identity/store.js";
|
|
|
13
13
|
import { effectiveGitEmail } from "../identity/detect.js";
|
|
14
14
|
import { roleSkills } from "../skills/role-skills.js";
|
|
15
15
|
import { readBoardConfig, setBoardBackend } from "../hu/board-config.js";
|
|
16
|
-
import { createHu, listHus, claim, setStatus, assertNotHeld } from "../hu/github-board.js";
|
|
16
|
+
import { createHu, listHus, claim, setStatus, assertNotHeld, isOffline } from "../hu/github-board.js";
|
|
17
|
+
import { listWithCache, queueMove, flushQueue } from "../hu/team-board-cache.js";
|
|
18
|
+
import { writeTeamMirror } from "../hu/team-board-mirror.js";
|
|
17
19
|
|
|
18
20
|
// KJC-TSK-0661 (Jorge's friction): every HU records WHO created it — the
|
|
19
21
|
// board must never show work nobody can trace. Host agent env → that agent;
|
|
@@ -57,20 +59,91 @@ function linesOf(v) {
|
|
|
57
59
|
/** KJC-TSK-1163: kj hu add on the shared board; the same no-duplicates rule, checked against the issues. */
|
|
58
60
|
async function addToTeamBoard({ repo, title, flags, deps, emit }) {
|
|
59
61
|
if (flags.id) throw new Error(`--id is not used with the shared board: GitHub gives each HU its id (the issue number)`);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
62
|
+
let hu;
|
|
63
|
+
try {
|
|
64
|
+
const { identical, similar } = findTitleMatches(title, await (deps.listTeamHus ?? listHus)({ repo }));
|
|
65
|
+
if (identical.length > 0) {
|
|
66
|
+
throw new Error(`An HU with this title already exists: ${identical[0].short_id} (status: ${identical[0].status}). Work that card or move it with: kj hu move ${identical[0].short_id} <status>`);
|
|
67
|
+
}
|
|
68
|
+
if (similar.length > 0) {
|
|
69
|
+
const candidates = similar.map((h) => ` · ${h.short_id} (${h.status}) "${h.title}"`).join("\n");
|
|
70
|
+
console.warn("⚠ possible duplicate — check before working it:\n" + candidates);
|
|
71
|
+
}
|
|
72
|
+
hu = await (deps.createTeamHu ?? createHu)({
|
|
73
|
+
repo, title, acceptanceCriteria: [...linesOf(flags.ac), ...linesOf(flags.criteria)], tests: linesOf(flags.tests), ref: flags.ref ? String(flags.ref).trim() : null,
|
|
74
|
+
});
|
|
75
|
+
} catch (err) {
|
|
76
|
+
// KJC-TSK-1165: an id is only ever minted by GitHub; nothing is created offline.
|
|
77
|
+
if (isOffline(err)) throw new Error(`creating a HU needs the network: GitHub gives it its id and no local one is made up (${err.message})`, { cause: err });
|
|
78
|
+
throw err;
|
|
67
79
|
}
|
|
68
|
-
const hu = await (deps.createTeamHu ?? createHu)({
|
|
69
|
-
repo, title, acceptanceCriteria: [...linesOf(flags.ac), ...linesOf(flags.criteria)], tests: linesOf(flags.tests), ref: flags.ref ? String(flags.ref).trim() : null,
|
|
70
|
-
});
|
|
71
80
|
return emit({ ...hu, board: "github", repo }, `✓ HU created: ${hu.short_id} (pending) in the shared board, issue #${hu.number} of ${repo}`);
|
|
72
81
|
}
|
|
73
82
|
|
|
83
|
+
/**
|
|
84
|
+
* KJC-TSK-1166: every live local HU (pending or running) becomes an issue with its criteria,
|
|
85
|
+
* tests and old id; a running one is claimed and moved by whoever migrates. The local card is
|
|
86
|
+
* marked skipped and points at its issue, so a second run has nothing left to move.
|
|
87
|
+
*/
|
|
88
|
+
async function migrateToTeamBoard({ projectDir, repo, who, dryRun, deps, emit }) {
|
|
89
|
+
const migrated = [];
|
|
90
|
+
const skipped = [];
|
|
91
|
+
for (const meta of await listPlans(projectDir)) {
|
|
92
|
+
const plan = await loadPlan(projectDir, meta.planId);
|
|
93
|
+
for (const h of plan.hus || []) {
|
|
94
|
+
const label = h.short_id || h.id;
|
|
95
|
+
const unfinished = h.migrated_to && h.migration_pending; // the issue exists, its running state does not yet
|
|
96
|
+
if (!unfinished && (h.migrated_to || !["pending", "running"].includes(h.status))) { skipped.push(label); continue; }
|
|
97
|
+
if (dryRun) { migrated.push({ from: label, to: h.migrated_to ?? null, status: h.migration_pending ?? h.status }); continue; }
|
|
98
|
+
const status = h.migration_pending ?? h.status;
|
|
99
|
+
if (!h.migrated_to) {
|
|
100
|
+
const was = `was ${label}`;
|
|
101
|
+
const issue = await (deps.createTeamHu ?? createHu)({ repo, title: h.title, acceptanceCriteria: h.acceptance_criteria ?? [], tests: h.acceptance_tests ?? [], ref: h.external_ref ? `${h.external_ref} · ${was}` : was });
|
|
102
|
+
// Persisted the moment the issue exists: a retry after a failure further on never
|
|
103
|
+
// creates it twice; what is still owed to it (its running state) is remembered.
|
|
104
|
+
h.migrated_to = issue.short_id;
|
|
105
|
+
h.external_ref = [h.external_ref, issue.short_id].filter(Boolean).join(" · ");
|
|
106
|
+
if (status === "running") h.migration_pending = "running";
|
|
107
|
+
await savePlan(projectDir, plan);
|
|
108
|
+
}
|
|
109
|
+
if (h.migration_pending === "running") {
|
|
110
|
+
const number = Number(h.migrated_to.slice(3));
|
|
111
|
+
await (deps.claimTeamHu ?? claim)({ repo, number, who, release: false });
|
|
112
|
+
await (deps.setTeamStatus ?? setStatus)({ repo, number, status: "running" });
|
|
113
|
+
delete h.migration_pending;
|
|
114
|
+
}
|
|
115
|
+
updateHuStatus(plan, h.id, "skipped");
|
|
116
|
+
await savePlan(projectDir, plan);
|
|
117
|
+
migrated.push({ from: label, to: h.migrated_to, status });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const verb = dryRun ? "would move" : "moved";
|
|
121
|
+
const lines = migrated.map((m) => ` ${m.from} → ${m.to ?? "(new issue)"} [${m.status}]`).join("\n");
|
|
122
|
+
const detail = lines ? "\n" + lines : "";
|
|
123
|
+
return emit({ migrated, skipped, dryRun }, `✓ ${verb} ${migrated.length} HU(s) to the shared board of ${repo}${detail}\n skipped ${skipped.length} (finished, discarded or already moved)`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** KJC-TSK-1165: apply the moves queued offline; conflicts are said, never forced; still offline is silent. */
|
|
127
|
+
async function applyQueuedMoves({ projectDir, repo, deps }) {
|
|
128
|
+
let now = null; // status of every HU as GitHub has it, read once, kept up to date as moves apply
|
|
129
|
+
const apply = async (op) => {
|
|
130
|
+
now ??= new Map((await (deps.listTeamHus ?? listHus)({ repo })).map((h) => [h.number, h.status]));
|
|
131
|
+
// The HU moved while this clone was offline: the newer state wins, the queued move is a conflict.
|
|
132
|
+
if (op.expected && now.get(op.number) !== op.expected) {
|
|
133
|
+
const err = new Error(`HU-${op.number} is ${now.get(op.number) ?? "gone"} now and was ${op.expected} when the move was queued`);
|
|
134
|
+
err.name = "BoardRejectedError";
|
|
135
|
+
throw err;
|
|
136
|
+
}
|
|
137
|
+
if (op.status === "running" || op.release) await (deps.claimTeamHu ?? claim)({ repo, number: op.number, who: op.who, release: op.release });
|
|
138
|
+
else await (deps.assertNotHeld ?? assertNotHeld)({ repo, number: op.number, who: op.who });
|
|
139
|
+
await (deps.setTeamStatus ?? setStatus)({ repo, number: op.number, status: op.status });
|
|
140
|
+
now.set(op.number, op.status);
|
|
141
|
+
};
|
|
142
|
+
const r = await flushQueue({ projectDir, apply });
|
|
143
|
+
if (r.applied) console.warn(`✓ ${r.applied} queued move(s) applied to the shared board`);
|
|
144
|
+
for (const c of r.conflicts) console.warn(`⚠ ${c}`);
|
|
145
|
+
}
|
|
146
|
+
|
|
74
147
|
/** KJC-TSK-1164: kj hu move on the shared board; starting it claims it (the assignee) first. */
|
|
75
148
|
async function moveOnTeamBoard({ repo, args, flags, deps, emit, who }) {
|
|
76
149
|
const [ref, status] = args;
|
|
@@ -123,8 +196,21 @@ export async function huCommand({ config = null, action, args = [], flags = {},
|
|
|
123
196
|
return emit(r, `✓ HU Board: ${where} — commit ${files} in a PR so the whole team inherits it`);
|
|
124
197
|
}
|
|
125
198
|
const board = await (deps.boardConfig ?? readBoardConfig)(projectDir);
|
|
199
|
+
const who = deps.who === undefined ? workerOf(projectDir) : deps.who;
|
|
200
|
+
// KJC-TSK-1166 (TEAM-GH-E): the HUs created before the shared board, moved to it once.
|
|
201
|
+
if (action === "migrate") {
|
|
202
|
+
if (flags.to !== "github") throw new Error("usage: kj hu migrate --to github [--dry-run]");
|
|
203
|
+
if (board.backend !== "github") throw new Error("the shared board is off in this repo: turn it on first with kj hu backend github");
|
|
204
|
+
return migrateToTeamBoard({ projectDir, repo: board.repo, who, dryRun: Boolean(flags.dryRun), deps, emit });
|
|
205
|
+
}
|
|
206
|
+
// KJC-TSK-1165 (TEAM-GH-D): moves made without network go first, once GitHub answers.
|
|
207
|
+
if (board.backend === "github") await applyQueuedMoves({ projectDir, repo: board.repo, deps });
|
|
126
208
|
if (board.backend === "github" && action === "list") {
|
|
127
|
-
const rows =
|
|
209
|
+
const { source, rows: hus, updatedAt } = await listWithCache({ projectDir, repo: board.repo, list: deps.listTeamHus ?? listHus });
|
|
210
|
+
if (source === "cache") console.warn(`⚠ GitHub did not answer: this is the local copy of the board from ${updatedAt}`);
|
|
211
|
+
// KJC-TSK-1166: what GitHub answered is what kj board shows, as a read-only plan.
|
|
212
|
+
else await writeTeamMirror(projectDir, hus, { repo: board.repo });
|
|
213
|
+
const rows = hus.map((h) => ({ ...h, id: h.short_id, plan: "github" }));
|
|
128
214
|
if (flags.json) { console.log(JSON.stringify(rows)); return rows; }
|
|
129
215
|
for (const r of rows) {
|
|
130
216
|
const held = r.claimed_by ? " · held by " + r.claimed_by : "";
|
|
@@ -135,7 +221,18 @@ export async function huCommand({ config = null, action, args = [], flags = {},
|
|
|
135
221
|
}
|
|
136
222
|
// KJC-BUG-0392: a zero-padded id (HU-0104) is a local HU from before the shared board: it stays local.
|
|
137
223
|
const legacyLocal = /^HU-0\d+$/i.test(String(args[0] ?? ""));
|
|
138
|
-
if (board.backend === "github" && action === "move" && !legacyLocal)
|
|
224
|
+
if (board.backend === "github" && action === "move" && !legacyLocal) {
|
|
225
|
+
try {
|
|
226
|
+
return await moveOnTeamBoard({ repo: board.repo, args, flags, deps, emit, who });
|
|
227
|
+
} catch (err) {
|
|
228
|
+
// Offline: the move waits in the queue; GitHub still decides when it is applied.
|
|
229
|
+
if (!isOffline(err)) throw err;
|
|
230
|
+
const number = Number(/^(?:HU-|#)(\d+)$/i.exec(args[0])[1]);
|
|
231
|
+
queueMove({ projectDir, number, status: args[1], who, release: Boolean(flags.release) });
|
|
232
|
+
console.warn(`⚠ GitHub did not answer: HU-${number} → ${args[1]} queued, applied when GitHub answers again`);
|
|
233
|
+
return emit({ id: `HU-${number}`, status: args[1], queued: true }, `⏳ HU-${number} → ${args[1]} (queued)`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
139
236
|
|
|
140
237
|
if (action === "list") {
|
|
141
238
|
const plans = await listPlans(projectDir);
|
package/src/commands/rag.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { isAbsolute, join, relative } from "node:path";
|
|
3
3
|
// KJC-PCS-0049 Step 6 — `kj rag` command group. Two subcommands:
|
|
4
4
|
// kj rag index [--project <slug>] [--with-sources]
|
|
@@ -13,7 +13,7 @@ import { query } from "../rag/retriever.js";
|
|
|
13
13
|
import { pathNamedHits, withPathNamedHits } from "../rag/path-hits.js";
|
|
14
14
|
import { installPostMergeHook, maybeAutoUpdate } from "../rag/auto-update.js";
|
|
15
15
|
import { indexLibrary, LIBRARY_PROJECT } from "../rag/library.js";
|
|
16
|
-
import { loadGoldenQueries, runEval } from "../rag/eval.js";
|
|
16
|
+
import { compareToBaseline, loadGoldenQueries, runEval } from "../rag/eval.js";
|
|
17
17
|
import { getKarajanHome } from "../utils/paths.js";
|
|
18
18
|
import { countProjectChunks, countProjectSources, emptyIndexRemedy, migrateProjectIndex } from "../rag/migrate.js";
|
|
19
19
|
import { openLibraryStore, openProjectStore, projectDbPath } from "../rag/project-store.js";
|
|
@@ -38,7 +38,7 @@ async function indexFiles({ files, projectDir, db, embedder, slug, logger, json,
|
|
|
38
38
|
logger.warn?.(`[rag] ${out}: not indexed`);
|
|
39
39
|
continue;
|
|
40
40
|
}
|
|
41
|
-
const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind() });
|
|
41
|
+
const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
|
|
42
42
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
43
43
|
}
|
|
44
44
|
const deleted = totals.deleted ? `, ${totals.deleted} chunks deleted` : "";
|
|
@@ -160,14 +160,22 @@ export async function ragQueryCommand({ text, config, logger, flags = {} }) {
|
|
|
160
160
|
logger.warn("[rag] this project's index holds none of its own files (only plans or briefs): run kj rag index --with-sources");
|
|
161
161
|
}
|
|
162
162
|
const mode = flags.mode || "hybrid";
|
|
163
|
-
|
|
163
|
+
// KJC-TSK-1180 (RAG-C, core 1.6.0): hybrid fusion is reciprocal rank; `alpha` weighs nothing
|
|
164
|
+
// any more, so a value that is still around is said once, never silently ignored.
|
|
165
|
+
if ((flags.alpha !== undefined && flags.alpha !== "0.6") || config?.rag?.search?.alpha !== undefined) {
|
|
166
|
+
logger.warn("[rag] alpha no longer applies: hybrid fusion is reciprocal rank (RRF); tune rag.search.rrf_k instead and drop alpha from the config");
|
|
167
|
+
}
|
|
168
|
+
const rrfK = config?.rag?.search?.rrf_k;
|
|
169
|
+
if (rrfK !== undefined && !(Number.isInteger(rrfK) && rrfK > 0)) throw new Error(`rag.search.rrf_k must be a positive integer, got ${JSON.stringify(rrfK)}`);
|
|
170
|
+
// KJC-TSK-1180 (RAG-B3): a relative `source~=` prefix is placed under this project's root.
|
|
171
|
+
const whereRoot = config?.projectDir || process.cwd();
|
|
164
172
|
// Safe by grammar: parseWhere (core where-parser.js) accepts ONLY
|
|
165
173
|
// AND-joined key=value pairs — an OR or parens never parses, so the
|
|
166
174
|
// composed clause cannot leak past the kind filter; it fails loudly.
|
|
167
175
|
let where = flags.where || null;
|
|
168
176
|
if (library) where = flags.where ? `kind=library AND ${flags.where}` : "kind=library";
|
|
169
177
|
const rerankOpts = flags.rerank ? { model: flags.rerankModel } : null;
|
|
170
|
-
const ranked = await query(db, makeGovernedEmbedder(config), text, { topK, scope, project, mode,
|
|
178
|
+
const ranked = await query(db, makeGovernedEmbedder(config), text, { topK, scope, project, mode, rrfK, where, whereRoot, rerankOpts });
|
|
171
179
|
const hits = library ? ranked : withPathNamedHits(pathNamedHits(db, text, { project, projectDir: config?.projectDir || process.cwd() }), ranked, topK);
|
|
172
180
|
if (flags.json) {
|
|
173
181
|
process.stdout.write(`${JSON.stringify(hits)}\n`);
|
|
@@ -201,20 +209,38 @@ export async function ragEvalCommand({ config, logger, flags = {} }) {
|
|
|
201
209
|
const embedder = makeGovernedEmbedder(config);
|
|
202
210
|
const runQuery = async (text, k) => query(db, embedder, text, { topK: k, scope: "all", project });
|
|
203
211
|
const report = await runEval(queries, runQuery, { topK, ks: [5, 10] });
|
|
212
|
+
const { aggregate: agg } = report;
|
|
204
213
|
if (flags.json) {
|
|
205
214
|
process.stdout.write(`${JSON.stringify(report)}\n`);
|
|
206
215
|
} else {
|
|
207
|
-
|
|
216
|
+
const lat = agg.latency ? ` · latency mean=${agg.latency.meanMs}ms p95=${agg.latency.p95Ms}ms` : "";
|
|
217
|
+
logger.info(`[rag-eval] ${agg.n} queries · recall@5=${agg.recall["@5"].toFixed(3)} · recall@10=${agg.recall["@10"].toFixed(3)} · MRR=${agg.mrr.toFixed(3)}${lat}`);
|
|
208
218
|
for (const r of report.results) {
|
|
209
219
|
const rank = r.firstRank == null ? "—" : `#${r.firstRank}`;
|
|
210
220
|
logger.info(` [${rank}] ${r.query}`);
|
|
211
221
|
}
|
|
212
222
|
}
|
|
213
223
|
const minRecall = Number(flags.minRecall);
|
|
214
|
-
if (Number.isFinite(minRecall) &&
|
|
215
|
-
logger.error?.(`[rag-eval] recall@5=${
|
|
224
|
+
if (Number.isFinite(minRecall) && agg.recall["@5"] < minRecall) {
|
|
225
|
+
logger.error?.(`[rag-eval] recall@5=${agg.recall["@5"].toFixed(3)} below --min-recall=${minRecall}`);
|
|
216
226
|
process.exitCode = 1;
|
|
217
227
|
}
|
|
228
|
+
// KJC-TSK-1170 (RAG-E): freeze the numbers, or fail when they got worse than the frozen ones.
|
|
229
|
+
if (flags.saveBaseline) {
|
|
230
|
+
const doc = { measuredAt: new Date().toISOString(), golden: goldenPath, topK, embedder: config?.rag?.embedder ?? null, search: config?.rag?.search ?? null, aggregate: agg };
|
|
231
|
+
writeFileSync(flags.saveBaseline, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
|
|
232
|
+
logger.info(`[rag-eval] baseline saved to ${flags.saveBaseline}`);
|
|
233
|
+
}
|
|
234
|
+
if (flags.baseline) {
|
|
235
|
+
const base = JSON.parse(readFileSync(flags.baseline, "utf8"));
|
|
236
|
+
const check = compareToBaseline(agg, base.aggregate ?? base);
|
|
237
|
+
if (!check.ok) {
|
|
238
|
+
logger.error?.(`[rag-eval] worse than the baseline ${flags.baseline}:\n - ${check.problems.join("\n - ")}`);
|
|
239
|
+
process.exitCode = 1;
|
|
240
|
+
} else {
|
|
241
|
+
logger.info(`[rag-eval] not worse than the baseline ${flags.baseline}`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
218
244
|
return report;
|
|
219
245
|
} finally {
|
|
220
246
|
db.close();
|
package/src/hu/github-board.js
CHANGED
|
@@ -12,11 +12,16 @@ export const STATUSES = ["pending", "running", "done", "failed", "skipped"];
|
|
|
12
12
|
const statusLabel = (s) => `kj:${s}`;
|
|
13
13
|
|
|
14
14
|
/** gh could not reach GitHub: the caller may queue the change, never invent data. */
|
|
15
|
-
export class BoardOfflineError extends Error {}
|
|
15
|
+
export class BoardOfflineError extends Error { name = "BoardOfflineError"; }
|
|
16
16
|
/** GitHub answered and said no (permissions, missing repo...). */
|
|
17
|
-
export class BoardRejectedError extends Error {}
|
|
17
|
+
export class BoardRejectedError extends Error { name = "BoardRejectedError"; }
|
|
18
18
|
/** The HU is running and someone else holds it: taking it over is explicit (release). */
|
|
19
|
-
export class HuClaimedError extends Error {}
|
|
19
|
+
export class HuClaimedError extends Error { name = "HuClaimedError"; }
|
|
20
|
+
|
|
21
|
+
// KJC-TSK-1165: by name, not only instanceof: a test runner may load this module twice.
|
|
22
|
+
export const isOffline = (err) => err instanceof BoardOfflineError || err?.name === "BoardOfflineError";
|
|
23
|
+
/** GitHub answered and refused (rejected, or the HU is held by someone else): a conflict, not a failure of kj. */
|
|
24
|
+
export const isRefusal = (err) => err instanceof BoardRejectedError || err instanceof HuClaimedError || ["BoardRejectedError", "HuClaimedError"].includes(err?.name);
|
|
20
25
|
|
|
21
26
|
// gh is not localized: these are the network failures it prints.
|
|
22
27
|
const OFFLINE = /dial tcp|no such host|could not resolve host|connection refused|i\/o timeout|network is unreachable|error connecting to/i;
|
|
@@ -48,6 +53,18 @@ export async function createHu({ repo, title, acceptanceCriteria = [], tests = [
|
|
|
48
53
|
const url = (await gh(run, ["issue", "create", "-R", repo, "--title", title, "--body", body, "--label", HU_LABEL, "--label", statusLabel("pending")])).trim();
|
|
49
54
|
const number = Number(/\/issues\/(\d+)$/.exec(url)?.[1]);
|
|
50
55
|
if (!Number.isInteger(number)) throw new BoardRejectedError(`gh issue create returned no issue URL: ${url}`);
|
|
56
|
+
// KJC-BUG-0395 (HU-2408): GitHub answered the create and left the issue without its labels, so
|
|
57
|
+
// the next move refused it as "not a kj HU". An HU the board does not recognise is not announced:
|
|
58
|
+
// the labels are checked, put back once, and a second absence is said loud.
|
|
59
|
+
const wanted = [HU_LABEL, statusLabel("pending")];
|
|
60
|
+
const labelsOf = async () => new Set(JSON.parse(await gh(run, ["issue", "view", String(number), "-R", repo, "--json", "labels"])).labels.map((l) => l.name));
|
|
61
|
+
let have = await labelsOf();
|
|
62
|
+
if (wanted.some((l) => !have.has(l))) {
|
|
63
|
+
await gh(run, ["issue", "edit", String(number), "-R", repo, ...wanted.filter((l) => !have.has(l)).flatMap((l) => ["--add-label", l])]);
|
|
64
|
+
have = await labelsOf();
|
|
65
|
+
const missing = wanted.filter((l) => !have.has(l));
|
|
66
|
+
if (missing.length) throw new BoardRejectedError(`issue #${number} was created without ${missing.join(", ")} and gh could not add them: the board does not recognise it, add the labels by hand`);
|
|
67
|
+
}
|
|
51
68
|
return { short_id: `HU-${number}`, number, status: "pending", title };
|
|
52
69
|
}
|
|
53
70
|
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KJC-TSK-1165 (TEAM-GH-D, ADR 0025): the local copy of the shared board, and the
|
|
3
|
+
* queue of moves made without network. The copy is a cache, never the truth: it
|
|
4
|
+
* is rewritten on every answer from GitHub and only read when GitHub does not
|
|
5
|
+
* answer. A queued move is applied when the network is back; one GitHub rejects
|
|
6
|
+
* (someone else holds the HU) is reported and dropped, never forced. Creating a
|
|
7
|
+
* HU offline is refused elsewhere: an id is only ever minted by GitHub.
|
|
8
|
+
*/
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { isOffline, isRefusal } from "./github-board.js";
|
|
12
|
+
|
|
13
|
+
const CACHE_FILE = ".karajan/board-cache.json";
|
|
14
|
+
// A fresh object every time: a shared default array would carry one read's queue into the next.
|
|
15
|
+
const empty = () => ({ hus: null, updatedAt: null, queue: [] });
|
|
16
|
+
|
|
17
|
+
export function readCache(projectDir) {
|
|
18
|
+
try {
|
|
19
|
+
return { ...empty(), ...JSON.parse(fs.readFileSync(path.join(projectDir, CACHE_FILE), "utf8")) };
|
|
20
|
+
} catch (err) {
|
|
21
|
+
if (err.code === "ENOENT") return empty();
|
|
22
|
+
throw err;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function writeCache(projectDir, cache) {
|
|
27
|
+
const file = path.join(projectDir, CACHE_FILE);
|
|
28
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
29
|
+
fs.writeFileSync(file, `${JSON.stringify(cache, null, 2)}\n`, "utf8");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The HUs from GitHub (and the copy refreshed), or the copy when GitHub cannot be reached. */
|
|
33
|
+
export async function listWithCache({ projectDir, repo, list }) {
|
|
34
|
+
try {
|
|
35
|
+
const rows = await list({ repo });
|
|
36
|
+
writeCache(projectDir, { ...readCache(projectDir), hus: rows, updatedAt: new Date().toISOString() });
|
|
37
|
+
return { source: "github", rows };
|
|
38
|
+
} catch (err) {
|
|
39
|
+
const cache = readCache(projectDir);
|
|
40
|
+
if (!isOffline(err) || !cache.hus) throw err;
|
|
41
|
+
return { source: "cache", rows: cache.hus, updatedAt: cache.updatedAt };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Remember a move to apply when the network is back. `expected` is the status the local
|
|
47
|
+
* copy shows now (null when the copy does not know the HU): when it is applied, a HU that
|
|
48
|
+
* moved meanwhile is a conflict, not something to overwrite. The copy takes the move so a
|
|
49
|
+
* second queued move on the same HU expects the first one's result.
|
|
50
|
+
*/
|
|
51
|
+
export function queueMove({ projectDir, number, status, who, release }) {
|
|
52
|
+
const cache = readCache(projectDir);
|
|
53
|
+
const row = cache.hus?.find((h) => h.number === number) ?? null;
|
|
54
|
+
cache.queue.push({ number, status, who, release: Boolean(release), expected: row?.status ?? null, queuedAt: new Date().toISOString() });
|
|
55
|
+
if (row) row.status = status;
|
|
56
|
+
writeCache(projectDir, cache);
|
|
57
|
+
return cache.queue.length;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Apply the queue in order. A move GitHub refuses is a conflict: reported, dropped, the
|
|
62
|
+
* HU stays as the other person left it. Still offline: stop, keep the rest for later.
|
|
63
|
+
* Any other error is kj's own: it surfaces, and the move stays queued.
|
|
64
|
+
* @returns {Promise<{applied: number, conflicts: string[], pending: number}>}
|
|
65
|
+
*/
|
|
66
|
+
export async function flushQueue({ projectDir, apply }) {
|
|
67
|
+
const cache = readCache(projectDir);
|
|
68
|
+
const conflicts = [];
|
|
69
|
+
let applied = 0;
|
|
70
|
+
while (cache.queue.length > 0) {
|
|
71
|
+
const op = cache.queue[0];
|
|
72
|
+
try {
|
|
73
|
+
await apply(op);
|
|
74
|
+
applied += 1;
|
|
75
|
+
} catch (err) {
|
|
76
|
+
if (isOffline(err)) break;
|
|
77
|
+
if (!isRefusal(err)) throw err;
|
|
78
|
+
conflicts.push(`HU-${op.number} → ${op.status} (queued ${op.queuedAt}) not applied: ${err.message}`);
|
|
79
|
+
}
|
|
80
|
+
cache.queue.shift();
|
|
81
|
+
writeCache(projectDir, cache);
|
|
82
|
+
}
|
|
83
|
+
return { applied, conflicts, pending: cache.queue.length };
|
|
84
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KJC-TSK-1166 (TEAM-GH-E, ADR 0025): what `kj board` shows of the shared board. The web
|
|
3
|
+
* board reads plans, so the shared board is mirrored into one read-only plan per project,
|
|
4
|
+
* rewritten from the issues every time kj reads them. It is a view: changing it is done
|
|
5
|
+
* with kj hu move, never from the web, and the board refuses to edit it.
|
|
6
|
+
*/
|
|
7
|
+
import { savePlan } from "../plan/plan-store.js";
|
|
8
|
+
|
|
9
|
+
export const MIRROR_PLAN_ID = "plan_github-board";
|
|
10
|
+
|
|
11
|
+
/** The shared board as a plan the web board can show, marked read-only. */
|
|
12
|
+
export async function writeTeamMirror(projectDir, rows, { repo }) {
|
|
13
|
+
const now = new Date().toISOString();
|
|
14
|
+
const plan = {
|
|
15
|
+
version: 2, planId: MIRROR_PLAN_ID, name: "github", alias: "github",
|
|
16
|
+
task: `Shared HU Board: GitHub Issues of ${repo}`,
|
|
17
|
+
status: "ready", readonly: "github", createdAt: now,
|
|
18
|
+
hus: rows.map((r) => ({
|
|
19
|
+
id: r.short_id, short_id: r.short_id, title: r.title, status: r.status,
|
|
20
|
+
claimed_by: r.claimed_by ?? null, created_by: "github", external_ref: `${repo}#${r.number}`,
|
|
21
|
+
acceptance_criteria: [], acceptance_tests: [], createdAt: now, updatedAt: now,
|
|
22
|
+
})),
|
|
23
|
+
};
|
|
24
|
+
await savePlan(projectDir, plan);
|
|
25
|
+
return plan;
|
|
26
|
+
}
|
package/src/lang/chunk-go.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// `type_declaration` contiene type_spec(s); usamos el primero como symbol.
|
|
9
9
|
|
|
10
10
|
import { getLoaded, loadLanguage } from "./tree-sitter-loader.js";
|
|
11
|
+
import { spanChunks } from "./chunk-span.js";
|
|
11
12
|
|
|
12
13
|
const DEFAULT_LIMIT = 800;
|
|
13
14
|
const DEFAULT_OVERLAP = 100;
|
|
@@ -21,16 +22,6 @@ export async function prepareGo() {
|
|
|
21
22
|
return loadLanguage("go");
|
|
22
23
|
}
|
|
23
24
|
|
|
24
|
-
function windowText(text, limit, overlap) {
|
|
25
|
-
if (text.length <= limit) return [text];
|
|
26
|
-
const step = Math.max(1, limit - overlap);
|
|
27
|
-
const out = [];
|
|
28
|
-
for (let i = 0; i < text.length; i += step) {
|
|
29
|
-
out.push(text.slice(i, i + limit));
|
|
30
|
-
if (i + limit >= text.length) break;
|
|
31
|
-
}
|
|
32
|
-
return out;
|
|
33
|
-
}
|
|
34
25
|
|
|
35
26
|
function symbolForType(node) {
|
|
36
27
|
for (const spec of node.namedChildren) {
|
|
@@ -74,21 +65,5 @@ function marksViaRegex(text) {
|
|
|
74
65
|
}
|
|
75
66
|
|
|
76
67
|
export function chunkGo(text, { path = "<src>", limit = DEFAULT_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
|
|
77
|
-
|
|
78
|
-
if (marks.length === 0) {
|
|
79
|
-
return windowText(text, limit, overlap).map((piece) => ({
|
|
80
|
-
text: piece,
|
|
81
|
-
metadata: { source: path, kind: "code", symbol: null, language: "go" },
|
|
82
|
-
}));
|
|
83
|
-
}
|
|
84
|
-
const out = [];
|
|
85
|
-
for (let i = 0; i < marks.length; i += 1) {
|
|
86
|
-
const start = marks[i].idx;
|
|
87
|
-
const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
|
|
88
|
-
const body = text.slice(start, end);
|
|
89
|
-
for (const piece of windowText(body, limit, overlap)) {
|
|
90
|
-
out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol, language: "go" } });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
return out;
|
|
68
|
+
return spanChunks(text, marksViaAST(text) ?? marksViaRegex(text), { path, language: "go", limit, overlap });
|
|
94
69
|
}
|
package/src/lang/chunk-java.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// sin prefijar (coherente con chunk-go), el RAG ya lleva contexto del fichero.
|
|
5
5
|
|
|
6
6
|
import { getLoaded, loadLanguage } from "./tree-sitter-loader.js";
|
|
7
|
+
import { spanChunks } from "./chunk-span.js";
|
|
7
8
|
|
|
8
9
|
const DEFAULT_LIMIT = 800;
|
|
9
10
|
const DEFAULT_OVERLAP = 100;
|
|
@@ -23,17 +24,6 @@ export async function prepareJava() {
|
|
|
23
24
|
return loadLanguage("java");
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
function windowText(text, limit, overlap) {
|
|
27
|
-
if (text.length <= limit) return [text];
|
|
28
|
-
const step = Math.max(1, limit - overlap);
|
|
29
|
-
const out = [];
|
|
30
|
-
for (let i = 0; i < text.length; i += step) {
|
|
31
|
-
out.push(text.slice(i, i + limit));
|
|
32
|
-
if (i + limit >= text.length) break;
|
|
33
|
-
}
|
|
34
|
-
return out;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
27
|
function nameOf(node) {
|
|
38
28
|
const n = node.childForFieldName("name");
|
|
39
29
|
return n ? n.text : null;
|
|
@@ -47,14 +37,15 @@ function marksViaAST(text) {
|
|
|
47
37
|
const marks = [];
|
|
48
38
|
for (const child of tree.rootNode.namedChildren) {
|
|
49
39
|
if (!CONTAINER_KINDS.has(child.type)) continue;
|
|
50
|
-
const
|
|
51
|
-
if (
|
|
40
|
+
const className = nameOf(child);
|
|
41
|
+
if (className) marks.push({ idx: child.startIndex, symbol: className });
|
|
52
42
|
const body = child.childForFieldName("body");
|
|
53
43
|
if (!body) continue;
|
|
54
44
|
for (const m of body.namedChildren) {
|
|
55
45
|
if (!MEMBER_KINDS.has(m.type)) continue;
|
|
56
|
-
const
|
|
57
|
-
|
|
46
|
+
const memberName = nameOf(m);
|
|
47
|
+
// KJC-TSK-1171: a member knows its class (by name); the chunk says both.
|
|
48
|
+
if (memberName) marks.push({ idx: m.startIndex, symbol: memberName, parent: className });
|
|
58
49
|
}
|
|
59
50
|
}
|
|
60
51
|
marks.sort((a, b) => a.idx - b.idx);
|
|
@@ -70,21 +61,5 @@ function marksViaRegex(text) {
|
|
|
70
61
|
}
|
|
71
62
|
|
|
72
63
|
export function chunkJava(text, { path = "<src>", limit = DEFAULT_LIMIT, overlap = DEFAULT_OVERLAP } = {}) {
|
|
73
|
-
|
|
74
|
-
if (marks.length === 0) {
|
|
75
|
-
return windowText(text, limit, overlap).map((piece) => ({
|
|
76
|
-
text: piece,
|
|
77
|
-
metadata: { source: path, kind: "code", symbol: null, language: "java" },
|
|
78
|
-
}));
|
|
79
|
-
}
|
|
80
|
-
const out = [];
|
|
81
|
-
for (let i = 0; i < marks.length; i += 1) {
|
|
82
|
-
const start = marks[i].idx;
|
|
83
|
-
const end = i + 1 < marks.length ? marks[i + 1].idx : text.length;
|
|
84
|
-
const body = text.slice(start, end);
|
|
85
|
-
for (const piece of windowText(body, limit, overlap)) {
|
|
86
|
-
out.push({ text: piece, metadata: { source: path, kind: "code", symbol: marks[i].symbol, language: "java" } });
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return out;
|
|
64
|
+
return spanChunks(text, marksViaAST(text) ?? marksViaRegex(text), { path, language: "java", limit, overlap });
|
|
90
65
|
}
|