karajan-code 5.0.0 → 5.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/package.json +2 -2
- package/scripts/postinstall.js +15 -1
- package/src/checks/config-files.js +23 -3
- package/src/checks/repo-state.js +26 -0
- package/src/cli/register-roles-skills.js +1 -1
- package/src/commands/init.js +11 -0
- package/src/commands/rag.js +46 -7
- package/src/commands/skills.js +4 -1
- package/src/commands/worktree.js +16 -4
- package/src/config/defaults.js +1 -0
- package/src/config/role-resolver.js +1 -0
- package/src/config/schema.js +1 -0
- package/src/harden/sentinel/sentinel-lib.mjs +14 -1
- package/src/hu/lane-prune.js +70 -8
- package/src/mcp/handlers/rag-handler.js +9 -1
- package/src/rag/auto-update.js +17 -3
- package/src/rag/bootstrap-index.js +22 -4
- package/src/rag/indexer.js +9 -4
- package/src/rag/pid-lease.js +115 -4
- package/src/review/publish-pr.js +4 -1
- package/src/review/solomon-arbitration.js +12 -6
- package/src/skills/atril-cli.js +19 -1
- package/src/skills/skills-search.js +72 -0
- package/src/skills/translate.js +62 -0
package/README.md
CHANGED
|
@@ -83,6 +83,10 @@ git config core.hooksPath .karajan/hooks
|
|
|
83
83
|
|
|
84
84
|
Requires git and at least one AI agent CLI — two enables cross-AI review; three enables arbitration. The npm route is `npm install -g @karajan-family/code` (published as `karajan-code` before joining the scope; the legacy name still installs the same versions). All install routes (npm, binaries, brew, Python wrapper) in the [install docs](https://karajancode.com/docs/v4/install/).
|
|
85
85
|
|
|
86
|
+
### Verify what you installed
|
|
87
|
+
|
|
88
|
+
Since 5.0.0 every release is published from GitHub Actions with signed provenance (SLSA), tied to the tagged commit that built it. Check the npm package with `npm audit signatures` in the folder where kj is installed (kj must be listed with verified attestations), and a standalone binary with `gh attestation verify kj-linux-x64 --repo manufosela/karajan-code` (GitHub CLI 2.49+). Details in the [install docs](https://karajancode.com/docs/v4/install/#verify-what-you-installed).
|
|
89
|
+
|
|
86
90
|
## The daily loop
|
|
87
91
|
|
|
88
92
|
1. You describe what you want. Your agent creates the card (`kj hu add`), queries the RAG, writes the failing test, then the code.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "karajan-code",
|
|
3
|
-
"version": "5.0.
|
|
3
|
+
"version": "5.0.2",
|
|
4
4
|
"description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "AGPL-3.0",
|
|
@@ -99,7 +99,7 @@
|
|
|
99
99
|
},
|
|
100
100
|
"dependencies": {
|
|
101
101
|
"@babel/parser": "^7.29.7",
|
|
102
|
-
"@karajan-family/atril": "^1.
|
|
102
|
+
"@karajan-family/atril": "^1.3.0",
|
|
103
103
|
"@karajan-family/governance": "^0.1.0",
|
|
104
104
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
105
105
|
"@xterm/xterm": "^6.0.0",
|
package/scripts/postinstall.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* - Idempotent: safe to run many times.
|
|
8
8
|
* - Never fails hard (exits 0 even on error) so `npm install` is not blocked.
|
|
9
9
|
*/
|
|
10
|
-
import { existsSync } from "node:fs";
|
|
10
|
+
import { existsSync, statSync } from "node:fs";
|
|
11
11
|
import fs from "node:fs/promises";
|
|
12
12
|
import os from "node:os";
|
|
13
13
|
import path from "node:path";
|
|
@@ -140,7 +140,21 @@ async function setupCodexMcp(kjHome) {
|
|
|
140
140
|
await fs.writeFile(configPath, updated, "utf8");
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
// KJC-BUG-0474 (#3128): a git worktree (a kj lane) has a `.git` FILE. `npm ci` there runs this
|
|
144
|
+
// script, and registering would point every session of the machine at the lane's server.
|
|
145
|
+
function insideLinkedWorktree(dir = ROOT_DIR) {
|
|
146
|
+
try {
|
|
147
|
+
return statSync(path.join(dir, ".git")).isFile();
|
|
148
|
+
} catch {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
143
153
|
async function main() {
|
|
154
|
+
if (insideLinkedWorktree()) {
|
|
155
|
+
process.stderr.write("karajan-mcp: not registered from a git worktree (a lane); the global MCP keeps pointing at the installed kj\n");
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
144
158
|
const claudeConfig = await readClaudeConfig();
|
|
145
159
|
const kjHome = await resolveKjHome(claudeConfig.mcpServers?.["karajan-mcp"]);
|
|
146
160
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* ~/.karajan/kj.config.yml (strategy: manual — structural fixes needed)
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { existsSync } from "node:fs";
|
|
11
|
+
import { existsSync, statSync } from "node:fs";
|
|
12
12
|
import fs from "node:fs/promises";
|
|
13
13
|
import os from "node:os";
|
|
14
14
|
import path from "node:path";
|
|
@@ -146,6 +146,19 @@ export function karajanMcpServerPath(config) {
|
|
|
146
146
|
return args.find((a) => typeof a === "string" && /[\\/]mcp[\\/]server\.js$/.test(a)) ?? null;
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
+
/**
|
|
150
|
+
* KJC-BUG-0474 (#3128): whether a file lives in a linked git worktree (a kj lane). The
|
|
151
|
+
* nearest `.git` up the tree decides: a worktree's is a file, a normal checkout's a folder.
|
|
152
|
+
*/
|
|
153
|
+
export function inLinkedWorktree(file) {
|
|
154
|
+
for (let dir = path.dirname(path.resolve(file)); ; dir = path.dirname(dir)) {
|
|
155
|
+
try {
|
|
156
|
+
return statSync(path.join(dir, ".git")).isFile();
|
|
157
|
+
} catch { /* no .git here: keep climbing */ }
|
|
158
|
+
if (path.dirname(dir) === dir) return false;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
149
162
|
/**
|
|
150
163
|
* Repair a stale karajan-mcp registration in place: re-point it to this
|
|
151
164
|
* install's server when that server exists, or drop the dead entry when this
|
|
@@ -191,12 +204,19 @@ function createStaleMcpPathCheck() {
|
|
|
191
204
|
}
|
|
192
205
|
const registered = karajanMcpServerPath(config);
|
|
193
206
|
if (!registered) return { ok: true, severity: "info", detail: "karajan-mcp not registered (skipped)" };
|
|
194
|
-
|
|
207
|
+
// KJC-BUG-0474 (#3128): a path inside a lane works until the lane goes, and meanwhile runs its code everywhere.
|
|
208
|
+
const inLane = inLinkedWorktree(registered);
|
|
209
|
+
if (!inLane && (await exists(registered))) return { ok: true, severity: "info", detail: "karajan-mcp path valid" };
|
|
195
210
|
const correct = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "mcp", "server.js");
|
|
211
|
+
if (inLane && inLinkedWorktree(correct)) {
|
|
212
|
+
return { ok: false, severity: "warn", detail: `karajan-mcp runs a lane's server: ${registered}`, fix: "Run kj doctor from the installed kj (not from a lane) to re-point it" };
|
|
213
|
+
}
|
|
196
214
|
return {
|
|
197
215
|
ok: false,
|
|
198
216
|
severity: "warn",
|
|
199
|
-
detail:
|
|
217
|
+
detail: inLane
|
|
218
|
+
? `karajan-mcp runs a lane's server (${registered}): every session runs that lane's code, and removing the lane breaks it`
|
|
219
|
+
: `karajan-mcp points at a path that no longer exists: ${registered} — this is the CONNECTION_CLOSED cause`,
|
|
200
220
|
fix: `Re-point karajan-mcp to ${correct} (or remove the entry) in ~/.claude.json`,
|
|
201
221
|
extra: { claudeJsonPath, correct },
|
|
202
222
|
};
|
package/src/checks/repo-state.js
CHANGED
|
@@ -29,6 +29,29 @@ const shellQuote = (value) => `'${String(value).replaceAll("'", "'\\''")}'`;
|
|
|
29
29
|
const git = (projectDir, args) => execFileSync("git", ["-C", projectDir, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
30
30
|
const gitOk = (projectDir, args) => { try { git(projectDir, args); return true; } catch { return false; } };
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* KJC-BUG-0473 (#3080): the main branch is main, always. With master as the default branch,
|
|
34
|
+
* every gate kj writes for [main] never runs, and nothing says so. Read from origin/HEAD when
|
|
35
|
+
* the clone knows it; with no origin, from a local master that has no main beside it.
|
|
36
|
+
* @returns {{branch: string, detail: string, fix: string} | null}
|
|
37
|
+
*/
|
|
38
|
+
export function defaultBranchProblem(projectDir) {
|
|
39
|
+
let branch = null;
|
|
40
|
+
try {
|
|
41
|
+
branch = git(projectDir, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).trim().replace(/^origin\//, "");
|
|
42
|
+
} catch {
|
|
43
|
+
if (gitOk(projectDir, ["rev-parse", "--verify", "-q", "refs/heads/master"])
|
|
44
|
+
&& !gitOk(projectDir, ["rev-parse", "--verify", "-q", "refs/heads/main"])) branch = "master";
|
|
45
|
+
}
|
|
46
|
+
if (!branch || branch === "main") return null;
|
|
47
|
+
return {
|
|
48
|
+
branch,
|
|
49
|
+
detail: `la rama por defecto es ${branch}, no main: los gates que kj escribe para main no se lanzan nunca`,
|
|
50
|
+
fix: `renombra ${branch} a main (en GitHub: Settings > Branches > Rename; si no existe, crea main) y actualiza el clon: `
|
|
51
|
+
+ `git branch -m ${branch} main && git fetch origin && git branch -u origin/main main && git remote set-head origin -a`,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
32
55
|
/** @internal Exported for dynamic import from tests. */
|
|
33
56
|
export function createRepoStateCheck({ projectDir }) {
|
|
34
57
|
return {
|
|
@@ -100,6 +123,9 @@ export function createRepoStateCheck({ projectDir }) {
|
|
|
100
123
|
});
|
|
101
124
|
}
|
|
102
125
|
|
|
126
|
+
const notMain = defaultBranchProblem(dir);
|
|
127
|
+
if (notMain) symptoms.push({ detail: notMain.detail, fix: notMain.fix });
|
|
128
|
+
|
|
103
129
|
if (symptoms.length === 0) return { ok: true, severity: "info", detail: "el repositorio está en orden" };
|
|
104
130
|
return {
|
|
105
131
|
ok: false,
|
|
@@ -10,7 +10,7 @@ import { withConfig } from "./_shared.js";
|
|
|
10
10
|
export function registerRolesSkills(program, { pkgVersion }) {
|
|
11
11
|
program
|
|
12
12
|
.command("skills [action] [args...]")
|
|
13
|
-
.description("Manage the skill cache (list | clear-cache | sync-addyosmani | list-addyosmani) and, through atril, the .karajan store (search | route | scan | install | remove | graph | enrich | config)")
|
|
13
|
+
.description("Manage the skill cache (list | clear-cache | sync-addyosmani | list-addyosmani) and, through atril, the .karajan store (search, which translates a non-English query first | route | scan | install | remove | graph | enrich | config)")
|
|
14
14
|
// KJC-TSK-1193: atril's options (--json, --kind, --role, --root) travel to atril untouched.
|
|
15
15
|
.allowUnknownOption()
|
|
16
16
|
.action(async (action, args) => {
|
package/src/commands/init.js
CHANGED
|
@@ -32,6 +32,7 @@ import { installKarajanCommands } from "./init-atril.js";
|
|
|
32
32
|
import { hardenCommand } from "./harden.js";
|
|
33
33
|
import { contractChanges } from "../environment/contract-commit.js";
|
|
34
34
|
import { detectProjectStack } from "../utils/stack-detect.js";
|
|
35
|
+
import { defaultBranchProblem } from "../checks/repo-state.js";
|
|
35
36
|
import { bootstrapSonarToken } from "../sonar/token-bootstrap.js";
|
|
36
37
|
|
|
37
38
|
/**
|
|
@@ -775,6 +776,15 @@ export async function resolveConfigScope({ flags, interactive }) {
|
|
|
775
776
|
return { configPath: getConfigPath(), scope: "global" };
|
|
776
777
|
}
|
|
777
778
|
|
|
779
|
+
/** KJC-BUG-0473 (#3080): said at the close, so a master repo never ends init looking governed. */
|
|
780
|
+
export function warnIfDefaultBranchNotMain({ projectDir, logger }) {
|
|
781
|
+
const problem = defaultBranchProblem(projectDir);
|
|
782
|
+
if (!problem) return false;
|
|
783
|
+
logger.warn(`⚠ ${problem.detail}`);
|
|
784
|
+
logger.warn(` Fix: ${problem.fix}`);
|
|
785
|
+
return true;
|
|
786
|
+
}
|
|
787
|
+
|
|
778
788
|
/**
|
|
779
789
|
* KJC-BUG-0187: the enforcement gates live in git hooks, so a project without
|
|
780
790
|
* a repo gets no harness at all. Init creates it, exactly like `kj run` does
|
|
@@ -1017,6 +1027,7 @@ export async function initCommand({ logger, flags = {} }) {
|
|
|
1017
1027
|
logger.info(hardened || skipHarden
|
|
1018
1028
|
? "✓ Karajan is set up in this project."
|
|
1019
1029
|
: "⚠ Karajan is set up EXCEPT the quality harness — without it there are no gates.");
|
|
1030
|
+
warnIfDefaultBranchNotMain({ projectDir: process.cwd(), logger });
|
|
1020
1031
|
logger.info(" Next: ask your agent for the work; it follows the method kj installed");
|
|
1021
1032
|
logger.info(" More: kj --help · kj doctor (check your setup)");
|
|
1022
1033
|
}
|
package/src/commands/rag.js
CHANGED
|
@@ -12,6 +12,8 @@ import { fileIndexState } from "../rag/coverage.js";
|
|
|
12
12
|
import { query } from "../rag/retriever.js";
|
|
13
13
|
import { pathNamedHits, withPathNamedHits } from "../rag/path-hits.js";
|
|
14
14
|
import { installPostMergeHook, maybeAutoUpdate } from "../rag/auto-update.js";
|
|
15
|
+
import { noteDeadline } from "../rag/bootstrap-index.js";
|
|
16
|
+
import { claimLease, indexLeasePath, releaseLease } from "../rag/pid-lease.js";
|
|
15
17
|
import { indexLibrary, LIBRARY_PROJECT } from "../rag/library.js";
|
|
16
18
|
import { compareToBaseline, loadGoldenQueries, runEval } from "../rag/eval.js";
|
|
17
19
|
import { getKarajanHome } from "../utils/paths.js";
|
|
@@ -47,8 +49,35 @@ async function indexFiles({ files, projectDir, db, embedder, slug, logger, json,
|
|
|
47
49
|
return totals;
|
|
48
50
|
}
|
|
49
51
|
|
|
50
|
-
|
|
52
|
+
// HU-3092: the background indexer is given the time it may stop at; unset, non-numeric or <= 0 means none.
|
|
53
|
+
const deadlineFromEnv = () => { const n = Number(process.env.KJ_RAG_DEADLINE); return Number.isFinite(n) && n > 0 ? n : null; };
|
|
54
|
+
|
|
55
|
+
// The partial index keeps its chunks but not its commit: the marker says why it stopped, the warning how to finish.
|
|
56
|
+
function reportDeadline({ totals, projectDir, logger, json }) {
|
|
57
|
+
noteDeadline(projectDir);
|
|
58
|
+
logger.warn?.(`[rag] KJ_RAG_DEADLINE reached after ${totals.files} file(s): the index is partial and its commit is not recorded; run kj rag index --with-sources by hand`);
|
|
59
|
+
if (json) process.stdout.write(`${JSON.stringify(totals)}\n`);
|
|
60
|
+
return totals;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// HU-3116: the cooperative stop only looks between files; a hung embed call is ended this long after the deadline.
|
|
64
|
+
// It stays inside DEADLINE_MARGIN_MS (60 s), so the child is gone before its lease reads as expired.
|
|
65
|
+
export const BACKSTOP_GRACE_MS = 30 * 1000;
|
|
66
|
+
|
|
67
|
+
export async function ragIndexCommand({ config, logger, flags = {}, backstop = {} }) {
|
|
68
|
+
const { setTimer = setTimeout, clearTimer = clearTimeout, exit = process.exit } = backstop;
|
|
51
69
|
const projectDir = config?.projectDir || process.cwd();
|
|
70
|
+
const deadline = deadlineFromEnv();
|
|
71
|
+
// Past already (a queued start, a slow machine): nothing is opened, claimed or embedded.
|
|
72
|
+
if (deadline !== null && Date.now() >= deadline) return reportDeadline({ totals: { indexed: 0, failed: 0, files: 0, stopped: "deadline" }, projectDir, logger, json: flags.json });
|
|
73
|
+
let heldLease = null;
|
|
74
|
+
const timer = deadline === null ? null : setTimer(() => {
|
|
75
|
+
if (heldLease) releaseLease(heldLease);
|
|
76
|
+
noteDeadline(projectDir);
|
|
77
|
+
logger.warn?.("[rag] KJ_RAG_DEADLINE passed with an embed call still hung: the index is partial and its commit is not recorded; run kj rag index --with-sources by hand");
|
|
78
|
+
exit(0);
|
|
79
|
+
}, Math.max(0, deadline + BACKSTOP_GRACE_MS - Date.now()));
|
|
80
|
+
timer?.unref?.();
|
|
52
81
|
const db = openDb(config);
|
|
53
82
|
try {
|
|
54
83
|
const slug = projectSlug(projectDir);
|
|
@@ -76,13 +105,22 @@ export async function ragIndexCommand({ config, logger, flags = {} }) {
|
|
|
76
105
|
}
|
|
77
106
|
}
|
|
78
107
|
if (!totals) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
108
|
+
// HU-2907: a full foreground index holds the machine lease, so background lanes defer to it.
|
|
109
|
+
// The bootstrap child finds its spawning parent in the lease (it names itself first): kin.
|
|
110
|
+
const lease = indexLeasePath();
|
|
111
|
+
const claim = claimLease(lease, { projectDir, slug }, { kin: [process.ppid] });
|
|
112
|
+
if (claim.holder) logger.warn?.(`[rag] another index is running (${claim.holder.slug ?? "?"}, pid ${claim.holder.pid}); indexing anyway`);
|
|
113
|
+
if (claim.claimed) heldLease = lease;
|
|
114
|
+
try {
|
|
115
|
+
totals = await indexProject(projectDir, {
|
|
116
|
+
db, embedder,
|
|
117
|
+
karajanHome: getKarajanHome(), logger,
|
|
118
|
+
withSources: Boolean(flags.withSources),
|
|
119
|
+
exclude: ragExclude(config), deadline,
|
|
120
|
+
});
|
|
121
|
+
} finally { if (claim.claimed) releaseLease(lease); }
|
|
85
122
|
}
|
|
123
|
+
if (totals.stopped) return reportDeadline({ totals, projectDir, logger, json: flags.json });
|
|
86
124
|
if (totals.head) setLastIndexedCommit(db, slug, totals.head);
|
|
87
125
|
// KJC-TSK-0697 — the library corpus refreshes on every index run.
|
|
88
126
|
// Best-effort and cheap: a handful of cards, content-hash dedup.
|
|
@@ -101,6 +139,7 @@ export async function ragIndexCommand({ config, logger, flags = {} }) {
|
|
|
101
139
|
}
|
|
102
140
|
return totals;
|
|
103
141
|
} finally {
|
|
142
|
+
if (timer) clearTimer(timer);
|
|
104
143
|
db.close();
|
|
105
144
|
}
|
|
106
145
|
}
|
package/src/commands/skills.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
import { listCached, clearCache, getCacheRoot, DEFAULT_TTL_MS } from "../skills/skills-cache.js";
|
|
14
14
|
import { refreshIfStale, listAvailableSlugs, getCatalogRoot, loadSkillBySlug } from "../skills/addyosmani-catalog.js";
|
|
15
15
|
import { ATRIL_COMMANDS, runAtril } from "../skills/atril-cli.js";
|
|
16
|
+
import { skillsSearch } from "../skills/skills-search.js";
|
|
16
17
|
|
|
17
18
|
function humanAge(cachedAt) {
|
|
18
19
|
const ms = Date.now() - Date.parse(cachedAt);
|
|
@@ -22,7 +23,9 @@ function humanAge(cachedAt) {
|
|
|
22
23
|
return `${days.toFixed(1)}d ago`;
|
|
23
24
|
}
|
|
24
25
|
|
|
25
|
-
export async function skillsCommand({ action = "list", args = [], deps = { atril: runAtril } } = {}) {
|
|
26
|
+
export async function skillsCommand({ action = "list", args = [], deps = { atril: runAtril, skillsSearch } } = {}) {
|
|
27
|
+
// HU-2782: search is kj's own, so the query can be translated into English first.
|
|
28
|
+
if (action === "search") return deps.skillsSearch({ args });
|
|
26
29
|
// KJC-TSK-1193: atril's own commands go to atril, on the .karajan store.
|
|
27
30
|
if (ATRIL_COMMANDS.includes(action)) return deps.atril({ action, args });
|
|
28
31
|
switch (action) {
|
package/src/commands/worktree.js
CHANGED
|
@@ -13,13 +13,17 @@ import { execFile } from "node:child_process";
|
|
|
13
13
|
import { promisify } from "node:util";
|
|
14
14
|
import { bootstrapWorktree } from "../hu/worktree-bootstrap.js";
|
|
15
15
|
import { laneWork, setupLane } from "../hu/lane-setup.js";
|
|
16
|
-
import { classifyLanes, commitsNotIn, liveProcesses, looseLaneBranches, resolveBase } from "../hu/lane-prune.js";
|
|
16
|
+
import { classifyLanes, commitsNotIn, liveProcesses, looseLaneBranches, resolveBase, stopLaneIndexers } from "../hu/lane-prune.js";
|
|
17
|
+
import { stopBootstrap } from "../rag/bootstrap-index.js";
|
|
17
18
|
import { recordGateDecision } from "../policy/decisions.js";
|
|
18
19
|
import { mainCheckout } from "../utils/main-checkout.js";
|
|
19
20
|
import { cardsOfLane, parseLanes, prsByBranch } from "../hu/lanes.js";
|
|
20
21
|
import { listPlans, loadPlan } from "../plan/plan-store.js";
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
// The pid itself, not the marker's expiry: an indexer past MAX_RUN_MS still runs. EPERM is a process too.
|
|
24
|
+
const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (err) { return err.code === "EPERM"; } };
|
|
25
|
+
|
|
26
|
+
const REF_SEPARATOR =" · "; // external_ref joins a tracker card and an issue (src/commands/hu.js)
|
|
23
27
|
|
|
24
28
|
// KJC-TSK-1316: ONE gh call for every lane, open and merged PRs alike.
|
|
25
29
|
const listPrs = async (cwd) => JSON.parse((await execFileAsync("gh", ["pr", "list", "--state", "all", "--limit", "200", "--json", "number,url,state,headRefName"], { cwd })).stdout);
|
|
@@ -121,6 +125,12 @@ export async function worktreeCommand({ config = null, action, args = [], flags
|
|
|
121
125
|
if (dirty && !flags.force) {
|
|
122
126
|
throw new Error(`lane "${slug}" has uncommitted changes — commit them (through the gate) or discard with --force`);
|
|
123
127
|
}
|
|
128
|
+
// HU-3114: the lane's background indexer must not outlive it (any age); a failure never blocks the removal.
|
|
129
|
+
const stopIndexer = deps.stopBootstrap ?? stopBootstrap;
|
|
130
|
+
const indexerStopped = await stopIndexer(wtPath, { force: true }).catch((err) => {
|
|
131
|
+
console.warn(`warning: could not stop the background indexer of "${slug}": ${err.message}`);
|
|
132
|
+
return null;
|
|
133
|
+
});
|
|
124
134
|
// The lane's real branch (headless kj-hu-* lanes share the folder); detached HEAD → null.
|
|
125
135
|
// symbolic-ref -q exits 1 on a detached HEAD; any other failure is a real error.
|
|
126
136
|
const branch = await git(["symbolic-ref", "--short", "-q", "HEAD"], wtPath).then((r) => r.stdout.trim() || null, (err) => {
|
|
@@ -140,13 +150,15 @@ export async function worktreeCommand({ config = null, action, args = [], flags
|
|
|
140
150
|
: ahead === null ? `branch ${branch} kept: base "${base}" not found, cannot tell if it is merged (git branch -D ${branch} to delete it)`
|
|
141
151
|
: `branch ${branch} kept: ${ahead} commit(s) not in ${base} (squash-merged or not fetched? git fetch, then git branch -D ${branch})`;
|
|
142
152
|
console.log(`✓ lane "${slug}" removed; ${note}`);
|
|
143
|
-
return { removed: true, slug, branch, branchDeleted, merged: branch ? ahead === 0 : null };
|
|
153
|
+
return { removed: true, slug, branch, branchDeleted, merged: branch ? ahead === 0 : null, indexerStopped };
|
|
144
154
|
}
|
|
145
155
|
|
|
146
156
|
if (action === "prune") {
|
|
147
157
|
const { processes = liveProcesses() } = deps;
|
|
148
158
|
const base = await resolveBase({ git, base: baseName });
|
|
149
159
|
const lanes = await classifyLanes({ git, laneRoot: lanesDir, base, processes, now: deps.now });
|
|
160
|
+
// HU-3092: a removable lane's own background indexer is stopped first (never in a dry run); one that survives keeps the lane.
|
|
161
|
+
if (!flags.dryRun) await stopLaneIndexers(lanes, { stop: deps.stopIndexer ?? ((l) => stopBootstrap(l.path, { force: true })), stillRunning: deps.indexerRunning ?? ((l) => pidAlive(l.indexer.pid)) });
|
|
150
162
|
const abandoned = lanes.filter((l) => l.reasons.length === 0);
|
|
151
163
|
const kept = lanes.filter((l) => l.reasons.length > 0);
|
|
152
164
|
for (const l of kept) console.log(`kept ${l.slug.padEnd(24)} ${l.reasons.join("; ")}`);
|
|
@@ -155,7 +167,7 @@ export async function worktreeCommand({ config = null, action, args = [], flags
|
|
|
155
167
|
const { empty: branches, withWork } = await looseLaneBranches({ git, base });
|
|
156
168
|
for (const b of withWork) console.log(`kept branch ${b.branch.padEnd(17)} ${b.reason}`);
|
|
157
169
|
if (flags.dryRun) {
|
|
158
|
-
for (const l of abandoned) console.log(`would retire ${l.slug} (${l.branch ?? "detached"})`);
|
|
170
|
+
for (const l of abandoned) console.log(`would retire ${l.slug} (${l.branch ?? "detached"})${l.indexer ? ` (stopping its background index, pid ${l.indexer.pid})` : ""}`);
|
|
159
171
|
for (const b of branches) console.log(`would delete branch ${b}`);
|
|
160
172
|
return { wouldRemove: abandoned, kept, wouldRemoveBranches: branches, branchesKept: withWork };
|
|
161
173
|
}
|
package/src/config/defaults.js
CHANGED
|
@@ -28,6 +28,7 @@ const DEFAULTS = {
|
|
|
28
28
|
discover: { provider: null, model: null },
|
|
29
29
|
architect: { provider: null, model: null },
|
|
30
30
|
hu_reviewer: { provider: null, model: null },
|
|
31
|
+
translator: { provider: null, model: null },
|
|
31
32
|
start: { provider: null, model: "haiku" }
|
|
32
33
|
},
|
|
33
34
|
pipeline: {
|
|
@@ -65,6 +65,7 @@ const CODER_INHERITED_ROLES = new Set([
|
|
|
65
65
|
"impeccable", "triage", "discover", "architect", "audit",
|
|
66
66
|
"hu_reviewer", "hu-reviewer",
|
|
67
67
|
"spec_reviewer", "spec-reviewer",
|
|
68
|
+
"translator",
|
|
68
69
|
]);
|
|
69
70
|
|
|
70
71
|
function resolveProvider(roleConfig, role, roles, legacyCoder, legacyReviewer) {
|
package/src/config/schema.js
CHANGED
|
@@ -134,9 +134,22 @@ export const holdLane = (top, now = Date.now()) => {
|
|
|
134
134
|
return "(no se pudo reclamar " + top + ")";
|
|
135
135
|
}
|
|
136
136
|
};
|
|
137
|
+
// KJC-BUG-0475 (#3125): during a rebase HEAD is detached; the branch being rebased is in head-name.
|
|
138
|
+
const rebasingBranch = (top) => {
|
|
139
|
+
for (const dir of ["rebase-merge", "rebase-apply"]) {
|
|
140
|
+
try {
|
|
141
|
+
const file = execSync("git rev-parse --path-format=absolute --git-path " + dir + "/head-name", { cwd: top, stdio: ["ignore", "pipe", "ignore"] }).toString().trim();
|
|
142
|
+
const name = readFileSync(file, "utf8").trim();
|
|
143
|
+
if (name.startsWith("refs/heads/")) return name.slice("refs/heads/".length);
|
|
144
|
+
} catch { /* no rebase of this kind in progress */ }
|
|
145
|
+
}
|
|
146
|
+
return null;
|
|
147
|
+
};
|
|
137
148
|
export const branchOf = (top = laneTop()) => {
|
|
138
|
-
|
|
149
|
+
let branch;
|
|
150
|
+
try { branch = execSync("git rev-parse --abbrev-ref HEAD", { cwd: top, stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); }
|
|
139
151
|
catch { return null; }
|
|
152
|
+
return branch === "HEAD" ? (rebasingBranch(top) ?? branch) : branch;
|
|
140
153
|
};
|
|
141
154
|
// ADR 0034 (KJC-TSK-1243): the lane a write lands in is the worktree holding its path when
|
|
142
155
|
// that worktree is of this repo; any other path is judged in the session's lane. It walks up
|
package/src/hu/lane-prune.js
CHANGED
|
@@ -5,31 +5,61 @@
|
|
|
5
5
|
* touched, so a session may run this without risking anyone's work.
|
|
6
6
|
*/
|
|
7
7
|
import fs from "node:fs";
|
|
8
|
+
import { spawnSync } from "node:child_process";
|
|
8
9
|
import path from "node:path";
|
|
9
|
-
import { laneWork } from "./lane-setup.js";
|
|
10
|
+
import { isLaneSetupPath, laneWork } from "./lane-setup.js";
|
|
11
|
+
import { readBootstrap } from "../rag/bootstrap-index.js";
|
|
10
12
|
|
|
11
13
|
const IDLE_HOURS = 24;
|
|
14
|
+
// HU-3092: the lane's index and what rides along it (-wal, -shm, .bootstrap, .embed-schema) is regenerable state.
|
|
15
|
+
const RAG_STATE = /^\.karajan\/rag\.db([.-]|$)/;
|
|
12
16
|
const REGENERABLE = new Set(["node_modules", "dist", "build", "coverage", ".venv", "venv", "__pycache__", ".cache", "target"]);
|
|
13
17
|
|
|
18
|
+
const LSOF = "/usr/sbin/lsof";
|
|
19
|
+
const LSOF_TIMEOUT_MS = 10_000;
|
|
20
|
+
|
|
21
|
+
/** HU-3117: `lsof -Fpn` output is `p<pid>` then `n<path>` lines; a pid with no path line is skipped. */
|
|
22
|
+
export function parseLsofCwds(out) {
|
|
23
|
+
const procs = [];
|
|
24
|
+
let pid = null;
|
|
25
|
+
for (const line of out.split("\n")) {
|
|
26
|
+
if (line.startsWith("p") && /^\d+$/.test(line.slice(1))) pid = Number(line.slice(1));
|
|
27
|
+
else if (line.startsWith("n") && pid !== null) { procs.push({ pid, cwd: line.slice(1) }); pid = null; }
|
|
28
|
+
}
|
|
29
|
+
return procs;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** darwin has no /proc: lsof lists this user's cwds. null when it is missing, fails or times out (never "no processes"). */
|
|
33
|
+
function lsofProcesses({ run, uid }) {
|
|
34
|
+
const r = run(LSOF, ["-a", "-d", "cwd", "-u", String(uid), "-Fpn"], { encoding: "utf8", timeout: LSOF_TIMEOUT_MS });
|
|
35
|
+
if (r.error || r.status !== 0 || typeof r.stdout !== "string") return null;
|
|
36
|
+
const procs = parseLsofCwds(r.stdout);
|
|
37
|
+
return { cwds: new Set(procs.map((p) => p.cwd)), unseen: 0, procs };
|
|
38
|
+
}
|
|
39
|
+
|
|
14
40
|
/**
|
|
15
41
|
* This user's live processes (another user's cannot write in our worktrees): the
|
|
16
42
|
* cwds that can be read and how many cannot (non-dumpable ones: ssh-agent, systemd,
|
|
17
|
-
* some JVMs). null when the platform
|
|
43
|
+
* some JVMs). null when the platform's listing is unavailable (no /proc, no lsof).
|
|
18
44
|
*/
|
|
19
|
-
export function liveProcesses() {
|
|
45
|
+
export function liveProcesses({ platform = process.platform, run = spawnSync, uid = process.getuid() } = {}) {
|
|
46
|
+
if (platform === "darwin") return lsofProcesses({ run, uid });
|
|
20
47
|
let pids;
|
|
21
48
|
try { pids = fs.readdirSync("/proc").filter((p) => /^\d+$/.test(p)); } catch { return null; }
|
|
22
49
|
const cwds = new Set();
|
|
50
|
+
const procs = [];
|
|
23
51
|
let unseen = 0;
|
|
24
52
|
for (const pid of pids) {
|
|
25
53
|
try {
|
|
26
54
|
if (fs.statSync(`/proc/${pid}`).uid !== process.getuid()) continue;
|
|
27
|
-
|
|
55
|
+
const cwd = fs.readlinkSync(`/proc/${pid}/cwd`);
|
|
56
|
+
cwds.add(cwd);
|
|
57
|
+
procs.push({ pid: Number(pid), cwd });
|
|
28
58
|
} catch (err) {
|
|
29
59
|
if (err.code !== "ENOENT" && err.code !== "ESRCH") unseen += 1;
|
|
30
60
|
}
|
|
31
61
|
}
|
|
32
|
-
return { cwds, unseen };
|
|
62
|
+
return { cwds, unseen, procs };
|
|
33
63
|
}
|
|
34
64
|
|
|
35
65
|
/** Hours since anything in the lane moved: its dir, and its git index and HEAD. */
|
|
@@ -43,6 +73,32 @@ async function idleHours(git, lanePath, now) {
|
|
|
43
73
|
|
|
44
74
|
const inside = (cwd, dir) => cwd === dir || cwd.startsWith(dir + path.sep);
|
|
45
75
|
|
|
76
|
+
/** A folder git lists whole (an ignored `.karajan/`) holds only index state and setup when every file in it does. */
|
|
77
|
+
function onlyRagState(lanePath, dir) {
|
|
78
|
+
try {
|
|
79
|
+
return fs.readdirSync(path.join(lanePath, dir), { recursive: true, withFileTypes: true }).filter((d) => !d.isDirectory()) // symlinks and the like are the user's, never state
|
|
80
|
+
.every((d) => { const rel = path.relative(lanePath, path.join(d.parentPath, d.name)); return RAG_STATE.test(rel) || isLaneSetupPath(rel); });
|
|
81
|
+
} catch { return false; }
|
|
82
|
+
}
|
|
83
|
+
const ragOnly = (lanePath, p) => RAG_STATE.test(p) || (p.endsWith("/") && onlyRagState(lanePath, p));
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The pid of the lane's background indexer: the one its marker names, while the marker notes no stop.
|
|
87
|
+
* null when KJ_RAG_DB puts the index elsewhere: the marker read is then not this lane's.
|
|
88
|
+
*/
|
|
89
|
+
function indexerOf(lanePath) {
|
|
90
|
+
if (process.env.KJ_RAG_DB) return null;
|
|
91
|
+
const marker = readBootstrap(lanePath);
|
|
92
|
+
return Number.isInteger(marker?.pid) && !marker.stoppedAt ? marker.pid : null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** D6: stops each removable lane's indexer; a lane whose indexer survives (marker still running) is kept, saying so. */
|
|
96
|
+
export async function stopLaneIndexers(lanes, { stop, stillRunning }) {
|
|
97
|
+
for (const l of lanes.filter((x) => x.indexer && x.reasons.length === 0)) {
|
|
98
|
+
if (!(await stop(l)) && (await stillRunning(l))) l.reasons.push(`its background index (pid ${l.indexer.pid}) could not be stopped`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
46
102
|
/** @returns {Promise<{slug: string, path: string, branch: string|null, reasons: string[]}[]>} */
|
|
47
103
|
export async function classifyLanes({ git, laneRoot, base, processes, now = Date.now() }) {
|
|
48
104
|
const { stdout } = await git(["worktree", "list", "--porcelain"]);
|
|
@@ -58,7 +114,7 @@ export async function classifyLanes({ git, laneRoot, base, processes, now = Date
|
|
|
58
114
|
const reasons = [];
|
|
59
115
|
// KJC-TSK-1371: the identity and Sentinel files lane setup wrote are not the lane's work.
|
|
60
116
|
const { stdout: status } = await git(["status", "--porcelain", "--ignored=matching", "--untracked-files=all"], lane.path);
|
|
61
|
-
const entries = laneWork(status, lane.path);
|
|
117
|
+
const entries = laneWork(status, lane.path).filter((e) => !ragOnly(lane.path, e.slice(3)));
|
|
62
118
|
if (entries.some((e) => !e.startsWith("!! "))) reasons.push("uncommitted changes");
|
|
63
119
|
// Ignored files can be valuable (.env, local artifacts); only regenerable dirs are not.
|
|
64
120
|
const precious = entries.filter((e) => e.startsWith("!! ") && !REGENERABLE.has(e.slice(3).split("/")[0]));
|
|
@@ -66,12 +122,18 @@ export async function classifyLanes({ git, laneRoot, base, processes, now = Date
|
|
|
66
122
|
// The lane's HEAD, branch or detached: a detached lane's commits have no other name.
|
|
67
123
|
const ahead = await commitsNotIn({ git, base, cwd: lane.path });
|
|
68
124
|
if (ahead > 0) reasons.push(`${ahead} commit(s) not in ${base}`);
|
|
125
|
+
// The lane's own indexer alone is no reason to keep it (HU-3092); without pids (older callers) any process still is.
|
|
126
|
+
const inLane = (processes?.procs ?? []).filter((p) => inside(p.cwd, lane.path));
|
|
127
|
+
const indexer = processes?.procs ? indexerOf(lane.path) : null;
|
|
128
|
+
const busy = processes !== null && [...processes.cwds].some((cwd) => inside(cwd, lane.path));
|
|
129
|
+
const owned = busy && indexer !== null && inLane.every((p) => p.pid === indexer) ? { pid: indexer } : null;
|
|
69
130
|
if (processes === null) reasons.push("cannot check for live processes on this platform");
|
|
70
|
-
else if (
|
|
131
|
+
else if (busy && !owned) reasons.push("a live process works in it");
|
|
132
|
+
// The owned indexer excluded, processes that cannot be inspected still protect a recently active lane.
|
|
71
133
|
else if (processes.unseen > 0 && (await idleHours(git, lane.path, now)) < IDLE_HOURS) {
|
|
72
134
|
reasons.push(`${processes.unseen} process(es) of yours cannot be inspected and the lane changed in the last ${IDLE_HOURS} h`);
|
|
73
135
|
}
|
|
74
|
-
return { slug: path.basename(lane.path), path: lane.path, branch: lane.branch, reasons };
|
|
136
|
+
return { slug: path.basename(lane.path), path: lane.path, branch: lane.branch, reasons, indexer: owned };
|
|
75
137
|
}));
|
|
76
138
|
}
|
|
77
139
|
|
|
@@ -10,6 +10,7 @@ import { indexProject } from "../../rag/indexer.js";
|
|
|
10
10
|
import { ragExclude } from "../../rag/indexable.js";
|
|
11
11
|
import { query } from "../../rag/retriever.js";
|
|
12
12
|
import { pathNamedHits, withPathNamedHits } from "../../rag/path-hits.js";
|
|
13
|
+
import { claimLease, indexLeasePath, releaseLease } from "../../rag/pid-lease.js";
|
|
13
14
|
import { getKarajanHome } from "../../utils/paths.js";
|
|
14
15
|
import { resolveProjectDir, buildConfig, responseText, failPayload } from "../shared-helpers.js";
|
|
15
16
|
|
|
@@ -56,7 +57,11 @@ export async function handleRagIndex(args, server) {
|
|
|
56
57
|
const projectDir = await resolveProjectDir(server, args?.projectDir);
|
|
57
58
|
const config = await buildConfig({ ...args, projectDir }, "rag-index");
|
|
58
59
|
const db = openProjectStore({ projectDir, dim: config?.rag?.embedder?.dim || 768 });
|
|
60
|
+
// HU-2907: the long-lived host holds the machine lease for the walk and releases it in the finally.
|
|
61
|
+
const lease = indexLeasePath();
|
|
62
|
+
let claim;
|
|
59
63
|
try {
|
|
64
|
+
claim = claimLease(lease, { projectDir, slug: projectSlug(projectDir) });
|
|
60
65
|
const totals = await indexProject(projectDir, {
|
|
61
66
|
db, embedder: makeGovernedEmbedder(config),
|
|
62
67
|
karajanHome: getKarajanHome(), logger: silentLogger,
|
|
@@ -64,7 +69,10 @@ export async function handleRagIndex(args, server) {
|
|
|
64
69
|
exclude: ragExclude(config), // KJC-TSK-0891
|
|
65
70
|
});
|
|
66
71
|
return responseText(totals);
|
|
67
|
-
} finally {
|
|
72
|
+
} finally {
|
|
73
|
+
if (claim?.claimed) releaseLease(lease);
|
|
74
|
+
db.close();
|
|
75
|
+
}
|
|
68
76
|
} catch (err) {
|
|
69
77
|
return failPayload(`kj_rag_index failed: ${err.message}`);
|
|
70
78
|
}
|
package/src/rag/auto-update.js
CHANGED
|
@@ -9,7 +9,8 @@ import { execa } from "execa";
|
|
|
9
9
|
import { projectSlug, getLastIndexedCommit, setLastIndexedCommit } from "./vec-store.js";
|
|
10
10
|
import { indexProject, indexProjectDelta } from "./indexer.js";
|
|
11
11
|
import { readRefreshedPaths, untrackedFiles, writeRefreshedPaths } from "./worktree-files.js";
|
|
12
|
-
import { startBootstrap } from "./bootstrap-index.js";
|
|
12
|
+
import { startBootstrap, stopBootstrap } from "./bootstrap-index.js";
|
|
13
|
+
import { indexLeasePath, stopLeaseHolder } from "./pid-lease.js";
|
|
13
14
|
import { ragExclude } from "./indexable.js";
|
|
14
15
|
import { makeGovernedEmbedder } from "./governed-embedder.js";
|
|
15
16
|
import { openProjectStore } from "./project-store.js";
|
|
@@ -33,8 +34,21 @@ const HOOK_SRC = resolve(fileURLToPath(import.meta.url), "../../../scripts/git-h
|
|
|
33
34
|
|
|
34
35
|
// KJC-TSK-1174 (RAG-A2): no index yet. It is built in the background, once; the session
|
|
35
36
|
// does not wait. A recent attempt that died (no Ollama) is not retried blindly.
|
|
36
|
-
function bootstrapInBackground({ projectDir, slug, head, logger }) {
|
|
37
|
+
async function bootstrapInBackground({ projectDir, slug, head, logger }) {
|
|
38
|
+
// HU-2758: an indexer past MAX_RUN_MS is stopped first: this project's, then whichever holds the machine lease
|
|
39
|
+
// (an orphan of an older build that no marker names). A failure here never keeps the bootstrap from happening.
|
|
40
|
+
for (const stopped of [await stopBootstrap(projectDir).catch(() => null), await stopLeaseHolder(indexLeasePath()).catch(() => null)]) {
|
|
41
|
+
if (!stopped) continue;
|
|
42
|
+
logger.info?.(`[rag] stopped the background index of ${stopped.slug ?? slug} (pid ${stopped.pid}): it ran past 60 min`);
|
|
43
|
+
// The stop never reached the marker (busy mutex): nothing on file says it was stopped, so it may start again.
|
|
44
|
+
if (stopped.marked === false) warnOnce(logger, `bootstrap-unmarked:${projectDir}`, `[rag] the stop of the background index of ${stopped.slug ?? slug} (pid ${stopped.pid}) could not be noted: it may be started again on the next command; run kj rag index --with-sources by hand`);
|
|
45
|
+
}
|
|
37
46
|
const boot = startBootstrap(projectDir);
|
|
47
|
+
if (boot.stopped) {
|
|
48
|
+
const why = boot.stopped === "deadline" ? "reached its deadline and left a partial index" : "ran past 60 min and was stopped";
|
|
49
|
+
warnOnce(logger, `bootstrap-stopped:${projectDir}`, `[rag] the background index of ${slug} ${why}: run kj rag index --with-sources by hand`);
|
|
50
|
+
return { skipped: true, reason: "bootstrap-failed", head };
|
|
51
|
+
}
|
|
38
52
|
if (boot.busy) {
|
|
39
53
|
warnOnce(logger, `bootstrap-busy:${projectDir}`, `[rag] ${slug} has no index yet: another background index is running (${boot.busy.slug ?? "?"}, pid ${boot.busy.pid ?? "?"}); it is tried again on the next command`);
|
|
40
54
|
return { skipped: true, reason: "bootstrap-queued", holder: boot.busy, head };
|
|
@@ -93,7 +107,7 @@ export async function maybeAutoUpdate({ projectDir, config, logger = console, fl
|
|
|
93
107
|
const db = openProjectStore({ projectDir, dim: config?.rag?.embedder?.dim || 768 });
|
|
94
108
|
try {
|
|
95
109
|
const since = getLastIndexedCommit(db, slug);
|
|
96
|
-
if (!since) return bootstrapInBackground({ projectDir, slug, head, logger });
|
|
110
|
+
if (!since) return await bootstrapInBackground({ projectDir, slug, head, logger });
|
|
97
111
|
// KJC-BUG-0304: a drift of hundreds of files (an index weeks old) reindexed the
|
|
98
112
|
// repo for half an hour before the query answered. Past the bound the query
|
|
99
113
|
// answers from the index as it is, and says how to bring it up to date.
|
|
@@ -6,12 +6,14 @@ import { dirname, resolve } from "node:path";
|
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { projectDbPath } from "./project-store.js";
|
|
8
8
|
import { projectSlug } from "./vec-store.js";
|
|
9
|
-
import { GRACE_MS, MAX_RUN_MS, indexLeasePath, readLease, takeLock, unlessGone, withMutex, writeLease } from "./pid-lease.js";
|
|
9
|
+
import { GRACE_MS, MAX_RUN_MS, indexLeasePath, noteStopped, readLease, stopLeaseHolder, takeLock, unlessGone, withMutex, writeLease } from "./pid-lease.js";
|
|
10
10
|
|
|
11
11
|
const KJ_BIN = resolve(fileURLToPath(import.meta.url), "../../../bin/kj.js");
|
|
12
12
|
// An attempt that died (no Ollama, no model) is not repeated on every command: this long apart.
|
|
13
13
|
export const RETRY_MS = 10 * 60 * 1000;
|
|
14
14
|
export { GRACE_MS, MAX_RUN_MS };
|
|
15
|
+
// HU-3092: the child stops itself this long before MAX_RUN_MS, so it ends before the lease reads as expired.
|
|
16
|
+
export const DEADLINE_MARGIN_MS = 60 * 1000;
|
|
15
17
|
const markerPath = (projectDir) => `${projectDbPath(projectDir)}.bootstrap`;
|
|
16
18
|
|
|
17
19
|
/** What the marker says of the last attempt: `{ pid, at, running }`, or null when none was ever made. */
|
|
@@ -33,7 +35,10 @@ export function startBootstrap(projectDir, { spawnFn = spawn, bin = KJ_BIN, now
|
|
|
33
35
|
try {
|
|
34
36
|
const last = readBootstrap(projectDir, { now });
|
|
35
37
|
if (last?.running) return { started: false, pid: last.pid };
|
|
36
|
-
|
|
38
|
+
// HU-2758: one stopped for running too long is not respawned in the background, at any age.
|
|
39
|
+
// HU-3092: so is one that stopped itself at its deadline.
|
|
40
|
+
if (last?.stoppedBy === "max-run" || last?.stoppedBy === "deadline") return { started: false, recent: true, stopped: last.stoppedBy };
|
|
41
|
+
if (last && now - (last.stoppedAt ?? last.at) < RETRY_MS) return { started: false, recent: true, pid: null };
|
|
37
42
|
// One indexer per machine: the lease is taken inside the project mutex, before the marker is
|
|
38
43
|
// written, so a queued project does not spend its RETRY_MS. Both mutexes yield, never wait.
|
|
39
44
|
const slug = projectSlug(projectDir);
|
|
@@ -42,10 +47,11 @@ export function startBootstrap(projectDir, { spawnFn = spawn, bin = KJ_BIN, now
|
|
|
42
47
|
if (holder?.running && holder.pid !== process.pid) return { started: false, busy: { projectDir: holder.projectDir, slug: holder.slug, pid: holder.pid } };
|
|
43
48
|
// Marker and lease land whole each time: first with the claimant's pid, so they read as
|
|
44
49
|
// running meanwhile, then with the indexer's.
|
|
45
|
-
const
|
|
50
|
+
const command = [process.execPath, bin, "rag", "index", "--with-sources"].join(" ");
|
|
51
|
+
const note = (pid) => { writeLease(file, { pid, at: now, command }); writeLease(lease, { pid, at: now, command, projectDir, slug }); };
|
|
46
52
|
note(process.pid);
|
|
47
53
|
try {
|
|
48
|
-
const child = spawnFn(process.execPath, [bin, "rag", "index", "--with-sources"], { cwd: projectDir, detached: true, stdio: "ignore" });
|
|
54
|
+
const child = spawnFn(process.execPath, [bin, "rag", "index", "--with-sources"], { cwd: projectDir, detached: true, stdio: "ignore", env: { ...process.env, KJ_RAG_DEADLINE: String(now + MAX_RUN_MS - DEADLINE_MARGIN_MS) } });
|
|
49
55
|
child.unref?.();
|
|
50
56
|
note(child.pid);
|
|
51
57
|
return { started: true, pid: child.pid };
|
|
@@ -58,3 +64,15 @@ export function startBootstrap(projectDir, { spawnFn = spawn, bin = KJ_BIN, now
|
|
|
58
64
|
return res.busy === true ? { started: false, busy: {} } : res;
|
|
59
65
|
} finally { unlessGone(() => rmdirSync(lock)); }
|
|
60
66
|
}
|
|
67
|
+
|
|
68
|
+
/** HU-3092: the indexer that reached its deadline notes it, only while the marker still names it. */
|
|
69
|
+
export const noteDeadline = (projectDir, { pid = process.pid, now = Date.now(), wait } = {}) => noteStopped(markerPath(projectDir), pid, now, "deadline", wait);
|
|
70
|
+
|
|
71
|
+
// HU-2758: stops the marker's indexer past MAX_RUN_MS (`force`: any age) and frees the lease naming it.
|
|
72
|
+
export async function stopBootstrap(projectDir, { lease = indexLeasePath(), now = Date.now(), ...opts } = {}) {
|
|
73
|
+
const stopped = await stopLeaseHolder(markerPath(projectDir), { ...opts, now });
|
|
74
|
+
// `marked: false` (the stop never reached the marker): the lease is kept, naming a dead pid, which readLease
|
|
75
|
+
// already reads as free. The caller warns; nothing here pretends the stop is on file.
|
|
76
|
+
if (stopped?.marked) withMutex(lease, now, () => { if (readLease(lease, { now })?.pid === stopped.pid) unlessGone(() => unlinkSync(lease)); });
|
|
77
|
+
return stopped;
|
|
78
|
+
}
|
package/src/rag/indexer.js
CHANGED
|
@@ -139,7 +139,7 @@ async function listFiles(dir, predicate) {
|
|
|
139
139
|
* indexing tens of thousands of files is expensive — the CLI exposes
|
|
140
140
|
* it as `--with-sources`.
|
|
141
141
|
*/
|
|
142
|
-
export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false, exclude = [] } = {}) {
|
|
142
|
+
export async function indexProject(projectDir, { db, embedder, karajanHome, logger = console, withSources = false, exclude = [], deadline = null, now = Date.now } = {}) {
|
|
143
143
|
// KJC-TSK-0640: a full index must stamp HEAD too — without it,
|
|
144
144
|
// last_indexed_commit stayed null after the FIRST index, so the drift
|
|
145
145
|
// delta-update (maybeAutoUpdate) never engaged until a manual --since.
|
|
@@ -150,20 +150,23 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
|
|
|
150
150
|
} catch { /* not a git repo (or zero commits) — nothing to stamp */ }
|
|
151
151
|
const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
|
|
152
152
|
await prepareAdapters(detectAdaptersForProject(projectDir), { logger });
|
|
153
|
+
// HU-3092: past the deadline (a background indexer's) the walk stops between files; what it indexed stays.
|
|
154
|
+
const stop = () => { if (deadline != null && now() >= deadline) totals.stopped = "deadline"; return Boolean(totals.stopped); };
|
|
153
155
|
const planRoot = join(karajanHome, PLANS_DIR, slug);
|
|
154
156
|
if (existsSync(planRoot)) {
|
|
155
157
|
const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
|
|
156
158
|
for (const p of plans) {
|
|
159
|
+
if (stop()) break;
|
|
157
160
|
const r = await indexFile(p, { db, embedder, logger, project: slug, root: projectDir });
|
|
158
161
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
159
162
|
}
|
|
160
163
|
}
|
|
161
164
|
const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
|
|
162
|
-
if (existsSync(onboarding)) {
|
|
165
|
+
if (existsSync(onboarding) && !stop()) {
|
|
163
166
|
const r = await indexFile(onboarding, { db, embedder, logger, project: slug, root: projectDir });
|
|
164
167
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
165
168
|
}
|
|
166
|
-
if (withSources) {
|
|
169
|
+
if (withSources && !totals.stopped) {
|
|
167
170
|
// KJC-BUG-0062: `.kj/worktrees/HU-*/` contains git worktrees Karajan spawns
|
|
168
171
|
// per HU during `kj run`; they hold complete copies of `src/` that
|
|
169
172
|
// duplicate every chunk and contaminate retrieval scores. `_diet` is the
|
|
@@ -182,12 +185,14 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
|
|
|
182
185
|
if (source.startsWith(prefix) && outOf(source) !== null) deleteChunksBySource(db, source);
|
|
183
186
|
}
|
|
184
187
|
for (const s of sources) {
|
|
188
|
+
if (stop()) break;
|
|
185
189
|
const r = await indexFile(s, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
|
|
186
190
|
totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
|
|
187
191
|
}
|
|
188
192
|
// KJC-TSK-1171: only a COMPLETE index of the sources carries the embed schema; a file the
|
|
189
193
|
// embedder failed on leaves the marker as it was, so the rebuild is asked for again.
|
|
190
|
-
if (totals.
|
|
194
|
+
if (totals.stopped) logger.info?.("[rag-indexer] deadline reached: the index is partial, no embed schema recorded");
|
|
195
|
+
else if (totals.failed === 0) writeEmbedSchema(projectDir);
|
|
191
196
|
else logger.warn?.(`[rag-indexer] ${totals.failed} chunk(s) failed: the embed schema is not recorded, the index will be rebuilt again`);
|
|
192
197
|
}
|
|
193
198
|
return totals;
|
package/src/rag/pid-lease.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
// HU-2757: a lease names the pid that holds a resource and when it took it.
|
|
2
|
-
//
|
|
3
|
-
|
|
1
|
+
// HU-2757: a lease names the pid that holds a resource and when it took it. The background indexers
|
|
2
|
+
// never release it (HU-2907: a foreground `kj rag index` releases its own): a dead pid, or an age
|
|
3
|
+
// of MAX_RUN_MS, means free. A mkdir mutex makes the decision atomic.
|
|
4
|
+
import { execFileSync } from "node:child_process";
|
|
5
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
4
6
|
import { dirname, join } from "node:path";
|
|
5
7
|
import { getKarajanHome } from "../utils/paths.js";
|
|
6
8
|
|
|
@@ -29,7 +31,7 @@ export function readLease(file, { now = Date.now() } = {}) {
|
|
|
29
31
|
if (raw === null) return null;
|
|
30
32
|
try {
|
|
31
33
|
const note = JSON.parse(raw);
|
|
32
|
-
return { ...note, running: alive(note.pid) && now - note.at < MAX_RUN_MS };
|
|
34
|
+
return { ...note, running: !note.stoppedAt && alive(note.pid) && now - note.at < MAX_RUN_MS };
|
|
33
35
|
} catch {
|
|
34
36
|
// Another command is writing it this very moment: running for a moment; past the grace, a crash.
|
|
35
37
|
const stat = unlessGone(() => statSync(file));
|
|
@@ -68,3 +70,112 @@ export function withMutex(file, now, fn) {
|
|
|
68
70
|
if (!takeLock(lock, now)) return { busy: true };
|
|
69
71
|
try { return fn(); } finally { unlessGone(() => rmdirSync(lock)); }
|
|
70
72
|
}
|
|
73
|
+
|
|
74
|
+
// HU-2758: an indexer past MAX_RUN_MS is stopped from outside, once ps confirms the pid still runs it.
|
|
75
|
+
export const KILL_AFTER_MS = 3000;
|
|
76
|
+
const POLL_MS = 100;
|
|
77
|
+
const START_SLACK_MS = 60 * 1000;
|
|
78
|
+
// By absolute path: a ps found through PATH could be forged (as in harden/human-act.js).
|
|
79
|
+
const PS_BIN = ["/bin/ps", "/usr/bin/ps"].find((p) => existsSync(p)) ?? null;
|
|
80
|
+
|
|
81
|
+
export function parsePs(out) {
|
|
82
|
+
const text = String(out ?? "").trim();
|
|
83
|
+
const split = text.search(/\s/);
|
|
84
|
+
const m = /^(?:(?:(\d+)-)?(\d+):)?(\d+):(\d+)$/.exec(text.slice(0, split < 0 ? 0 : split));
|
|
85
|
+
const command = text.slice(split).trim();
|
|
86
|
+
if (!m || split < 0 || !command) return null;
|
|
87
|
+
const [d, h, min, s] = [m[1], m[2], m[3], m[4]].map((n) => Number(n ?? 0));
|
|
88
|
+
return { elapsedMs: (((d * 24 + h) * 60 + min) * 60 + s) * 1000, command };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const psInfo = (pid) => {
|
|
92
|
+
if (!PS_BIN) return null;
|
|
93
|
+
try { return parsePs(execFileSync(PS_BIN, ["-o", "etime=", "-o", "command=", "-p", String(pid)], { encoding: "utf8", env: {}, timeout: 2000 })); }
|
|
94
|
+
catch { return null; }
|
|
95
|
+
};
|
|
96
|
+
// ps flattens argv, so the identity is the exact command recorded at spawn; none recorded (older format): not ours.
|
|
97
|
+
const isIndexer = (info, at, now, command) => Boolean(info && command) && info.command === command && info.elapsedMs >= Math.max(0, now - at - START_SLACK_MS);
|
|
98
|
+
const isAliveVia = (kill, pid) => { try { kill(pid, 0); return true; } catch { return false; } };
|
|
99
|
+
const signal = (kill, pid, sig) => {
|
|
100
|
+
try { kill(-pid, sig); } catch { try { kill(pid, sig); } catch (err) { if (err.code !== "ESRCH") throw err; } }
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/** SIGTERM to a verified indexer, SIGKILL if it outlives `killAfterMs`. True only once it is confirmed gone. */
|
|
104
|
+
export async function stopIndexer(pid, at, { command, now = Date.now(), ps = psInfo, kill = (p, s) => process.kill(p, s), sleep = (ms) => new Promise((r) => setTimeout(r, ms)), killAfterMs = KILL_AFTER_MS } = {}) {
|
|
105
|
+
if (!isIndexer(ps(pid), at, now, command)) return false;
|
|
106
|
+
signal(kill, pid, "SIGTERM");
|
|
107
|
+
for (let waited = 0; waited < killAfterMs; waited += POLL_MS) {
|
|
108
|
+
await sleep(POLL_MS);
|
|
109
|
+
if (!isAliveVia(kill, pid)) return true;
|
|
110
|
+
}
|
|
111
|
+
const again = ps(pid); // verified again: no answer is gone-or-unverified, never signalled blind
|
|
112
|
+
if (!again) return !isAliveVia(kill, pid);
|
|
113
|
+
if (!isIndexer(again, at, now, command)) return true; // the pid was reused: the indexer is gone
|
|
114
|
+
signal(kill, pid, "SIGKILL");
|
|
115
|
+
await sleep(POLL_MS);
|
|
116
|
+
return !isAliveVia(kill, pid);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const sleepSync = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
120
|
+
// Notes the stop while the file still names that pid: 5 tries at the mutex, 20 ms apart; false if never free.
|
|
121
|
+
export const noteStopped = (file, pid, now, reason, wait = sleepSync) => {
|
|
122
|
+
const mark = () => withMutex(file, now, () => {
|
|
123
|
+
const { running: _running, ...note } = readLease(file, { now }) ?? {};
|
|
124
|
+
if (note.pid === pid && !note.stoppedAt) writeLease(file, { ...note, stoppedAt: now, stoppedBy: reason });
|
|
125
|
+
return true;
|
|
126
|
+
});
|
|
127
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
128
|
+
if (attempt > 0) wait(20);
|
|
129
|
+
if (mark() === true) return true;
|
|
130
|
+
}
|
|
131
|
+
return false;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// Stops the process `file` names past MAX_RUN_MS (or `force`) and notes it there: `{ pid, slug }` or null.
|
|
135
|
+
export async function stopLeaseHolder(file, { force = false, now = Date.now(), ...io } = {}) {
|
|
136
|
+
const held = readLease(file, { now });
|
|
137
|
+
const kill = io.kill ?? ((p, s) => process.kill(p, s));
|
|
138
|
+
if (!Number.isInteger(held?.pid) || held.pid <= 1 || held.pid === process.pid || held.stoppedAt || !isAliveVia(kill, held.pid)) return null;
|
|
139
|
+
if (!force && now - held.at < MAX_RUN_MS) return null;
|
|
140
|
+
if (!(await stopIndexer(held.pid, held.at, { ...io, command: held.command, now, kill }))) return null;
|
|
141
|
+
const marked = noteStopped(file, held.pid, now, force ? "lane-done" : "max-run", io.wait ?? sleepSync);
|
|
142
|
+
return { pid: held.pid, slug: held.slug, marked };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Overlapping claims of one process (two MCP requests in a host share its pid): the lease goes with the last.
|
|
146
|
+
const claims = new Map();
|
|
147
|
+
const RELEASE_TRIES = 5;
|
|
148
|
+
const RELEASE_WAIT_MS = 20;
|
|
149
|
+
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* HU-2907: claims the lease for `pid` unless another live process holds it. Never waits. A holder
|
|
153
|
+
* in `kin` (the spawning parent, which names itself before the child's pid lands) is not "another".
|
|
154
|
+
*/
|
|
155
|
+
export function claimLease(file, record, { now = Date.now(), pid = process.pid, kin = [] } = {}) {
|
|
156
|
+
const res = withMutex(file, now, () => {
|
|
157
|
+
const holder = readLease(file, { now });
|
|
158
|
+
if (holder?.running && holder.pid !== pid && !kin.includes(holder.pid)) return { claimed: false, holder };
|
|
159
|
+
writeLease(file, { ...record, pid, at: now });
|
|
160
|
+
claims.set(file, (claims.get(file) ?? 0) + 1);
|
|
161
|
+
return { claimed: true };
|
|
162
|
+
});
|
|
163
|
+
return res.busy === true ? { claimed: false, busy: true } : res;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Removes the lease only while it still names `pid`: never another process's claim. */
|
|
167
|
+
export function releaseLease(file, { pid = process.pid, now = Date.now(), sleep = pause } = {}) {
|
|
168
|
+
// This claim is over whatever the mutex says: counted down once, so a busy mutex never leaves it stale.
|
|
169
|
+
const left = (claims.get(file) ?? 1) - 1;
|
|
170
|
+
if (left > 0) { claims.set(file, left); return { released: false, shared: true }; }
|
|
171
|
+
claims.delete(file);
|
|
172
|
+
const release = () => withMutex(file, now, () => {
|
|
173
|
+
if (readLease(file, { now })?.pid !== pid) return { released: false };
|
|
174
|
+
unlessGone(() => unlinkSync(file));
|
|
175
|
+
return { released: true };
|
|
176
|
+
});
|
|
177
|
+
// Up to RELEASE_TRIES attempts while the mutex is busy; past that the lease expires on its own (dead pid, or MAX_RUN_MS).
|
|
178
|
+
let res = release();
|
|
179
|
+
for (let attempt = 1; res.busy === true && attempt < RELEASE_TRIES; attempt += 1) { sleep(RELEASE_WAIT_MS); res = release(); }
|
|
180
|
+
return res;
|
|
181
|
+
}
|
package/src/review/publish-pr.js
CHANGED
|
@@ -53,7 +53,10 @@ export async function publishPrReview({ projectDir, pr, run = runCommand, sealed
|
|
|
53
53
|
await labelKarajanPr({ url: String(pr), cwd: projectDir, run });
|
|
54
54
|
const reviews = await json(run, "gh", ["api", `repos/{owner}/{repo}/pulls/${pr}/reviews`], projectDir);
|
|
55
55
|
// KJC-BUG-0456: one a non-member posted (gh switched to another account) does not count in CI.
|
|
56
|
-
|
|
56
|
+
// KJC-BUG-0477 (#3127): nor does one that misses a commit the PR gained under the same head
|
|
57
|
+
// (a stacked PR retargeted to main); verify-pr adds up every kj review, so publish another.
|
|
58
|
+
const listsAll = (body) => [...rows, ...seals].every((c) => body.includes(`\`${c.oid.slice(0, 7)}\``));
|
|
59
|
+
if ((reviews ?? []).some((r) => TRUSTED.has(r.author_association) && String(r.body ?? "").includes(MARKER(info.headRefOid)) && listsAll(String(r.body)))) {
|
|
57
60
|
return { published: false, reason: `a kj review for head ${info.headRefOid.slice(0, 7)} is already published` };
|
|
58
61
|
}
|
|
59
62
|
const body = [
|
|
@@ -23,11 +23,16 @@ const SECURITY_PATTERN = /injection|xss|csrf|secret|credential|password|token le
|
|
|
23
23
|
// Structured signal first (`category: "security"` — part of the reviewer's
|
|
24
24
|
// JSON schema since AB-E); the free-text pattern stays as a fail-closed net
|
|
25
25
|
// for verdicts recorded before the schema carried categories.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
// KJC-BUG-0476 (#3137): returns why the issue counts as security, or null.
|
|
27
|
+
function securityReason(issue) {
|
|
28
|
+
if (issue.category === "security") return "category security";
|
|
29
|
+
const match = SECURITY_PATTERN.exec(`${issue.id || ""} ${issue.description || ""}`);
|
|
30
|
+
return match ? `matches "${match[0]}"` : null;
|
|
29
31
|
}
|
|
30
32
|
|
|
33
|
+
const describeSecurityIssue = (issue) =>
|
|
34
|
+
`${issue.id || "?"} (${securityReason(issue)}): ${String(issue.description || "").slice(0, 160)}`;
|
|
35
|
+
|
|
31
36
|
export async function pickThirdParty({ config, hostAgent, reviewerAgent, detectAgents = detectAvailableAgents }) {
|
|
32
37
|
const excluded = new Set([hostAgent, reviewerAgent]);
|
|
33
38
|
const configured = config?.roles?.solomon?.provider;
|
|
@@ -52,13 +57,14 @@ export async function runSolomonArbitration({
|
|
|
52
57
|
}
|
|
53
58
|
|
|
54
59
|
// ABSOLUTE RULE: security findings are never overridable.
|
|
55
|
-
const securityIssues = (verdict.issues || []).filter(
|
|
60
|
+
const securityIssues = (verdict.issues || []).filter((issue) => securityReason(issue));
|
|
56
61
|
if (securityIssues.length > 0) {
|
|
62
|
+
const named = securityIssues.map(describeSecurityIssue).join("; ");
|
|
57
63
|
const record = await saveVerdict(projectDir, diff, {
|
|
58
64
|
...verdict,
|
|
59
|
-
arbitration: { position, ruling: "reject", reasoning:
|
|
65
|
+
arbitration: { position, ruling: "reject", reasoning: `security issues from the reviewer are never overridable: ${named}`, solomon: null },
|
|
60
66
|
});
|
|
61
|
-
return { ruling: "reject", reasoning:
|
|
67
|
+
return { ruling: "reject", reasoning: `security issues are never overridable, fix them: ${named}`, solomon: null, record };
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
const solomon = await pickThirdParty({ config, hostAgent, reviewerAgent: verdict.reviewer, detectAgents });
|
package/src/skills/atril-cli.js
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
* contract between them stays narrow. The store is the project's `.karajan/`
|
|
5
5
|
* unless the user names another root.
|
|
6
6
|
*/
|
|
7
|
-
import { spawnSync } from "node:child_process";
|
|
7
|
+
import { execFile, spawnSync } from "node:child_process";
|
|
8
8
|
import { existsSync } from "node:fs";
|
|
9
9
|
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { promisify } from "node:util";
|
|
10
11
|
import { ATRIL_ROOT } from "./atril-skills.js";
|
|
11
12
|
|
|
12
13
|
export const ATRIL_COMMANDS = ["search", "route", "scan", "install", "remove", "graph", "enrich", "config"];
|
|
@@ -32,6 +33,23 @@ export function atrilBin(resolveFn = (spec) => import.meta.resolve(spec)) {
|
|
|
32
33
|
return fileURLToPath(new URL("../bin/atril.js", entry));
|
|
33
34
|
}
|
|
34
35
|
|
|
36
|
+
/** `atril search --json` through atril's own CLI (KJC-TSK-1193): `{ mode, hits }`, or an error carrying atril's stderr. */
|
|
37
|
+
export async function atrilSearch(query, { kind, root = ATRIL_ROOT } = {}, { exec = promisify(execFile), bin = atrilBin() } = {}) {
|
|
38
|
+
// `--` keeps a query that starts with a dash from being read as an atril option.
|
|
39
|
+
const args = [bin, "search", ...(kind ? ["--kind", kind] : []), "--root", root, "--json", "--", query];
|
|
40
|
+
let stdout;
|
|
41
|
+
try {
|
|
42
|
+
({ stdout } = await exec(process.execPath, args));
|
|
43
|
+
} catch (err) {
|
|
44
|
+
throw new Error(`atril search failed: ${String(err.stderr || err.message).trim()}`, { cause: err });
|
|
45
|
+
}
|
|
46
|
+
try {
|
|
47
|
+
return JSON.parse(stdout);
|
|
48
|
+
} catch (err) {
|
|
49
|
+
throw new Error(`atril search printed invalid JSON: ${err.message}`, { cause: err });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
35
53
|
/** Runs atril with this node; its output is the user's, its exit code is kj's. */
|
|
36
54
|
export function runAtril({ action, args = [], spawn = spawnSync, bin = atrilBin(), exists = existsSync }) {
|
|
37
55
|
if (spawn === spawnSync && !exists(bin)) throw new Error(`the atril CLI is not at ${bin}: ${INSTALL_HINT}`);
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HU-2782: `kj skills search` asks atril in English. kj prints the results itself
|
|
3
|
+
* (atril's line format), so the translation step can sit between query and search.
|
|
4
|
+
*/
|
|
5
|
+
import { parseArgs } from "node:util";
|
|
6
|
+
import { loadConfig } from "../config/loader.js";
|
|
7
|
+
import { createLogger } from "../utils/logger.js";
|
|
8
|
+
import { ATRIL_ROOT } from "./atril-skills.js";
|
|
9
|
+
import { atrilSearch } from "./atril-cli.js";
|
|
10
|
+
import { looksNonEnglish, panelAsk, translateDescriptions, translateQuery } from "./translate.js";
|
|
11
|
+
|
|
12
|
+
// Text from an artifact author or an agent must not drive the terminal.
|
|
13
|
+
const printable = (text) => [...String(text)]
|
|
14
|
+
.map((ch) => { const code = ch.codePointAt(0); return code < 32 || (code >= 127 && code < 160) ? " " : ch; })
|
|
15
|
+
.join("").replaceAll(/\s+/g, " ").trim();
|
|
16
|
+
|
|
17
|
+
// Errors only, to stderr: the agent's progress lines must never mix into the results on stdout.
|
|
18
|
+
const lazyAsk = async (prompt) => panelAsk({ config: (await loadConfig()).config, logger: createLogger("error") })(prompt);
|
|
19
|
+
|
|
20
|
+
const TOP_HITS = 10;
|
|
21
|
+
|
|
22
|
+
// The top hits' descriptions back into the asker's language, in one call; never the name, kind, source or score.
|
|
23
|
+
async function describeIn(hits, lang, { ask, err, translation }) {
|
|
24
|
+
const top = hits.slice(0, TOP_HITS);
|
|
25
|
+
if (top.length === 0) return hits;
|
|
26
|
+
try {
|
|
27
|
+
const texts = await translateDescriptions({ texts: top.map((h) => h.description), lang, ask });
|
|
28
|
+
return hits.map((h, i) => (i < TOP_HITS ? { ...h, description: texts[i], description_en: h.description } : h));
|
|
29
|
+
} catch (cause) {
|
|
30
|
+
translation.error = cause.message;
|
|
31
|
+
err(`kj: translation unavailable (${cause.message}); showing the original descriptions`);
|
|
32
|
+
return hits;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export async function skillsSearch({ args, deps = {} }) {
|
|
37
|
+
const { search = atrilSearch, ask = lazyAsk, out = (line) => process.stdout.write(`${line}\n`), err = (line) => process.stderr.write(`${line}\n`) } = deps;
|
|
38
|
+
const { values, positionals } = parseArgs({
|
|
39
|
+
args, allowPositionals: true, strict: false,
|
|
40
|
+
options: { kind: { type: "string" }, root: { type: "string" }, json: { type: "boolean" }, "no-translate": { type: "boolean" } },
|
|
41
|
+
});
|
|
42
|
+
const query = positionals.join(" ");
|
|
43
|
+
const opts = { kind: values.kind, root: values.root ?? ATRIL_ROOT };
|
|
44
|
+
try {
|
|
45
|
+
let found = await search(query, opts);
|
|
46
|
+
const translation = { english: query, lang: null, translated: false };
|
|
47
|
+
if (!values["no-translate"] && (found.hits.length === 0 || looksNonEnglish(query))) {
|
|
48
|
+
try {
|
|
49
|
+
const { english, lang } = await translateQuery({ query, ask });
|
|
50
|
+
if (english !== query) {
|
|
51
|
+
found = await search(english, opts);
|
|
52
|
+
err(`kj: searched in English: ${JSON.stringify(english)}`);
|
|
53
|
+
}
|
|
54
|
+
Object.assign(translation, { english, lang, translated: true });
|
|
55
|
+
if (lang && lang !== "en") found = { ...found, hits: await describeIn(found.hits, lang, { ask, err, translation }) };
|
|
56
|
+
} catch (cause) {
|
|
57
|
+
translation.error = cause.message;
|
|
58
|
+
err(`kj: translation unavailable (${cause.message}); showing results for the untranslated query`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
err(`kj: search mode ${found.mode}`);
|
|
62
|
+
if (values.json) {
|
|
63
|
+
const { english, lang, translated, error } = translation;
|
|
64
|
+
out(JSON.stringify({ mode: found.mode, query: { original: query, english, lang }, translated, hits: found.hits, ...(error && { translation_error: error }) }, null, 2));
|
|
65
|
+
} else if (found.hits.length === 0) out(`no results for ${JSON.stringify(translation.english)}`);
|
|
66
|
+
else for (const h of found.hits) out(`${h.score}\t${h.kind}\t${printable(h.name)}\t${h.source}\t${printable(h.description)}`);
|
|
67
|
+
return 0;
|
|
68
|
+
} catch (failure) {
|
|
69
|
+
err(`kj: ${failure.message}`);
|
|
70
|
+
return 1;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HU-2782 (ADR 0039, decision 2): atril indexes English, so a query in another
|
|
3
|
+
* language is translated first, by the `translator` role (it inherits the coder).
|
|
4
|
+
*/
|
|
5
|
+
import { createAgent } from "../agents/index.js";
|
|
6
|
+
import { assertAgentsAvailable } from "../agents/availability.js";
|
|
7
|
+
import { resolveRole } from "../config/role-resolver.js";
|
|
8
|
+
import { agentAnswerText } from "../review/parser.js";
|
|
9
|
+
|
|
10
|
+
// The same Spanish stopword list as packages/atril/src/lang.js (HU-2781); copied, not imported from atril's internals.
|
|
11
|
+
const SPANISH_STOPWORDS = new Set("de del la las el los para por con sin una unos unas que como cuando donde este esta estos estas pero sobre entre segun mas muy tambien hacer usar y o en es son al su sus lo se".split(" "));
|
|
12
|
+
const stripAccents = (text) => text.normalize("NFD").replaceAll(/\p{Diacritic}/gu, "");
|
|
13
|
+
|
|
14
|
+
/** Deterministic gate, no model needed: text outside ASCII or a Spanish stopword means not English. */
|
|
15
|
+
export const looksNonEnglish = (query) => /[^\u0000-\u007F]/.test(query)
|
|
16
|
+
|| stripAccents(query.toLowerCase()).split(/[^a-z]+/).some((token) => SPANISH_STOPWORDS.has(token));
|
|
17
|
+
|
|
18
|
+
const PROMPT = (query) => `Detect the language of this skill-search query and translate it to English.
|
|
19
|
+
Answer ONLY a JSON object {"lang":"<ISO 639-1 code>","english":"<the query in English>"}, no fences, no explanation.
|
|
20
|
+
|
|
21
|
+
Query: ${JSON.stringify(query)}`;
|
|
22
|
+
|
|
23
|
+
const DESCRIPTIONS_PROMPT = (texts, lang) => `Translate each of these skill descriptions from English into the language with ISO 639-1 code ${JSON.stringify(lang)}.
|
|
24
|
+
Answer ONLY a JSON array of ${texts.length} strings, in the same order, no fences, no explanation.
|
|
25
|
+
|
|
26
|
+
Descriptions: ${JSON.stringify(texts)}`;
|
|
27
|
+
|
|
28
|
+
async function askJson(ask, prompt) {
|
|
29
|
+
const text = String(await ask(prompt)).replace(/^```[a-z]*\n?|```\s*$/g, "").trim();
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(text);
|
|
32
|
+
} catch (err) {
|
|
33
|
+
throw new Error(`the translator answer is not valid JSON (${err.message})`, { cause: err });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** One batched agent call back into the asker's language. Throws unless the answer is one string per text. */
|
|
38
|
+
export async function translateDescriptions({ texts, lang, ask }) {
|
|
39
|
+
const parsed = await askJson(ask, DESCRIPTIONS_PROMPT(texts, lang));
|
|
40
|
+
if (!Array.isArray(parsed) || parsed.length !== texts.length || !parsed.every((t) => typeof t === "string")) {
|
|
41
|
+
throw new Error(`the translator answer is not a JSON array of ${texts.length} strings`);
|
|
42
|
+
}
|
|
43
|
+
return parsed;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** One agent call: `ask(prompt)` returns the agent's text. Throws when the answer is unusable. */
|
|
47
|
+
export async function translateQuery({ query, ask }) {
|
|
48
|
+
const parsed = await askJson(ask, PROMPT(query));
|
|
49
|
+
if (typeof parsed?.english !== "string" || !parsed.english.trim()) throw new Error("the translator answer has no english text");
|
|
50
|
+
return { lang: typeof parsed.lang === "string" ? parsed.lang : null, english: parsed.english.trim() };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The panel's translator, as `ask(prompt) => text`; never a silent fallback to another agent. */
|
|
54
|
+
export function panelAsk({ config, logger, create = createAgent, check = assertAgentsAvailable }) {
|
|
55
|
+
return async (prompt) => {
|
|
56
|
+
const { provider } = resolveRole(config, "translator");
|
|
57
|
+
await check([provider]);
|
|
58
|
+
const res = await create(provider, config, logger).runTask({ prompt, role: "translator" });
|
|
59
|
+
if (!res?.ok) throw new Error(`${provider}: ${res?.error || "no output"}`);
|
|
60
|
+
return agentAnswerText(res.output);
|
|
61
|
+
};
|
|
62
|
+
}
|