karajan-code 2.32.0 → 2.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/package.json +1 -1
- package/scripts/esbuild-sea.config.mjs +10 -1
- package/src/audit/audit-history-display.js +61 -0
- package/src/audit/audit-history.js +102 -0
- package/src/audit/harness-scorecard.js +51 -0
- package/src/audit/harness-section.js +77 -0
- package/src/checks/harness-scorecard.js +41 -0
- package/src/cli/register-meta.js +4 -0
- package/src/commands/audit.js +73 -16
- package/src/commands/doctor.js +2 -0
package/README.md
CHANGED
|
@@ -345,6 +345,13 @@ Opt out: set `telemetry: false` in `~/.karajan/kj.config.yml`
|
|
|
345
345
|
|
|
346
346
|
## Recent releases
|
|
347
347
|
|
|
348
|
+
> **v2.33.0 released** — Minor. **AI Harness Scorecard golden metric (KJC-PCS-0051, Plan B)** — every `kj audit` now boots a Docker one-shot of `addyosmani/ai-harness-scorecard`, gets a deterministic 0–100 score + A–F grade, persists it to a per-project `audit-history.db`, and on the next run renders the delta vs the previous baseline plus an optional Unicode-bar trend sparkline. One golden number for "how AI-friendly is this repo today vs last week," zero LLM tokens spent.
|
|
349
|
+
>
|
|
350
|
+
> - **Harness Docker one-shot** (PR #877, KJC-TSK-0470): `src/audit/harness-runner.js` runs `docker run --rm -v <repo>:/repo addyosmani/ai-harness-scorecard analyze`, parses the verdict, auto-skips on missing Docker. `--no-harness` opts out.
|
|
351
|
+
> - **`kj audit` integrates the score** (PR #878, KJC-TSK-0471): new harness-section renderer; JSON output gains `harnessScore`; runs during the deterministic phase so `--deterministic-only` users get it too.
|
|
352
|
+
> - **Per-project audit history** (PR #879, KJC-TSK-0472): `.karajan/audit-history.db` (better-sqlite3 + WAL + `PRAGMA user_version=1`) persists every run. SEA bundle stubbed (degrades gracefully); npm install unlocks history.
|
|
353
|
+
> - **Diff + trend sparkline** (PR #880, KJC-TSK-0473): `src/audit/audit-history-display.js` (pure module, safe in SEA) computes overall delta + per-category deltas + biggest improvement/regression + stale-baseline flag (>30 days). New `--trend` flag prints a `▁▂▃▄▅▆▇█` sparkline over the last N runs.
|
|
354
|
+
>
|
|
348
355
|
> **v2.32.0 released** — Minor. **AI Harness Scorecard hardening (KJC-PCS-0051)** — Plan A closes five FAILs from the external scorecard audit in one sprint: Prettier `--check`, Coverage v8 reports, Conventional Commits enforcement, nightly drift detection, and an unsafe-code lint policy. Plus two bug fixes shipped alongside.
|
|
349
356
|
>
|
|
350
357
|
> - **Prettier `--check` CI job** (PR #868, KJC-TSK-0464): new `format` job blocks PRs whose formatting drifts from `.prettierrc.json`. Scope narrow at first (`.github/workflows/`, root config); future PRs fold in more dirs under the shrink-budget cap.
|
package/package.json
CHANGED
|
@@ -191,6 +191,15 @@ const ragStubPlugin = {
|
|
|
191
191
|
},
|
|
192
192
|
};
|
|
193
193
|
|
|
194
|
+
// KJC-TSK-0472 — audit-history sqlite stub (same reasoning as rag/hu-board stubs).
|
|
195
|
+
const auditHistoryStubPlugin = {
|
|
196
|
+
name: "audit-history-stub",
|
|
197
|
+
setup(b) {
|
|
198
|
+
b.onResolve({ filter: /[\\/]audit[\\/]audit-history\.js$/ }, (a) => ({ path: a.path, namespace: "ahs" }));
|
|
199
|
+
b.onLoad({ filter: /.*/, namespace: "ahs" }, () => ({ contents: "module.exports = { __esModule: false, persistAuditRun: () => ({ ok: false, error: 'history disabled in SEA' }), getAuditHistoryDbPath: () => null, openAuditHistoryDb: () => null, recordAuditRun: () => null, listRecentRuns: () => [], countRuns: () => 0, pruneOldRuns: () => 0, getLatestPreviousRun: () => null, getRecentScores: () => [] };", loader: "js" }));
|
|
200
|
+
},
|
|
201
|
+
};
|
|
202
|
+
|
|
194
203
|
/** @type {import('esbuild').BuildOptions} */
|
|
195
204
|
export const seaBuildOptions = {
|
|
196
205
|
entryPoints: ["src/cli.js"],
|
|
@@ -221,6 +230,6 @@ export const seaBuildOptions = {
|
|
|
221
230
|
// throws → collector returns available:false. npm installs get knip
|
|
222
231
|
// resolved normally from node_modules.
|
|
223
232
|
external: ["better-sqlite3", "madge", "knip", "oxc-parser", "oxc-resolver"],
|
|
224
|
-
plugins: [seaTransformPlugin, huBoardStubPlugin, ragStubPlugin],
|
|
233
|
+
plugins: [seaTransformPlugin, huBoardStubPlugin, ragStubPlugin, auditHistoryStubPlugin],
|
|
225
234
|
logLevel: "info",
|
|
226
235
|
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// KJC-TSK-0473 — Pure display helpers for audit history (diff + sparkline).
|
|
2
|
+
// No DB / native-module deps; safe to bundle into the SEA binary.
|
|
3
|
+
|
|
4
|
+
const SPARK_BARS = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"];
|
|
5
|
+
|
|
6
|
+
function scoreOf(c) {
|
|
7
|
+
if (!c || typeof c !== "object") return null;
|
|
8
|
+
return typeof c.score === "number" ? c.score : typeof c.percent === "number" ? c.percent : null;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function computeHistoryDiff(currentHarness, previousRow) {
|
|
12
|
+
if (!currentHarness || typeof currentHarness.score !== "number") return null;
|
|
13
|
+
if (!previousRow || typeof previousRow.score !== "number") return { previousScore: null, firstRun: true };
|
|
14
|
+
let prevCats = {};
|
|
15
|
+
try { prevCats = previousRow.categories_json ? JSON.parse(previousRow.categories_json) || {} : {}; } catch { /* keep empty */ }
|
|
16
|
+
const currCats = currentHarness.categories || {};
|
|
17
|
+
const categoryDeltas = [];
|
|
18
|
+
for (const name of Object.keys(currCats)) {
|
|
19
|
+
const cur = scoreOf(currCats[name]);
|
|
20
|
+
const prev = scoreOf(prevCats[name]);
|
|
21
|
+
if (typeof cur === "number" && typeof prev === "number") {
|
|
22
|
+
categoryDeltas.push({ name, current: cur, previous: prev, delta: cur - prev });
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const sorted = [...categoryDeltas].sort((a, b) => b.delta - a.delta);
|
|
26
|
+
const t = previousRow.timestamp ? Date.parse(previousRow.timestamp) : NaN;
|
|
27
|
+
const ageDays = Number.isNaN(t) ? null : Math.floor((Date.now() - t) / 86400000);
|
|
28
|
+
return {
|
|
29
|
+
previousScore: previousRow.score,
|
|
30
|
+
previousGrade: previousRow.grade || null,
|
|
31
|
+
previousTimestamp: previousRow.timestamp,
|
|
32
|
+
delta: currentHarness.score - previousRow.score,
|
|
33
|
+
categoryDeltas,
|
|
34
|
+
biggestImprovement: sorted.find((c) => c.delta > 0) || null,
|
|
35
|
+
biggestRegression: [...sorted].reverse().find((c) => c.delta < 0) || null,
|
|
36
|
+
baselineAgeDays: ageDays,
|
|
37
|
+
baselineStale: ageDays != null && ageDays > 30,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function formatHistoryDiff(diff) {
|
|
42
|
+
if (!diff) return "";
|
|
43
|
+
if (diff.firstRun) return "_First run for this project — no baseline yet._";
|
|
44
|
+
const arrow = diff.delta > 0 ? "↑" : diff.delta < 0 ? "↓" : "=";
|
|
45
|
+
const sign = diff.delta > 0 ? "+" : "";
|
|
46
|
+
const prevDate = (diff.previousTimestamp || "").slice(0, 10);
|
|
47
|
+
const lines = [`**Δ ${sign}${diff.delta} ${arrow} vs run anterior** (${prevDate}, ${diff.previousScore}/100 ${diff.previousGrade || ""})`];
|
|
48
|
+
if (diff.baselineStale) lines.push(`> ⚠ baseline antigua (${diff.baselineAgeDays} días) — considera ejecutar \`kj audit\` con más frecuencia.`);
|
|
49
|
+
if (diff.biggestImprovement) lines.push(`- Mayor mejora: **${diff.biggestImprovement.name}** (+${diff.biggestImprovement.delta})`);
|
|
50
|
+
if (diff.biggestRegression) lines.push(`- Mayor regresión: **${diff.biggestRegression.name}** (${diff.biggestRegression.delta})`);
|
|
51
|
+
return lines.join("\n");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function formatTrendSparkline(scores) {
|
|
55
|
+
const xs = (scores || []).map((r) => (typeof r === "number" ? r : r?.score)).filter((n) => typeof n === "number");
|
|
56
|
+
if (xs.length === 0) return "";
|
|
57
|
+
if (xs.length === 1) return `Score trend: ${xs[0]} (single run)`;
|
|
58
|
+
const min = Math.min(...xs), max = Math.max(...xs), range = max - min || 1;
|
|
59
|
+
const bars = xs.map((n) => SPARK_BARS[Math.min(SPARK_BARS.length - 1, Math.floor(((n - min) / range) * (SPARK_BARS.length - 1)))]).join("");
|
|
60
|
+
return `Score trend (last ${xs.length} runs): ${xs[0]} ${bars} ${xs[xs.length - 1]}`;
|
|
61
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// KJC-TSK-0472 — Per-project audit history (sqlite at .karajan/audit-history.db).
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import Database from "better-sqlite3";
|
|
5
|
+
|
|
6
|
+
const SCHEMA_VERSION = 1;
|
|
7
|
+
|
|
8
|
+
export function getAuditHistoryDbPath(projectDir) {
|
|
9
|
+
return path.join(projectDir, ".karajan", "audit-history.db");
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function openAuditHistoryDb(projectDir) {
|
|
13
|
+
const dbPath = getAuditHistoryDbPath(projectDir);
|
|
14
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
15
|
+
const db = new Database(dbPath);
|
|
16
|
+
db.pragma("journal_mode = WAL");
|
|
17
|
+
migrate(db);
|
|
18
|
+
return db;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function migrate(db) {
|
|
22
|
+
const current = db.pragma("user_version", { simple: true });
|
|
23
|
+
if (current >= SCHEMA_VERSION) return;
|
|
24
|
+
db.exec(`
|
|
25
|
+
CREATE TABLE IF NOT EXISTS audits (
|
|
26
|
+
run_id TEXT PRIMARY KEY,
|
|
27
|
+
timestamp TEXT NOT NULL,
|
|
28
|
+
git_sha TEXT,
|
|
29
|
+
score INTEGER,
|
|
30
|
+
grade TEXT,
|
|
31
|
+
categories_json TEXT,
|
|
32
|
+
raw_report_path TEXT
|
|
33
|
+
);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS findings (
|
|
35
|
+
run_id TEXT NOT NULL,
|
|
36
|
+
category TEXT,
|
|
37
|
+
check_id TEXT,
|
|
38
|
+
status TEXT,
|
|
39
|
+
weight REAL,
|
|
40
|
+
FOREIGN KEY(run_id) REFERENCES audits(run_id) ON DELETE CASCADE
|
|
41
|
+
);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS idx_audits_timestamp ON audits(timestamp DESC);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS idx_findings_run_id ON findings(run_id);
|
|
44
|
+
`);
|
|
45
|
+
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// `harness` may be null (--no-harness / docker offline): row still inserted with NULL score.
|
|
49
|
+
export function recordAuditRun(db, { runId, timestamp, gitSha, harness, rawReportPath }) {
|
|
50
|
+
const categories = harness?.categories ? JSON.stringify(harness.categories) : null;
|
|
51
|
+
const checks = Array.isArray(harness?.checks) ? harness.checks : [];
|
|
52
|
+
const insertRun = db.prepare(`INSERT INTO audits (run_id, timestamp, git_sha, score, grade, categories_json, raw_report_path) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
53
|
+
const insertFinding = db.prepare(`INSERT INTO findings (run_id, category, check_id, status, weight) VALUES (?, ?, ?, ?, ?)`);
|
|
54
|
+
db.transaction(() => {
|
|
55
|
+
insertRun.run(runId, timestamp, gitSha || null, harness?.score ?? null, harness?.grade ?? null, categories, rawReportPath || null);
|
|
56
|
+
for (const c of checks) {
|
|
57
|
+
insertFinding.run(runId, c.category || null, c.name || c.id || null, c.passed ? "pass" : "fail", typeof c.weight === "number" ? c.weight : null);
|
|
58
|
+
}
|
|
59
|
+
})();
|
|
60
|
+
return { runId, inserted: 1 + checks.length };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function listRecentRuns(db, limit = 10) {
|
|
64
|
+
return db.prepare(`SELECT run_id, timestamp, git_sha, score, grade FROM audits ORDER BY timestamp DESC LIMIT ?`).all(limit);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// KJC-TSK-0473 — Fetch previous run (excluding current) for diff rendering.
|
|
68
|
+
export function getLatestPreviousRun(db, currentTimestamp) {
|
|
69
|
+
return db.prepare(`SELECT run_id, timestamp, git_sha, score, grade, categories_json FROM audits WHERE timestamp < ? AND score IS NOT NULL ORDER BY timestamp DESC LIMIT 1`).get(currentTimestamp || new Date().toISOString()) || null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// KJC-TSK-0473 — Score timeline (ASC) for sparkline rendering.
|
|
73
|
+
export function getRecentScores(db, limit = 10) {
|
|
74
|
+
const rows = db.prepare(`SELECT timestamp, score FROM audits WHERE score IS NOT NULL ORDER BY timestamp DESC LIMIT ?`).all(limit);
|
|
75
|
+
return rows.reverse();
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function countRuns(db) {
|
|
79
|
+
return db.prepare(`SELECT COUNT(*) AS n FROM audits`).get().n;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function pruneOldRuns(db, keep = 50) {
|
|
83
|
+
const ids = db.prepare(`SELECT run_id FROM audits ORDER BY timestamp DESC LIMIT -1 OFFSET ?`).all(keep).map((r) => r.run_id);
|
|
84
|
+
if (ids.length === 0) return 0;
|
|
85
|
+
const ph = ids.map(() => "?").join(",");
|
|
86
|
+
db.transaction(() => { db.prepare(`DELETE FROM findings WHERE run_id IN (${ph})`).run(...ids); db.prepare(`DELETE FROM audits WHERE run_id IN (${ph})`).run(...ids); })();
|
|
87
|
+
return ids.length;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// High-level helper used by `kj audit`. Errors returned via `ok` so a broken db never breaks the audit.
|
|
91
|
+
export function persistAuditRun(projectDir, payload) {
|
|
92
|
+
try {
|
|
93
|
+
const db = openAuditHistoryDb(projectDir);
|
|
94
|
+
recordAuditRun(db, payload);
|
|
95
|
+
const n = countRuns(db);
|
|
96
|
+
db.close();
|
|
97
|
+
return { ok: true, total: n, pruneHint: n > 100 };
|
|
98
|
+
} catch (e) {
|
|
99
|
+
return { ok: false, error: e?.message || String(e) };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// KJC-TSK-0470 — Bootstrap Docker de ai-harness-scorecard.
|
|
2
|
+
// One-shot container: detect Docker, auto-pull on first run (cache hit
|
|
3
|
+
// thereafter), execute `assess <repo>` with the project mounted read-only.
|
|
4
|
+
// No compose, no port discovery — assess is a single process that exits.
|
|
5
|
+
import { runCommand } from "../utils/process.js";
|
|
6
|
+
|
|
7
|
+
const D = { image: "markmishaev76/ai-harness-scorecard:latest", pullTimeoutMs: 300000, assessTimeoutMs: 180000 };
|
|
8
|
+
const pos = (v, d) => (Number(v) > 0 ? Number(v) : d);
|
|
9
|
+
|
|
10
|
+
export function normalizeHarnessConfig(h = {}) {
|
|
11
|
+
return {
|
|
12
|
+
enabled: h.enabled !== false,
|
|
13
|
+
image: h.image || D.image,
|
|
14
|
+
pullTimeoutMs: pos(h.pull_timeout_ms, D.pullTimeoutMs),
|
|
15
|
+
assessTimeoutMs: pos(h.assess_timeout_ms, D.assessTimeoutMs),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function isDockerAvailable() {
|
|
20
|
+
try {
|
|
21
|
+
const r = await runCommand("docker", ["version", "--format", "{{.Server.Version}}"], { timeout: 10000 });
|
|
22
|
+
return r.exitCode === 0;
|
|
23
|
+
} catch { return false; }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function isHarnessImagePresent(image) {
|
|
27
|
+
const r = await runCommand("docker", ["images", "-q", image], { timeout: 10000 });
|
|
28
|
+
return r.exitCode === 0 && r.stdout.trim().length > 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function ensureHarnessImage(harness = null) {
|
|
32
|
+
const cfg = normalizeHarnessConfig(harness || {});
|
|
33
|
+
if (await isHarnessImagePresent(cfg.image)) return { pulled: false, reused: true, image: cfg.image };
|
|
34
|
+
const r = await runCommand("docker", ["pull", cfg.image], { timeout: cfg.pullTimeoutMs });
|
|
35
|
+
if (r.exitCode !== 0) return { pulled: false, reused: false, image: cfg.image, error: r.stderr?.trim() || `exit ${r.exitCode}` };
|
|
36
|
+
return { pulled: true, reused: false, image: cfg.image };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function runHarnessAssess(repoPath, harness = null) {
|
|
40
|
+
const cfg = normalizeHarnessConfig(harness || {});
|
|
41
|
+
const r = await runCommand(
|
|
42
|
+
"docker",
|
|
43
|
+
["run", "--rm", "-v", `${repoPath}:/repo:ro`, cfg.image, "assess", "/repo", "--format", "json"],
|
|
44
|
+
{ timeout: cfg.assessTimeoutMs },
|
|
45
|
+
);
|
|
46
|
+
if (r.exitCode !== 0) return { ok: false, error: r.stderr?.trim() || r.stdout?.trim() || `exit ${r.exitCode}` };
|
|
47
|
+
try {
|
|
48
|
+
const parsed = JSON.parse(r.stdout);
|
|
49
|
+
return { ok: true, score: parsed.overall_score, grade: parsed.grade, raw: parsed };
|
|
50
|
+
} catch (e) { return { ok: false, error: `Invalid JSON from harness: ${e.message}` }; }
|
|
51
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// KJC-TSK-0471 — Harness Scorecard audit stage.
|
|
2
|
+
// Runs the markmishaev76/ai-harness-scorecard container after the deterministic
|
|
3
|
+
// phase, persists scorecard.json + scorecard-report.md under
|
|
4
|
+
// .karajan/audit-runs/<ts>/, and returns a summary the renderer surfaces as a
|
|
5
|
+
// "Harness Health Score" markdown block. KJC-TSK-0472 will consume the
|
|
6
|
+
// persisted artefacts to build audit-history.db.
|
|
7
|
+
import fs from "node:fs/promises";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { normalizeHarnessConfig, runHarnessAssess } from "./harness-scorecard.js";
|
|
10
|
+
|
|
11
|
+
const CATEGORIES = [
|
|
12
|
+
["architectural_docs", "Architectural Docs", 20],
|
|
13
|
+
["mechanical_constraints", "Mechanical Constraints", 25],
|
|
14
|
+
["testing_stability", "Testing & Stability", 25],
|
|
15
|
+
["review_drift_prevention", "Review & Drift Prevention", 15],
|
|
16
|
+
["ai_specific_safeguards", "AI-Specific Safeguards", 15],
|
|
17
|
+
];
|
|
18
|
+
const labelFor = (k) => CATEGORIES.find(([key]) => key === k)?.[1] || k;
|
|
19
|
+
const ts = () => new Date().toISOString().replaceAll(/[:.]/g, "-");
|
|
20
|
+
|
|
21
|
+
export async function runHarnessSection({ projectDir, harnessConfig = null, runAssess = runHarnessAssess } = {}) {
|
|
22
|
+
const cfg = normalizeHarnessConfig(harnessConfig || {});
|
|
23
|
+
if (!cfg.enabled) return { ok: true, skipped: true, reason: "disabled" };
|
|
24
|
+
const res = await runAssess(projectDir, cfg);
|
|
25
|
+
if (!res.ok) return { ok: false, error: res.error };
|
|
26
|
+
const outDir = path.join(projectDir, ".karajan", "audit-runs", ts());
|
|
27
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
28
|
+
const scorecardPath = path.join(outDir, "scorecard.json");
|
|
29
|
+
const rawReportPath = path.join(outDir, "scorecard-report.md");
|
|
30
|
+
await fs.writeFile(scorecardPath, JSON.stringify(res.raw, null, 2), "utf8");
|
|
31
|
+
await fs.writeFile(rawReportPath, res.raw?.report_markdown || res.raw?.markdown || "", "utf8");
|
|
32
|
+
return {
|
|
33
|
+
ok: true, score: res.score, grade: res.grade,
|
|
34
|
+
categories: res.raw?.categories || {},
|
|
35
|
+
checks: Array.isArray(res.raw?.checks) ? res.raw.checks : [],
|
|
36
|
+
scorecardPath, rawReportPath, image: cfg.image,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function formatHarnessSection(summary) {
|
|
41
|
+
if (!summary || summary.skipped) return "";
|
|
42
|
+
if (!summary.ok) return `## Harness Scorecard\n\n_skipped: ${summary.error}_\n`;
|
|
43
|
+
const lines = [
|
|
44
|
+
`## Harness Health Score: ${summary.score}/100 (${summary.grade})`,
|
|
45
|
+
"",
|
|
46
|
+
"| Category | Weight | Score |",
|
|
47
|
+
"| --- | --- | --- |",
|
|
48
|
+
];
|
|
49
|
+
for (const [key, label, weight] of CATEGORIES) {
|
|
50
|
+
const score = summary.categories?.[key]?.score ?? "—";
|
|
51
|
+
lines.push(`| ${label} | ${weight}% | ${score} |`);
|
|
52
|
+
}
|
|
53
|
+
const checks = Array.isArray(summary.checks) ? summary.checks : [];
|
|
54
|
+
if (checks.length) {
|
|
55
|
+
const groups = new Map();
|
|
56
|
+
for (const c of checks) {
|
|
57
|
+
const k = c.category || "other";
|
|
58
|
+
if (!groups.has(k)) groups.set(k, []);
|
|
59
|
+
groups.get(k).push(c);
|
|
60
|
+
}
|
|
61
|
+
lines.push("", "### Checks");
|
|
62
|
+
for (const [k, list] of groups) {
|
|
63
|
+
lines.push(`- **${labelFor(k)}**`);
|
|
64
|
+
for (const c of list) lines.push(` - ${c.passed ? "✓" : "✗"} ${c.name}${c.detail ? ` — ${c.detail}` : ""}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return lines.join("\n") + "\n";
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function harnessSummaryForJson(summary) {
|
|
71
|
+
if (!summary?.ok || summary.skipped) return null;
|
|
72
|
+
return {
|
|
73
|
+
score: summary.score, grade: summary.grade,
|
|
74
|
+
categories: summary.categories, checks: summary.checks,
|
|
75
|
+
rawReportPath: summary.rawReportPath, scorecardPath: summary.scorecardPath,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// KJC-TSK-0470 — `kj doctor` check for the AI Harness Scorecard image.
|
|
2
|
+
// Reports Disabled / Docker missing / Image not pulled / OK. The image
|
|
3
|
+
// is pulled lazily by `kj audit` on first use; this check is informational.
|
|
4
|
+
import { isDockerAvailable, isHarnessImagePresent, normalizeHarnessConfig } from "../audit/harness-scorecard.js";
|
|
5
|
+
import { STRATEGY } from "./types.js";
|
|
6
|
+
|
|
7
|
+
function harnessEnabled(config) { return config?.audit?.harness?.enabled !== false; }
|
|
8
|
+
|
|
9
|
+
function createHarnessScorecardCheck() {
|
|
10
|
+
return {
|
|
11
|
+
name: "harness-scorecard",
|
|
12
|
+
label: "AI Harness Scorecard (audit)",
|
|
13
|
+
strategy: STRATEGY.NONE,
|
|
14
|
+
async detect({ config }) {
|
|
15
|
+
const cfg = normalizeHarnessConfig(config?.audit?.harness || {});
|
|
16
|
+
if (!harnessEnabled(config)) {
|
|
17
|
+
return { ok: true, severity: "info", detail: "Disabled in config (audit.harness.enabled=false)" };
|
|
18
|
+
}
|
|
19
|
+
let docker; try { docker = await isDockerAvailable(); } catch { docker = false; }
|
|
20
|
+
if (!docker) return {
|
|
21
|
+
ok: false, severity: "warn",
|
|
22
|
+
detail: "Docker not available — kj audit will skip the harness step.",
|
|
23
|
+
fix: "Install Docker (or run `kj audit --no-harness` to silence this).",
|
|
24
|
+
extra: { docker: false, image: cfg.image },
|
|
25
|
+
};
|
|
26
|
+
let cached; try { cached = await isHarnessImagePresent(cfg.image); } catch { cached = false; }
|
|
27
|
+
if (!cached) return {
|
|
28
|
+
ok: true, severity: "info",
|
|
29
|
+
detail: `Will be pulled on first \`kj audit\` run (${cfg.image}).`,
|
|
30
|
+
extra: { docker: true, cached: false, image: cfg.image },
|
|
31
|
+
};
|
|
32
|
+
return {
|
|
33
|
+
ok: true, severity: "info",
|
|
34
|
+
detail: `Image cached locally (${cfg.image}).`,
|
|
35
|
+
extra: { docker: true, cached: true, image: cfg.image },
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function getHarnessScorecardChecks() { return [createHarnessScorecardCheck()]; }
|
package/src/cli/register-meta.js
CHANGED
|
@@ -161,6 +161,8 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
161
161
|
.option("--no-sonar", "Skip the SonarQube findings collector (faster, less context). Sonar findings are also skipped automatically when SonarQube is unreachable.")
|
|
162
162
|
.option("--no-osv", "Skip the OSV-Scanner vulnerability collector. Findings are also skipped automatically when osv-scanner is not installed.")
|
|
163
163
|
.option("--no-semgrep", "Skip the Semgrep SAST collector. Findings are also skipped automatically when semgrep is not installed (install via 'pipx install semgrep' or 'brew install semgrep').")
|
|
164
|
+
.option("--no-harness", "Skip the AI Harness Scorecard stage (Docker one-shot). Auto-skipped when Docker is unavailable. Each run is also persisted to .karajan/audit-history.db (KJC-TSK-0472) for diff/trend tracking.")
|
|
165
|
+
.option("--trend", "Append a sparkline of the harness score across the last 10 runs (uses .karajan/audit-history.db). KJC-TSK-0473.")
|
|
164
166
|
.option("--report-file <path>", "Write the audit report to disk in addition to stdout. <path> may be a file (extension drives format: .md or .json) or a directory (creates audit-<ISO>.<md|json> inside). $KJ_AUDIT_REPORT_DIR env var is used as default directory if no --report-file is given.")
|
|
165
167
|
.option("--deterministic-only", "Skip the LLM analysis entirely. Print/persist only the deterministic findings (basalCost, sonar, stack, growth-delta, webperf). Zero tokens spent. Compatible with --report-file and --json.")
|
|
166
168
|
.option("-y, --yes", "Auto-confirm the 'Continue with LLM analysis?' prompt. Useful in scripts that want the full audit non-interactively. CI/non-TTY paths already auto-confirm without this flag.")
|
|
@@ -182,9 +184,11 @@ export function registerMeta(program, { pkgVersion }) {
|
|
|
182
184
|
noSonar: flags.sonar === false,
|
|
183
185
|
noOsv: flags.osv === false,
|
|
184
186
|
noSemgrep: flags.semgrep === false,
|
|
187
|
+
noHarness: flags.harness === false,
|
|
185
188
|
reportFile: flags.reportFile || null,
|
|
186
189
|
deterministicOnly: Boolean(flags.deterministicOnly),
|
|
187
190
|
yes: Boolean(flags.yes),
|
|
191
|
+
trend: Boolean(flags.trend),
|
|
188
192
|
});
|
|
189
193
|
});
|
|
190
194
|
});
|
package/src/commands/audit.js
CHANGED
|
@@ -11,6 +11,9 @@ import { withCliRunLog } from "../utils/cli-run-log.js";
|
|
|
11
11
|
import { createCliProgressReporter } from "../utils/cli-progress.js";
|
|
12
12
|
import { runAgentReadiness, formatAgentReadinessReport } from "../audit/agent-readiness.js";
|
|
13
13
|
import { formatDeterministicSummary } from "../audit/deterministic-summary.js";
|
|
14
|
+
import { runHarnessSection, formatHarnessSection, harnessSummaryForJson } from "../audit/harness-section.js";
|
|
15
|
+
import { persistAuditRun, openAuditHistoryDb, getLatestPreviousRun, getRecentScores } from "../audit/audit-history.js";
|
|
16
|
+
import { computeHistoryDiff, formatHistoryDiff, formatTrendSparkline } from "../audit/audit-history-display.js";
|
|
14
17
|
|
|
15
18
|
const execFileAsync = promisify(execFile);
|
|
16
19
|
|
|
@@ -191,20 +194,43 @@ async function buildReportHeader(projectDir, invocation) {
|
|
|
191
194
|
return lines.join("\n");
|
|
192
195
|
}
|
|
193
196
|
|
|
194
|
-
function
|
|
197
|
+
function composeHistoryBlock({ diffMd, trendMd }) {
|
|
198
|
+
const parts = [];
|
|
199
|
+
if (diffMd) parts.push(diffMd);
|
|
200
|
+
if (trendMd) parts.push(trendMd);
|
|
201
|
+
return parts.length ? `\n\n## Harness Score History\n${parts.join("\n\n")}` : "";
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// KJC-TSK-0473 — Resolve diff vs previous run + optional trend sparkline.
|
|
205
|
+
// Errors silenced: history is a nice-to-have; a broken db must not break audit.
|
|
206
|
+
function loadAuditHistoryContext(projectDir, currentHarness, { showTrend = false } = {}) {
|
|
207
|
+
let db;
|
|
208
|
+
try {
|
|
209
|
+
db = openAuditHistoryDb(projectDir);
|
|
210
|
+
if (!db) return { diff: null, diffMd: "", trendMd: "" };
|
|
211
|
+
const previous = currentHarness?.score != null ? getLatestPreviousRun(db, new Date().toISOString()) : null;
|
|
212
|
+
const diff = currentHarness?.score != null ? computeHistoryDiff(currentHarness, previous) : null;
|
|
213
|
+
const trendMd = showTrend ? formatTrendSparkline(getRecentScores(db, 10)) : "";
|
|
214
|
+
return { diff, diffMd: formatHistoryDiff(diff), trendMd };
|
|
215
|
+
} catch { return { diff: null, diffMd: "", trendMd: "" }; }
|
|
216
|
+
finally { try { db?.close(); } catch { /* noop */ } }
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function describeInvocation({ task, dimensions, noSonar, noOsv, noSemgrep, noHarness, json, deterministicOnly, yes }) {
|
|
195
220
|
const parts = ["kj audit"];
|
|
196
221
|
if (task && task !== "Analyze the full codebase") parts.push(JSON.stringify(task));
|
|
197
222
|
if (dimensions && dimensions !== "all") parts.push(`--dimensions=${dimensions}`);
|
|
198
223
|
if (noSonar) parts.push("--no-sonar");
|
|
199
224
|
if (noOsv) parts.push("--no-osv");
|
|
200
225
|
if (noSemgrep) parts.push("--no-semgrep");
|
|
226
|
+
if (noHarness) parts.push("--no-harness");
|
|
201
227
|
if (deterministicOnly) parts.push("--deterministic-only");
|
|
202
228
|
if (yes) parts.push("--yes");
|
|
203
229
|
if (json) parts.push("--json");
|
|
204
230
|
return parts.join(" ");
|
|
205
231
|
}
|
|
206
232
|
|
|
207
|
-
export async function auditCommand({ task, config, logger, dimensions, json, agentReadiness, path: pathArg, noSonar = false, noOsv = false, noSemgrep = false, reportFile = null, deterministicOnly = false, yes = false, promptFn = null }) {
|
|
233
|
+
export async function auditCommand({ task, config, logger, dimensions, json, agentReadiness, path: pathArg, noSonar = false, noOsv = false, noSemgrep = false, noHarness = false, reportFile = null, deterministicOnly = false, yes = false, trend = false, promptFn = null }) {
|
|
208
234
|
// --agent-readiness is a STANDALONE, deterministic, LLM-free audit
|
|
209
235
|
// dimension. It scores any third-party repo for AI-agent readability
|
|
210
236
|
// (llms.txt presence, page token budgets, robots allowlist, etc.).
|
|
@@ -247,6 +273,27 @@ export async function auditCommand({ task, config, logger, dimensions, json, age
|
|
|
247
273
|
const deterministicCtx = await role.collectDeterministic(roleInput);
|
|
248
274
|
const deterministicMd = formatDeterministicSummary(deterministicCtx);
|
|
249
275
|
|
|
276
|
+
// KJC-TSK-0471 — Harness Scorecard (Docker one-shot, golden metric).
|
|
277
|
+
// Runs after the deterministic phase, before the LLM. Auto-skipped on
|
|
278
|
+
// --no-harness or if `runHarnessSection` surfaces an error (graceful
|
|
279
|
+
// degradation: a missing Docker daemon must not break `kj audit`).
|
|
280
|
+
let harnessSummary = null;
|
|
281
|
+
if (!noHarness) {
|
|
282
|
+
try {
|
|
283
|
+
harnessSummary = await runHarnessSection({
|
|
284
|
+
projectDir: config?.projectDir || process.cwd(),
|
|
285
|
+
harnessConfig: config?.audit?.harness || null,
|
|
286
|
+
});
|
|
287
|
+
if (harnessSummary?.ok && !harnessSummary.skipped) {
|
|
288
|
+
runLog.logText(`[audit] harness score=${harnessSummary.score}/100 grade=${harnessSummary.grade}`);
|
|
289
|
+
}
|
|
290
|
+
} catch (e) {
|
|
291
|
+
harnessSummary = { ok: false, error: e?.message || String(e) };
|
|
292
|
+
runLog.logText(`[audit] harness skipped: ${harnessSummary.error}`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
const harnessMd = formatHarnessSection(harnessSummary);
|
|
296
|
+
|
|
250
297
|
// --json suppresses the interactive flow regardless: it would mangle
|
|
251
298
|
// a script's stdout. JSON consumers always get the full audit unless
|
|
252
299
|
// they explicitly pass deterministicOnly.
|
|
@@ -263,23 +310,28 @@ export async function auditCommand({ task, config, logger, dimensions, json, age
|
|
|
263
310
|
if (!runLlm) {
|
|
264
311
|
// Deterministic-only path. Produce the report from what we already
|
|
265
312
|
// have, persist it (md or json), and exit. Zero tokens spent.
|
|
313
|
+
const harnessJson = harnessSummaryForJson(harnessSummary);
|
|
314
|
+
const currentHarness = harnessSummary?.ok && !harnessSummary.skipped ? harnessSummary : null;
|
|
315
|
+
const history = loadAuditHistoryContext(config?.projectDir || process.cwd(), currentHarness, { showTrend: trend });
|
|
316
|
+
const historyMd = composeHistoryBlock(history);
|
|
266
317
|
const stdoutContent = json
|
|
267
|
-
? JSON.stringify({ deterministic: deterministicCtx, mode: "deterministic-only" }, null, 2)
|
|
268
|
-
: (deterministicOnly ? deterministicMd :
|
|
269
|
-
if (json || deterministicOnly) console.log(stdoutContent);
|
|
318
|
+
? JSON.stringify({ deterministic: deterministicCtx, harnessScore: harnessJson, harnessScoreDiff: history.diff, mode: "deterministic-only" }, null, 2)
|
|
319
|
+
: (deterministicOnly ? `${deterministicMd}\n${harnessMd}${historyMd}` : `${harnessMd}${historyMd}`);
|
|
320
|
+
if (json || deterministicOnly || harnessMd) console.log(stdoutContent);
|
|
270
321
|
|
|
271
322
|
if (reportPath) {
|
|
272
323
|
let payload;
|
|
273
324
|
if (reportPath.endsWith(".json")) {
|
|
274
|
-
payload = JSON.stringify({ deterministic: deterministicCtx, mode: "deterministic-only" }, null, 2);
|
|
325
|
+
payload = JSON.stringify({ deterministic: deterministicCtx, harnessScore: harnessJson, harnessScoreDiff: history.diff, mode: "deterministic-only" }, null, 2);
|
|
275
326
|
} else {
|
|
276
|
-
const header = await buildReportHeader(config?.projectDir, describeInvocation({ task, dimensions, noSonar, noOsv, noSemgrep, json, deterministicOnly, yes }));
|
|
277
|
-
payload = header
|
|
327
|
+
const header = await buildReportHeader(config?.projectDir, describeInvocation({ task, dimensions, noSonar, noOsv, noSemgrep, noHarness, json, deterministicOnly, yes }));
|
|
328
|
+
payload = `${header}${deterministicMd}\n${harnessMd}${historyMd}`;
|
|
278
329
|
}
|
|
279
330
|
await fs.writeFile(reportPath, payload, "utf8");
|
|
280
331
|
runLog.logText(`[audit] report written (deterministic-only) → ${reportPath}`);
|
|
281
332
|
logger.info(`Audit report written: ${reportPath}`);
|
|
282
333
|
}
|
|
334
|
+
persistAuditRun(config?.projectDir || process.cwd(), { runId: new Date().toISOString(), timestamp: new Date().toISOString(), gitSha: null, harness: harnessSummary?.ok && !harnessSummary.skipped ? harnessSummary : null, rawReportPath: reportPath });
|
|
283
335
|
logger.info("Audit completed (deterministic-only).");
|
|
284
336
|
return { ok: true, mode: "deterministic-only", reportPath: reportPath || undefined };
|
|
285
337
|
}
|
|
@@ -306,18 +358,22 @@ export async function auditCommand({ task, config, logger, dimensions, json, age
|
|
|
306
358
|
runLog.logText(`[audit] usage tokens=${usage.total_tokens} cost=$${(usage.cost_usd ?? 0).toFixed(4)} duration=${(usage.durationMs / 1000).toFixed(1)}s`);
|
|
307
359
|
}
|
|
308
360
|
|
|
361
|
+
const harnessJson = harnessSummaryForJson(harnessSummary);
|
|
362
|
+
const currentHarness = harnessSummary?.ok && !harnessSummary.skipped ? harnessSummary : null;
|
|
363
|
+
const history = loadAuditHistoryContext(config?.projectDir || process.cwd(), currentHarness, { showTrend: trend });
|
|
364
|
+
const historyMd = composeHistoryBlock(history);
|
|
309
365
|
let stdoutContent;
|
|
310
366
|
if (json) {
|
|
311
367
|
const jsonPayload = parsed
|
|
312
|
-
? { ...parsed, usage: usage || undefined }
|
|
313
|
-
: { ...roleResult, usage: usage || undefined };
|
|
368
|
+
? { ...parsed, harnessScore: harnessJson, harnessScoreDiff: history.diff, usage: usage || undefined }
|
|
369
|
+
: { ...roleResult, harnessScore: harnessJson, harnessScoreDiff: history.diff, usage: usage || undefined };
|
|
314
370
|
stdoutContent = JSON.stringify(jsonPayload, null, 2);
|
|
315
371
|
} else if (parsed?.summary?.overallHealth) {
|
|
316
|
-
stdoutContent = formatAudit(parsed, usage)
|
|
372
|
+
stdoutContent = `${formatAudit(parsed, usage)}\n${harnessMd}${historyMd}`;
|
|
317
373
|
} else if (parsed?.raw) {
|
|
318
|
-
stdoutContent = parsed.raw + (usage ? `\n\n${formatUsageSummary(usage) || ""}` : "")
|
|
374
|
+
stdoutContent = parsed.raw + (usage ? `\n\n${formatUsageSummary(usage) || ""}` : "") + `\n${harnessMd}${historyMd}`;
|
|
319
375
|
} else {
|
|
320
|
-
stdoutContent = roleResult.summary || "Audit complete."
|
|
376
|
+
stdoutContent = `${roleResult.summary || "Audit complete."}\n${harnessMd}${historyMd}`;
|
|
321
377
|
}
|
|
322
378
|
console.log(stdoutContent);
|
|
323
379
|
|
|
@@ -325,11 +381,11 @@ export async function auditCommand({ task, config, logger, dimensions, json, age
|
|
|
325
381
|
let payload;
|
|
326
382
|
if (reportPath.endsWith(".json")) {
|
|
327
383
|
const jsonPayload = parsed
|
|
328
|
-
? { ...parsed, usage: usage || undefined }
|
|
329
|
-
: { ...roleResult, usage: usage || undefined };
|
|
384
|
+
? { ...parsed, harnessScore: harnessJson, harnessScoreDiff: history.diff, usage: usage || undefined }
|
|
385
|
+
: { ...roleResult, harnessScore: harnessJson, harnessScoreDiff: history.diff, usage: usage || undefined };
|
|
330
386
|
payload = JSON.stringify(jsonPayload, null, 2);
|
|
331
387
|
} else {
|
|
332
|
-
const header = await buildReportHeader(config?.projectDir, describeInvocation({ task, dimensions, noSonar, noOsv, noSemgrep, json, deterministicOnly, yes }));
|
|
388
|
+
const header = await buildReportHeader(config?.projectDir, describeInvocation({ task, dimensions, noSonar, noOsv, noSemgrep, noHarness, json, deterministicOnly, yes }));
|
|
333
389
|
payload = header + stdoutContent + "\n";
|
|
334
390
|
}
|
|
335
391
|
await fs.writeFile(reportPath, payload, "utf8");
|
|
@@ -337,6 +393,7 @@ export async function auditCommand({ task, config, logger, dimensions, json, age
|
|
|
337
393
|
logger.info(`Audit report written: ${reportPath}`);
|
|
338
394
|
}
|
|
339
395
|
|
|
396
|
+
persistAuditRun(config?.projectDir || process.cwd(), { runId: new Date().toISOString(), timestamp: new Date().toISOString(), gitSha: null, harness: harnessSummary?.ok && !harnessSummary.skipped ? harnessSummary : null, rawReportPath: reportPath });
|
|
340
397
|
logger.info("Audit completed.");
|
|
341
398
|
return { ok: true, reportPath: reportPath || undefined };
|
|
342
399
|
});
|
package/src/commands/doctor.js
CHANGED
|
@@ -19,6 +19,7 @@ import { getConfigFileChecks } from "../checks/config-files.js";
|
|
|
19
19
|
import { getBinaryChecks } from "../checks/binaries.js";
|
|
20
20
|
import { getSonarChecks } from "../checks/sonar.js";
|
|
21
21
|
import { getOllamaChecks } from "../checks/ollama.js";
|
|
22
|
+
import { getHarnessScorecardChecks } from "../checks/harness-scorecard.js";
|
|
22
23
|
import { getCiChecks } from "../checks/ci.js";
|
|
23
24
|
import { getRtkChecks } from "../checks/rtk.js";
|
|
24
25
|
import { getNodeChecks } from "../checks/node.js";
|
|
@@ -54,6 +55,7 @@ function buildChecks(config, { projectOnly = false } = {}) {
|
|
|
54
55
|
...getBinaryChecks(),
|
|
55
56
|
...getSonarChecks(),
|
|
56
57
|
...getOllamaChecks(),
|
|
58
|
+
...getHarnessScorecardChecks(),
|
|
57
59
|
...getPortChecks(),
|
|
58
60
|
...getTokenChecks(config),
|
|
59
61
|
...getMcpHealthChecks(),
|