@olegkoval/agent-skills 1.77.0 → 1.78.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +43 -4
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/evidence-bundle.mjs +39 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/native-completion.mjs +39 -0
- package/docs/lekker-openrewrite-pilot.md +94 -0
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +43 -4
- package/plugins/olko-github-pr/skills/lekker-review/scripts/evidence-bundle.mjs +39 -0
- package/plugins/olko-github-pr/skills/lekker-review/scripts/native-completion.mjs +39 -0
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
3
|
//
|
|
4
4
|
// Usage:
|
|
5
|
-
// node eval-runner.mjs --manifest
|
|
5
|
+
// node eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]
|
|
6
6
|
//
|
|
7
7
|
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
8
|
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
22
22
|
|
|
23
23
|
import { readFileSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
24
25
|
import { fileURLToPath } from 'node:url';
|
|
25
26
|
|
|
26
27
|
export const LINE_WINDOW = 10;
|
|
@@ -74,6 +75,7 @@ export function validateManifest(manifest) {
|
|
|
74
75
|
return prs;
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
/** Normalize archived review evidence, returning findings only when every identity check passes. */
|
|
77
79
|
export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
78
80
|
const key = caseKey(entry);
|
|
79
81
|
const unscored = (reason) => ({ key, reason });
|
|
@@ -98,6 +100,7 @@ export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
// Validate archive references before reading any run directories.
|
|
103
|
+
/** Reject duplicate archive run IDs before archive paths are read. */
|
|
101
104
|
export function validateArchiveRunIds(entries) {
|
|
102
105
|
const duplicate = entries.find((entry, index) => entry.runId
|
|
103
106
|
&& entries.slice(0, index).some(previous => previous.runId === entry.runId));
|
|
@@ -105,6 +108,30 @@ export function validateArchiveRunIds(entries) {
|
|
|
105
108
|
return entries;
|
|
106
109
|
}
|
|
107
110
|
|
|
111
|
+
function readArchiveJson(path) {
|
|
112
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Load valid archived findings and retain a reason for each rejected run. */
|
|
116
|
+
function archivedResults(entries, runsDir) {
|
|
117
|
+
const results = {};
|
|
118
|
+
const reasons = [];
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (typeof entry.runId !== 'string' || !RUN_ID.test(entry.runId)) {
|
|
121
|
+
reasons.push({ key: caseKey(entry), reason: 'unsafe or missing runId' });
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const dir = join(runsDir, entry.runId);
|
|
125
|
+
const evidence = normalizeArchivedEvidence(entry,
|
|
126
|
+
readArchiveJson(join(dir, 'run.json')),
|
|
127
|
+
readArchiveJson(join(dir, 'worktree.json')),
|
|
128
|
+
readArchiveJson(join(dir, 'findings.json')));
|
|
129
|
+
if (evidence.reason) reasons.push({ key: evidence.key, reason: evidence.reason });
|
|
130
|
+
else results[evidence.key] = evidence.findings;
|
|
131
|
+
}
|
|
132
|
+
return { results, reasons };
|
|
133
|
+
}
|
|
134
|
+
|
|
108
135
|
export function score(manifest, results) {
|
|
109
136
|
const rows = validateManifest(manifest).map((e) => {
|
|
110
137
|
const key = caseKey(e);
|
|
@@ -147,12 +174,24 @@ export function score(manifest, results) {
|
|
|
147
174
|
|
|
148
175
|
function main(argv) {
|
|
149
176
|
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
150
|
-
|
|
151
|
-
|
|
177
|
+
const results = flag('results');
|
|
178
|
+
const archiveIndex = argv.indexOf('--archive');
|
|
179
|
+
const archive = archiveIndex === -1 ? null : argv[archiveIndex + 1];
|
|
180
|
+
const resultsIndex = argv.indexOf('--results');
|
|
181
|
+
if (!flag('manifest') || (archiveIndex !== -1 && (!archive || archive.startsWith('--')))
|
|
182
|
+
|| (resultsIndex !== -1 && (!results || results.startsWith('--')))
|
|
183
|
+
|| Boolean(results) === Boolean(archive)) {
|
|
184
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]');
|
|
152
185
|
process.exit(2);
|
|
153
186
|
}
|
|
154
|
-
const
|
|
187
|
+
const manifest = JSON.parse(readFileSync(flag('manifest'), 'utf8'));
|
|
188
|
+
const source = archive
|
|
189
|
+
? archivedResults(validateArchiveRunIds(validateManifest(manifest)), archive)
|
|
190
|
+
: { results: JSON.parse(readFileSync(results, 'utf8')), reasons: null };
|
|
191
|
+
const out = score(manifest, source.results);
|
|
192
|
+
if (source.reasons) out.unscoredReasons = source.reasons;
|
|
155
193
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
194
|
+
for (const { key, reason } of source.reasons || []) console.error(`unscored ${key}: ${reason}`);
|
|
156
195
|
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
157
196
|
if (out.buggyRuns === undefined) {
|
|
158
197
|
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { collectGitEvidence } from './git-evidence.mjs';
|
|
5
|
+
import { validateLintReceipt } from './validate-lint-receipt.mjs';
|
|
6
|
+
|
|
7
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
const unavailableLint = reason => ({ captureStatus: 'unavailable', reason });
|
|
9
|
+
const unavailableBundle = reason => ({ schema: 1,
|
|
10
|
+
git: { schema: 1, status: 'unavailable', reason }, lint: unavailableLint(reason) });
|
|
11
|
+
|
|
12
|
+
export function collectEvidence(worktreeMetadata) {
|
|
13
|
+
if (!record(worktreeMetadata)) return unavailableBundle('worktree metadata is missing or malformed');
|
|
14
|
+
const git = collectGitEvidence({ worktreePath: worktreeMetadata.worktreePath,
|
|
15
|
+
mergeBase: worktreeMetadata.mergeBase, headSha: worktreeMetadata.headSha });
|
|
16
|
+
if (git.status !== 'available') return { schema: 1, git, lint: unavailableLint(`Git evidence unavailable: ${git.reason}`) };
|
|
17
|
+
if (typeof worktreeMetadata.eslintJson !== 'string' || !worktreeMetadata.eslintJson) return { schema: 1, git, lint: unavailableLint('lint report path unavailable') };
|
|
18
|
+
if (typeof worktreeMetadata.eslintProvenance !== 'string' || !worktreeMetadata.eslintProvenance) return { schema: 1, git, lint: unavailableLint('lint provenance path unavailable') };
|
|
19
|
+
let reportBytes, receipt;
|
|
20
|
+
try { reportBytes = readFileSync(worktreeMetadata.eslintJson); } catch { return { schema: 1, git, lint: unavailableLint('lint report could not be read') }; }
|
|
21
|
+
try { receipt = JSON.parse(readFileSync(worktreeMetadata.eslintProvenance, 'utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint provenance is missing or malformed') }; }
|
|
22
|
+
const provenance = validateLintReceipt(receipt, { baseSha: git.provenance.baseSha,
|
|
23
|
+
headSha: git.provenance.headSha, trackedClean: git.provenance.trackedCleanBefore && git.provenance.trackedCleanAfter }, reportBytes);
|
|
24
|
+
if (provenance.status !== 'available') return { schema: 1, git, lint: unavailableLint(provenance.reason) };
|
|
25
|
+
let report;
|
|
26
|
+
try { report = JSON.parse(reportBytes.toString('utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint report is malformed JSON') }; }
|
|
27
|
+
if (!Array.isArray(report)) return { schema: 1, git, lint: unavailableLint('lint report is not a JSON array') };
|
|
28
|
+
return { schema: 1, git, lint: { captureStatus: 'available', exitCode: provenance.exitCode,
|
|
29
|
+
reportSha256: provenance.reportSha256, report } };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function main(argv) {
|
|
33
|
+
const i = argv.indexOf('--worktree'), path = i < 0 ? null : argv[i + 1];
|
|
34
|
+
if (!path || path.startsWith('--')) { console.error('usage: evidence-bundle.mjs --worktree <worktree.json>'); process.exitCode = 2; return; }
|
|
35
|
+
try { console.log(JSON.stringify(collectEvidence(JSON.parse(readFileSync(path, 'utf8'))))); }
|
|
36
|
+
catch { console.log(JSON.stringify(unavailableBundle('worktree metadata could not be read'))); process.exitCode = 2; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const DIMENSIONS = {
|
|
4
|
+
deps: ['deps'],
|
|
5
|
+
scan: ['triage-quality', 'triage-logic', 'project-rules'],
|
|
6
|
+
medium: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
7
|
+
deep: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
8
|
+
};
|
|
9
|
+
const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i;
|
|
10
|
+
const HASH = /^[a-f0-9]{64}$/i;
|
|
11
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
12
|
+
const count = value => Number.isSafeInteger(value) && value >= 0;
|
|
13
|
+
|
|
14
|
+
/** Validate a native completion receipt against run policy and exact findings bytes. */
|
|
15
|
+
export function validateNativeCompletion(receipt, expected, findingsBytes) {
|
|
16
|
+
const unavailable = reason => ({ available: false, reason });
|
|
17
|
+
if (!record(expected) || !record(expected.eligible) || !record(receipt)) return unavailable('receipt or expected inputs are malformed');
|
|
18
|
+
if (!['deps', 'scan', 'medium', 'deep'].includes(expected.depth) || typeof expected.worktreeAvailable !== 'boolean'
|
|
19
|
+
|| !count(expected.eligible.verify) || !count(expected.eligible.prove)) return unavailable('expected policy inputs are malformed');
|
|
20
|
+
if (typeof expected.runId !== 'string' || !expected.runId || typeof expected.headSha !== 'string' || !SHA.test(expected.headSha)) return unavailable('expected identity is malformed');
|
|
21
|
+
if (!(findingsBytes instanceof Uint8Array)) return unavailable('findings bytes are unavailable');
|
|
22
|
+
if (receipt.schema !== 1 || receipt.engine !== 'codex' || receipt.status !== 'complete') return unavailable('native completion is not complete');
|
|
23
|
+
if (receipt.runId !== expected.runId || receipt.depth !== expected.depth) return unavailable('run identity or depth mismatch');
|
|
24
|
+
if (typeof receipt.headSha !== 'string' || !SHA.test(receipt.headSha) || receipt.headSha.toLowerCase() !== expected.headSha.toLowerCase()) return unavailable('head SHA mismatch');
|
|
25
|
+
if (typeof receipt.findingsSha256 !== 'string' || !HASH.test(receipt.findingsSha256)
|
|
26
|
+
|| createHash('sha256').update(findingsBytes).digest('hex') !== receipt.findingsSha256.toLowerCase()) return unavailable('findings hash mismatch');
|
|
27
|
+
const required = DIMENSIONS[expected.depth];
|
|
28
|
+
if (!Array.isArray(receipt.dimensions) || receipt.dimensions.length !== required.length) return unavailable('required review dimensions are missing or duplicated');
|
|
29
|
+
const dimensions = Array.from(receipt.dimensions);
|
|
30
|
+
if (dimensions.some(d => !record(d) || !required.includes(d.name) || d.status !== 'completed')) return unavailable('review dimensions are missing, unknown, or incomplete');
|
|
31
|
+
if (new Set(dimensions.map(d => d.name)).size !== required.length) return unavailable('review dimensions are duplicated');
|
|
32
|
+
const verify = expected.depth === 'deps' ? 0 : expected.eligible.verify;
|
|
33
|
+
const prove = ['medium', 'deep'].includes(expected.depth) && expected.worktreeAvailable ? Math.min(5, expected.eligible.prove) : 0;
|
|
34
|
+
const validStage = (stage, scheduled) => record(stage) && count(stage.scheduled) && count(stage.completed)
|
|
35
|
+
&& count(stage.failed) && stage.scheduled === scheduled && stage.completed === scheduled && stage.failed === 0;
|
|
36
|
+
if (!record(receipt.stages) || !validStage(receipt.stages.verify, verify) || !validStage(receipt.stages.prove, prove)) return unavailable('verification or proof stage accounting is incomplete');
|
|
37
|
+
if (receipt.stages.critic !== (expected.depth === 'deep' ? 'completed' : 'not_required')) return unavailable('critic stage status does not match depth');
|
|
38
|
+
return { available: true, reason: null };
|
|
39
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# OpenRewrite native JavaScript pilot
|
|
2
|
+
|
|
3
|
+
This optional fixture experiment runs a shipped OpenRewrite recipe without the
|
|
4
|
+
Moderne CLI. It adds no default review dependency and applies no source fixes.
|
|
5
|
+
It tests a concrete static-analysis opportunity while all six semantic review
|
|
6
|
+
dimensions and Verify/Critic/Prove remain required.
|
|
7
|
+
|
|
8
|
+
## Reproduce in isolation
|
|
9
|
+
|
|
10
|
+
Requires npm and Docker. Run from this repository's root. Installation disables
|
|
11
|
+
package lifecycle scripts. The recipe runs with no network, host mounts or
|
|
12
|
+
forwarded credentials; only the temporary fixture project enters the container.
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
pilot_dir=$(mktemp -d)
|
|
16
|
+
npm install --prefix "$pilot_dir" --ignore-scripts --omit=optional \
|
|
17
|
+
--no-audit --no-fund --save-exact @openrewrite/rewrite@8.100.3
|
|
18
|
+
cp tests/openrewrite-pilot.mjs "$pilot_dir/pilot.mjs"
|
|
19
|
+
pilot_container=$(docker create --network none --cap-drop ALL \
|
|
20
|
+
--security-opt no-new-privileges --memory 1g --pids-limit 128 --workdir /work \
|
|
21
|
+
node@sha256:0e1754758c2df4cfa3e68d59e2e1b7374bb7fd9ea6328eaaacf15cc934af8202 \
|
|
22
|
+
node pilot.mjs)
|
|
23
|
+
docker cp "$pilot_dir/." "$pilot_container:/work"
|
|
24
|
+
docker start -a "$pilot_container" > "$pilot_dir/first.stdout" 2> "$pilot_dir/first.stderr"
|
|
25
|
+
docker inspect "$pilot_container" --format '{{.State.ExitCode}}'
|
|
26
|
+
docker cp "$pilot_container:/work/artifacts" "$pilot_dir/first-artifacts"
|
|
27
|
+
docker start -a "$pilot_container" > "$pilot_dir/second.stdout" 2> "$pilot_dir/second.stderr"
|
|
28
|
+
docker inspect "$pilot_container" --format '{{.State.ExitCode}}'
|
|
29
|
+
docker cp "$pilot_container:/work/artifacts" "$pilot_dir/second-artifacts"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Both exit codes must be zero. Each process checks its two fresh-input passes
|
|
33
|
+
for identical normalized output and validates every expected result. Compare
|
|
34
|
+
the `outputSha256` values across the two stdout files as a separate cold-process
|
|
35
|
+
repeatability gate. Preserve stderr, package-lock.json, expectations, outputs and
|
|
36
|
+
patches before removing the stopped container and temporary directory. A failed
|
|
37
|
+
process is unavailable evidence, never a clean review. No fixtures execute the
|
|
38
|
+
JavaScript being analyzed.
|
|
39
|
+
|
|
40
|
+
## Frozen experiment
|
|
41
|
+
|
|
42
|
+
Package: `@openrewrite/rewrite@8.100.3`.
|
|
43
|
+
Tested runtime: Node v22.23.4 in the pinned Linux/arm64 container image above.
|
|
44
|
+
Recipe: `org.openrewrite.javascript.cleanup.async-callback-in-sync-array-method`.
|
|
45
|
+
The harness uses the package's JavaScriptParser, scheduleRun and TreePrinters;
|
|
46
|
+
it implements no substitute detection rule.
|
|
47
|
+
|
|
48
|
+
| Fixture | Expected and observed |
|
|
49
|
+
|---|---|
|
|
50
|
+
| Array.filter with async callback | Warning |
|
|
51
|
+
| Array.forEach with async callback | Warning |
|
|
52
|
+
| Array.filter with sync callback | Unchanged |
|
|
53
|
+
| Array.map with async callback | Unchanged |
|
|
54
|
+
| Typed custom filter accepting async callback | Unchanged |
|
|
55
|
+
| Known-safe custom filter with erased receiver type | Warning: characterized false positive |
|
|
56
|
+
| Invalid TypeScript syntax | Parse-error / unavailable |
|
|
57
|
+
|
|
58
|
+
The extra untyped custom filter returns the callback's promise to its caller;
|
|
59
|
+
unlike Array.filter, it never interprets that promise as a boolean. The warning
|
|
60
|
+
is therefore not a demonstrated defect. Its expected warning characterizes the
|
|
61
|
+
recipe's limitation, not a passing semantic-quality gate. The `any` annotation
|
|
62
|
+
exists only to make the lost type attribution explicit in this isolated fixture.
|
|
63
|
+
|
|
64
|
+
The original six-fixture 7 November 2026 pilot passed in two cold processes with two
|
|
65
|
+
passes each. All four normalized outputs matched SHA-256
|
|
66
|
+
`ae1a6381c263289b5d006eda96cd73e87d1c9f5cbb95040fdb7c3e916d410bb9`.
|
|
67
|
+
Pass times were 3492/3078 ms and 2880/2136 ms, excluding image/npm setup,
|
|
68
|
+
container startup and coordinator work. These tiny-fixture timings establish
|
|
69
|
+
neither repository-scale speed nor whole-review savings. The local raw archive
|
|
70
|
+
is `~/.local/state/lekker-review-evaluation/runs/20261107-openrewrite-native`.
|
|
71
|
+
The wrapper checks the package version; the retained lockfile pins transitive
|
|
72
|
+
dependencies for that historical run. A fresh install may resolve different
|
|
73
|
+
transitives and is a new experiment, not an exact historical replay.
|
|
74
|
+
The expanded seven-fixture run also makes the forEach callback suspend before
|
|
75
|
+
checking whether its work finished. Including the safe untyped receiver, it produced
|
|
76
|
+
`52ba49e0a0fdc5a7fb0ad13574e04f11a102c19b029cf2e41d95bfab460e2613`.
|
|
77
|
+
|
|
78
|
+
## Adoption limits
|
|
79
|
+
|
|
80
|
+
This proves one recipe/version on synthetic fixtures, not population recall or
|
|
81
|
+
false-positive rates. A false positive on a safe untyped receiver was observed;
|
|
82
|
+
warnings need semantic verification. Preserve parse failures as unavailable.
|
|
83
|
+
Printed markers/patches are evidence only; do not auto-apply them. The fixture
|
|
84
|
+
runner is opt-in and intentionally absent from the normal dependency-free suite.
|
|
85
|
+
|
|
86
|
+
The native package route worked without authentication; this does not prove
|
|
87
|
+
every recipe works without Java/RPC or change the documented Moderne CLI's
|
|
88
|
+
authentication requirements. The JavaScript package uses the Moderne Source
|
|
89
|
+
Available License, distinct from Apache-licensed OpenRewrite core. No package
|
|
90
|
+
redistribution or mandatory production integration is proposed here.
|
|
91
|
+
|
|
92
|
+
References: [JavaScript development environment](https://docs.openrewrite.org/authoring-recipes/javascript-recipe-development-environment),
|
|
93
|
+
[recipe testing example](https://docs.openrewrite.org/authoring-recipes/writing-a-javascript-refactoring-recipe),
|
|
94
|
+
and [licensing](https://docs.openrewrite.org/licensing/openrewrite-licensing).
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
3
|
//
|
|
4
4
|
// Usage:
|
|
5
|
-
// node eval-runner.mjs --manifest
|
|
5
|
+
// node eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]
|
|
6
6
|
//
|
|
7
7
|
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
8
|
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
22
22
|
|
|
23
23
|
import { readFileSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
24
25
|
import { fileURLToPath } from 'node:url';
|
|
25
26
|
|
|
26
27
|
export const LINE_WINDOW = 10;
|
|
@@ -74,6 +75,7 @@ export function validateManifest(manifest) {
|
|
|
74
75
|
return prs;
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
/** Normalize archived review evidence, returning findings only when every identity check passes. */
|
|
77
79
|
export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
78
80
|
const key = caseKey(entry);
|
|
79
81
|
const unscored = (reason) => ({ key, reason });
|
|
@@ -98,6 +100,7 @@ export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
// Validate archive references before reading any run directories.
|
|
103
|
+
/** Reject duplicate archive run IDs before archive paths are read. */
|
|
101
104
|
export function validateArchiveRunIds(entries) {
|
|
102
105
|
const duplicate = entries.find((entry, index) => entry.runId
|
|
103
106
|
&& entries.slice(0, index).some(previous => previous.runId === entry.runId));
|
|
@@ -105,6 +108,30 @@ export function validateArchiveRunIds(entries) {
|
|
|
105
108
|
return entries;
|
|
106
109
|
}
|
|
107
110
|
|
|
111
|
+
function readArchiveJson(path) {
|
|
112
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Load valid archived findings and retain a reason for each rejected run. */
|
|
116
|
+
function archivedResults(entries, runsDir) {
|
|
117
|
+
const results = {};
|
|
118
|
+
const reasons = [];
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (typeof entry.runId !== 'string' || !RUN_ID.test(entry.runId)) {
|
|
121
|
+
reasons.push({ key: caseKey(entry), reason: 'unsafe or missing runId' });
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const dir = join(runsDir, entry.runId);
|
|
125
|
+
const evidence = normalizeArchivedEvidence(entry,
|
|
126
|
+
readArchiveJson(join(dir, 'run.json')),
|
|
127
|
+
readArchiveJson(join(dir, 'worktree.json')),
|
|
128
|
+
readArchiveJson(join(dir, 'findings.json')));
|
|
129
|
+
if (evidence.reason) reasons.push({ key: evidence.key, reason: evidence.reason });
|
|
130
|
+
else results[evidence.key] = evidence.findings;
|
|
131
|
+
}
|
|
132
|
+
return { results, reasons };
|
|
133
|
+
}
|
|
134
|
+
|
|
108
135
|
export function score(manifest, results) {
|
|
109
136
|
const rows = validateManifest(manifest).map((e) => {
|
|
110
137
|
const key = caseKey(e);
|
|
@@ -147,12 +174,24 @@ export function score(manifest, results) {
|
|
|
147
174
|
|
|
148
175
|
function main(argv) {
|
|
149
176
|
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
150
|
-
|
|
151
|
-
|
|
177
|
+
const results = flag('results');
|
|
178
|
+
const archiveIndex = argv.indexOf('--archive');
|
|
179
|
+
const archive = archiveIndex === -1 ? null : argv[archiveIndex + 1];
|
|
180
|
+
const resultsIndex = argv.indexOf('--results');
|
|
181
|
+
if (!flag('manifest') || (archiveIndex !== -1 && (!archive || archive.startsWith('--')))
|
|
182
|
+
|| (resultsIndex !== -1 && (!results || results.startsWith('--')))
|
|
183
|
+
|| Boolean(results) === Boolean(archive)) {
|
|
184
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]');
|
|
152
185
|
process.exit(2);
|
|
153
186
|
}
|
|
154
|
-
const
|
|
187
|
+
const manifest = JSON.parse(readFileSync(flag('manifest'), 'utf8'));
|
|
188
|
+
const source = archive
|
|
189
|
+
? archivedResults(validateArchiveRunIds(validateManifest(manifest)), archive)
|
|
190
|
+
: { results: JSON.parse(readFileSync(results, 'utf8')), reasons: null };
|
|
191
|
+
const out = score(manifest, source.results);
|
|
192
|
+
if (source.reasons) out.unscoredReasons = source.reasons;
|
|
155
193
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
194
|
+
for (const { key, reason } of source.reasons || []) console.error(`unscored ${key}: ${reason}`);
|
|
156
195
|
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
157
196
|
if (out.buggyRuns === undefined) {
|
|
158
197
|
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { collectGitEvidence } from './git-evidence.mjs';
|
|
5
|
+
import { validateLintReceipt } from './validate-lint-receipt.mjs';
|
|
6
|
+
|
|
7
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
const unavailableLint = reason => ({ captureStatus: 'unavailable', reason });
|
|
9
|
+
const unavailableBundle = reason => ({ schema: 1,
|
|
10
|
+
git: { schema: 1, status: 'unavailable', reason }, lint: unavailableLint(reason) });
|
|
11
|
+
|
|
12
|
+
export function collectEvidence(worktreeMetadata) {
|
|
13
|
+
if (!record(worktreeMetadata)) return unavailableBundle('worktree metadata is missing or malformed');
|
|
14
|
+
const git = collectGitEvidence({ worktreePath: worktreeMetadata.worktreePath,
|
|
15
|
+
mergeBase: worktreeMetadata.mergeBase, headSha: worktreeMetadata.headSha });
|
|
16
|
+
if (git.status !== 'available') return { schema: 1, git, lint: unavailableLint(`Git evidence unavailable: ${git.reason}`) };
|
|
17
|
+
if (typeof worktreeMetadata.eslintJson !== 'string' || !worktreeMetadata.eslintJson) return { schema: 1, git, lint: unavailableLint('lint report path unavailable') };
|
|
18
|
+
if (typeof worktreeMetadata.eslintProvenance !== 'string' || !worktreeMetadata.eslintProvenance) return { schema: 1, git, lint: unavailableLint('lint provenance path unavailable') };
|
|
19
|
+
let reportBytes, receipt;
|
|
20
|
+
try { reportBytes = readFileSync(worktreeMetadata.eslintJson); } catch { return { schema: 1, git, lint: unavailableLint('lint report could not be read') }; }
|
|
21
|
+
try { receipt = JSON.parse(readFileSync(worktreeMetadata.eslintProvenance, 'utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint provenance is missing or malformed') }; }
|
|
22
|
+
const provenance = validateLintReceipt(receipt, { baseSha: git.provenance.baseSha,
|
|
23
|
+
headSha: git.provenance.headSha, trackedClean: git.provenance.trackedCleanBefore && git.provenance.trackedCleanAfter }, reportBytes);
|
|
24
|
+
if (provenance.status !== 'available') return { schema: 1, git, lint: unavailableLint(provenance.reason) };
|
|
25
|
+
let report;
|
|
26
|
+
try { report = JSON.parse(reportBytes.toString('utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint report is malformed JSON') }; }
|
|
27
|
+
if (!Array.isArray(report)) return { schema: 1, git, lint: unavailableLint('lint report is not a JSON array') };
|
|
28
|
+
return { schema: 1, git, lint: { captureStatus: 'available', exitCode: provenance.exitCode,
|
|
29
|
+
reportSha256: provenance.reportSha256, report } };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function main(argv) {
|
|
33
|
+
const i = argv.indexOf('--worktree'), path = i < 0 ? null : argv[i + 1];
|
|
34
|
+
if (!path || path.startsWith('--')) { console.error('usage: evidence-bundle.mjs --worktree <worktree.json>'); process.exitCode = 2; return; }
|
|
35
|
+
try { console.log(JSON.stringify(collectEvidence(JSON.parse(readFileSync(path, 'utf8'))))); }
|
|
36
|
+
catch { console.log(JSON.stringify(unavailableBundle('worktree metadata could not be read'))); process.exitCode = 2; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const DIMENSIONS = {
|
|
4
|
+
deps: ['deps'],
|
|
5
|
+
scan: ['triage-quality', 'triage-logic', 'project-rules'],
|
|
6
|
+
medium: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
7
|
+
deep: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
8
|
+
};
|
|
9
|
+
const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i;
|
|
10
|
+
const HASH = /^[a-f0-9]{64}$/i;
|
|
11
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
12
|
+
const count = value => Number.isSafeInteger(value) && value >= 0;
|
|
13
|
+
|
|
14
|
+
/** Validate a native completion receipt against run policy and exact findings bytes. */
|
|
15
|
+
export function validateNativeCompletion(receipt, expected, findingsBytes) {
|
|
16
|
+
const unavailable = reason => ({ available: false, reason });
|
|
17
|
+
if (!record(expected) || !record(expected.eligible) || !record(receipt)) return unavailable('receipt or expected inputs are malformed');
|
|
18
|
+
if (!['deps', 'scan', 'medium', 'deep'].includes(expected.depth) || typeof expected.worktreeAvailable !== 'boolean'
|
|
19
|
+
|| !count(expected.eligible.verify) || !count(expected.eligible.prove)) return unavailable('expected policy inputs are malformed');
|
|
20
|
+
if (typeof expected.runId !== 'string' || !expected.runId || typeof expected.headSha !== 'string' || !SHA.test(expected.headSha)) return unavailable('expected identity is malformed');
|
|
21
|
+
if (!(findingsBytes instanceof Uint8Array)) return unavailable('findings bytes are unavailable');
|
|
22
|
+
if (receipt.schema !== 1 || receipt.engine !== 'codex' || receipt.status !== 'complete') return unavailable('native completion is not complete');
|
|
23
|
+
if (receipt.runId !== expected.runId || receipt.depth !== expected.depth) return unavailable('run identity or depth mismatch');
|
|
24
|
+
if (typeof receipt.headSha !== 'string' || !SHA.test(receipt.headSha) || receipt.headSha.toLowerCase() !== expected.headSha.toLowerCase()) return unavailable('head SHA mismatch');
|
|
25
|
+
if (typeof receipt.findingsSha256 !== 'string' || !HASH.test(receipt.findingsSha256)
|
|
26
|
+
|| createHash('sha256').update(findingsBytes).digest('hex') !== receipt.findingsSha256.toLowerCase()) return unavailable('findings hash mismatch');
|
|
27
|
+
const required = DIMENSIONS[expected.depth];
|
|
28
|
+
if (!Array.isArray(receipt.dimensions) || receipt.dimensions.length !== required.length) return unavailable('required review dimensions are missing or duplicated');
|
|
29
|
+
const dimensions = Array.from(receipt.dimensions);
|
|
30
|
+
if (dimensions.some(d => !record(d) || !required.includes(d.name) || d.status !== 'completed')) return unavailable('review dimensions are missing, unknown, or incomplete');
|
|
31
|
+
if (new Set(dimensions.map(d => d.name)).size !== required.length) return unavailable('review dimensions are duplicated');
|
|
32
|
+
const verify = expected.depth === 'deps' ? 0 : expected.eligible.verify;
|
|
33
|
+
const prove = ['medium', 'deep'].includes(expected.depth) && expected.worktreeAvailable ? Math.min(5, expected.eligible.prove) : 0;
|
|
34
|
+
const validStage = (stage, scheduled) => record(stage) && count(stage.scheduled) && count(stage.completed)
|
|
35
|
+
&& count(stage.failed) && stage.scheduled === scheduled && stage.completed === scheduled && stage.failed === 0;
|
|
36
|
+
if (!record(receipt.stages) || !validStage(receipt.stages.verify, verify) || !validStage(receipt.stages.prove, prove)) return unavailable('verification or proof stage accounting is incomplete');
|
|
37
|
+
if (receipt.stages.critic !== (expected.depth === 'deep' ? 'completed' : 'not_required')) return unavailable('critic stage status does not match depth');
|
|
38
|
+
return { available: true, reason: null };
|
|
39
|
+
}
|