@olegkoval/agent-skills 1.77.0 → 1.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +43 -4
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/evidence-bundle.mjs +39 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/native-completion.mjs +39 -0
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +43 -4
- package/plugins/olko-github-pr/skills/lekker-review/scripts/evidence-bundle.mjs +39 -0
- package/plugins/olko-github-pr/skills/lekker-review/scripts/native-completion.mjs +39 -0
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
3
|
//
|
|
4
4
|
// Usage:
|
|
5
|
-
// node eval-runner.mjs --manifest
|
|
5
|
+
// node eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]
|
|
6
6
|
//
|
|
7
7
|
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
8
|
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
22
22
|
|
|
23
23
|
import { readFileSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
24
25
|
import { fileURLToPath } from 'node:url';
|
|
25
26
|
|
|
26
27
|
export const LINE_WINDOW = 10;
|
|
@@ -74,6 +75,7 @@ export function validateManifest(manifest) {
|
|
|
74
75
|
return prs;
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
/** Normalize archived review evidence, returning findings only when every identity check passes. */
|
|
77
79
|
export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
78
80
|
const key = caseKey(entry);
|
|
79
81
|
const unscored = (reason) => ({ key, reason });
|
|
@@ -98,6 +100,7 @@ export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
// Validate archive references before reading any run directories.
|
|
103
|
+
/** Reject duplicate archive run IDs before archive paths are read. */
|
|
101
104
|
export function validateArchiveRunIds(entries) {
|
|
102
105
|
const duplicate = entries.find((entry, index) => entry.runId
|
|
103
106
|
&& entries.slice(0, index).some(previous => previous.runId === entry.runId));
|
|
@@ -105,6 +108,30 @@ export function validateArchiveRunIds(entries) {
|
|
|
105
108
|
return entries;
|
|
106
109
|
}
|
|
107
110
|
|
|
111
|
+
function readArchiveJson(path) {
|
|
112
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Load valid archived findings and retain a reason for each rejected run. */
|
|
116
|
+
function archivedResults(entries, runsDir) {
|
|
117
|
+
const results = {};
|
|
118
|
+
const reasons = [];
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (typeof entry.runId !== 'string' || !RUN_ID.test(entry.runId)) {
|
|
121
|
+
reasons.push({ key: caseKey(entry), reason: 'unsafe or missing runId' });
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const dir = join(runsDir, entry.runId);
|
|
125
|
+
const evidence = normalizeArchivedEvidence(entry,
|
|
126
|
+
readArchiveJson(join(dir, 'run.json')),
|
|
127
|
+
readArchiveJson(join(dir, 'worktree.json')),
|
|
128
|
+
readArchiveJson(join(dir, 'findings.json')));
|
|
129
|
+
if (evidence.reason) reasons.push({ key: evidence.key, reason: evidence.reason });
|
|
130
|
+
else results[evidence.key] = evidence.findings;
|
|
131
|
+
}
|
|
132
|
+
return { results, reasons };
|
|
133
|
+
}
|
|
134
|
+
|
|
108
135
|
export function score(manifest, results) {
|
|
109
136
|
const rows = validateManifest(manifest).map((e) => {
|
|
110
137
|
const key = caseKey(e);
|
|
@@ -147,12 +174,24 @@ export function score(manifest, results) {
|
|
|
147
174
|
|
|
148
175
|
function main(argv) {
|
|
149
176
|
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
150
|
-
|
|
151
|
-
|
|
177
|
+
const results = flag('results');
|
|
178
|
+
const archiveIndex = argv.indexOf('--archive');
|
|
179
|
+
const archive = archiveIndex === -1 ? null : argv[archiveIndex + 1];
|
|
180
|
+
const resultsIndex = argv.indexOf('--results');
|
|
181
|
+
if (!flag('manifest') || (archiveIndex !== -1 && (!archive || archive.startsWith('--')))
|
|
182
|
+
|| (resultsIndex !== -1 && (!results || results.startsWith('--')))
|
|
183
|
+
|| Boolean(results) === Boolean(archive)) {
|
|
184
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]');
|
|
152
185
|
process.exit(2);
|
|
153
186
|
}
|
|
154
|
-
const
|
|
187
|
+
const manifest = JSON.parse(readFileSync(flag('manifest'), 'utf8'));
|
|
188
|
+
const source = archive
|
|
189
|
+
? archivedResults(validateArchiveRunIds(validateManifest(manifest)), archive)
|
|
190
|
+
: { results: JSON.parse(readFileSync(results, 'utf8')), reasons: null };
|
|
191
|
+
const out = score(manifest, source.results);
|
|
192
|
+
if (source.reasons) out.unscoredReasons = source.reasons;
|
|
155
193
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
194
|
+
for (const { key, reason } of source.reasons || []) console.error(`unscored ${key}: ${reason}`);
|
|
156
195
|
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
157
196
|
if (out.buggyRuns === undefined) {
|
|
158
197
|
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { collectGitEvidence } from './git-evidence.mjs';
|
|
5
|
+
import { validateLintReceipt } from './validate-lint-receipt.mjs';
|
|
6
|
+
|
|
7
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
const unavailableLint = reason => ({ captureStatus: 'unavailable', reason });
|
|
9
|
+
const unavailableBundle = reason => ({ schema: 1,
|
|
10
|
+
git: { schema: 1, status: 'unavailable', reason }, lint: unavailableLint(reason) });
|
|
11
|
+
|
|
12
|
+
export function collectEvidence(worktreeMetadata) {
|
|
13
|
+
if (!record(worktreeMetadata)) return unavailableBundle('worktree metadata is missing or malformed');
|
|
14
|
+
const git = collectGitEvidence({ worktreePath: worktreeMetadata.worktreePath,
|
|
15
|
+
mergeBase: worktreeMetadata.mergeBase, headSha: worktreeMetadata.headSha });
|
|
16
|
+
if (git.status !== 'available') return { schema: 1, git, lint: unavailableLint(`Git evidence unavailable: ${git.reason}`) };
|
|
17
|
+
if (typeof worktreeMetadata.eslintJson !== 'string' || !worktreeMetadata.eslintJson) return { schema: 1, git, lint: unavailableLint('lint report path unavailable') };
|
|
18
|
+
if (typeof worktreeMetadata.eslintProvenance !== 'string' || !worktreeMetadata.eslintProvenance) return { schema: 1, git, lint: unavailableLint('lint provenance path unavailable') };
|
|
19
|
+
let reportBytes, receipt;
|
|
20
|
+
try { reportBytes = readFileSync(worktreeMetadata.eslintJson); } catch { return { schema: 1, git, lint: unavailableLint('lint report could not be read') }; }
|
|
21
|
+
try { receipt = JSON.parse(readFileSync(worktreeMetadata.eslintProvenance, 'utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint provenance is missing or malformed') }; }
|
|
22
|
+
const provenance = validateLintReceipt(receipt, { baseSha: git.provenance.baseSha,
|
|
23
|
+
headSha: git.provenance.headSha, trackedClean: git.provenance.trackedCleanBefore && git.provenance.trackedCleanAfter }, reportBytes);
|
|
24
|
+
if (provenance.status !== 'available') return { schema: 1, git, lint: unavailableLint(provenance.reason) };
|
|
25
|
+
let report;
|
|
26
|
+
try { report = JSON.parse(reportBytes.toString('utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint report is malformed JSON') }; }
|
|
27
|
+
if (!Array.isArray(report)) return { schema: 1, git, lint: unavailableLint('lint report is not a JSON array') };
|
|
28
|
+
return { schema: 1, git, lint: { captureStatus: 'available', exitCode: provenance.exitCode,
|
|
29
|
+
reportSha256: provenance.reportSha256, report } };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function main(argv) {
|
|
33
|
+
const i = argv.indexOf('--worktree'), path = i < 0 ? null : argv[i + 1];
|
|
34
|
+
if (!path || path.startsWith('--')) { console.error('usage: evidence-bundle.mjs --worktree <worktree.json>'); process.exitCode = 2; return; }
|
|
35
|
+
try { console.log(JSON.stringify(collectEvidence(JSON.parse(readFileSync(path, 'utf8'))))); }
|
|
36
|
+
catch { console.log(JSON.stringify(unavailableBundle('worktree metadata could not be read'))); process.exitCode = 2; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const DIMENSIONS = {
|
|
4
|
+
deps: ['deps'],
|
|
5
|
+
scan: ['triage-quality', 'triage-logic', 'project-rules'],
|
|
6
|
+
medium: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
7
|
+
deep: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
8
|
+
};
|
|
9
|
+
const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i;
|
|
10
|
+
const HASH = /^[a-f0-9]{64}$/i;
|
|
11
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
12
|
+
const count = value => Number.isSafeInteger(value) && value >= 0;
|
|
13
|
+
|
|
14
|
+
/** Validate a native completion receipt against run policy and exact findings bytes. */
|
|
15
|
+
export function validateNativeCompletion(receipt, expected, findingsBytes) {
|
|
16
|
+
const unavailable = reason => ({ available: false, reason });
|
|
17
|
+
if (!record(expected) || !record(expected.eligible) || !record(receipt)) return unavailable('receipt or expected inputs are malformed');
|
|
18
|
+
if (!['deps', 'scan', 'medium', 'deep'].includes(expected.depth) || typeof expected.worktreeAvailable !== 'boolean'
|
|
19
|
+
|| !count(expected.eligible.verify) || !count(expected.eligible.prove)) return unavailable('expected policy inputs are malformed');
|
|
20
|
+
if (typeof expected.runId !== 'string' || !expected.runId || typeof expected.headSha !== 'string' || !SHA.test(expected.headSha)) return unavailable('expected identity is malformed');
|
|
21
|
+
if (!(findingsBytes instanceof Uint8Array)) return unavailable('findings bytes are unavailable');
|
|
22
|
+
if (receipt.schema !== 1 || receipt.engine !== 'codex' || receipt.status !== 'complete') return unavailable('native completion is not complete');
|
|
23
|
+
if (receipt.runId !== expected.runId || receipt.depth !== expected.depth) return unavailable('run identity or depth mismatch');
|
|
24
|
+
if (typeof receipt.headSha !== 'string' || !SHA.test(receipt.headSha) || receipt.headSha.toLowerCase() !== expected.headSha.toLowerCase()) return unavailable('head SHA mismatch');
|
|
25
|
+
if (typeof receipt.findingsSha256 !== 'string' || !HASH.test(receipt.findingsSha256)
|
|
26
|
+
|| createHash('sha256').update(findingsBytes).digest('hex') !== receipt.findingsSha256.toLowerCase()) return unavailable('findings hash mismatch');
|
|
27
|
+
const required = DIMENSIONS[expected.depth];
|
|
28
|
+
if (!Array.isArray(receipt.dimensions) || receipt.dimensions.length !== required.length) return unavailable('required review dimensions are missing or duplicated');
|
|
29
|
+
const dimensions = Array.from(receipt.dimensions);
|
|
30
|
+
if (dimensions.some(d => !record(d) || !required.includes(d.name) || d.status !== 'completed')) return unavailable('review dimensions are missing, unknown, or incomplete');
|
|
31
|
+
if (new Set(dimensions.map(d => d.name)).size !== required.length) return unavailable('review dimensions are duplicated');
|
|
32
|
+
const verify = expected.depth === 'deps' ? 0 : expected.eligible.verify;
|
|
33
|
+
const prove = ['medium', 'deep'].includes(expected.depth) && expected.worktreeAvailable ? Math.min(5, expected.eligible.prove) : 0;
|
|
34
|
+
const validStage = (stage, scheduled) => record(stage) && count(stage.scheduled) && count(stage.completed)
|
|
35
|
+
&& count(stage.failed) && stage.scheduled === scheduled && stage.completed === scheduled && stage.failed === 0;
|
|
36
|
+
if (!record(receipt.stages) || !validStage(receipt.stages.verify, verify) || !validStage(receipt.stages.prove, prove)) return unavailable('verification or proof stage accounting is incomplete');
|
|
37
|
+
if (receipt.stages.critic !== (expected.depth === 'deep' ? 'completed' : 'not_required')) return unavailable('critic stage status does not match depth');
|
|
38
|
+
return { available: true, reason: null };
|
|
39
|
+
}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
3
|
//
|
|
4
4
|
// Usage:
|
|
5
|
-
// node eval-runner.mjs --manifest
|
|
5
|
+
// node eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]
|
|
6
6
|
//
|
|
7
7
|
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
8
|
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
22
22
|
|
|
23
23
|
import { readFileSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
24
25
|
import { fileURLToPath } from 'node:url';
|
|
25
26
|
|
|
26
27
|
export const LINE_WINDOW = 10;
|
|
@@ -74,6 +75,7 @@ export function validateManifest(manifest) {
|
|
|
74
75
|
return prs;
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
/** Normalize archived review evidence, returning findings only when every identity check passes. */
|
|
77
79
|
export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
78
80
|
const key = caseKey(entry);
|
|
79
81
|
const unscored = (reason) => ({ key, reason });
|
|
@@ -98,6 +100,7 @@ export function normalizeArchivedEvidence(entry, run, worktree, document) {
|
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
// Validate archive references before reading any run directories.
|
|
103
|
+
/** Reject duplicate archive run IDs before archive paths are read. */
|
|
101
104
|
export function validateArchiveRunIds(entries) {
|
|
102
105
|
const duplicate = entries.find((entry, index) => entry.runId
|
|
103
106
|
&& entries.slice(0, index).some(previous => previous.runId === entry.runId));
|
|
@@ -105,6 +108,30 @@ export function validateArchiveRunIds(entries) {
|
|
|
105
108
|
return entries;
|
|
106
109
|
}
|
|
107
110
|
|
|
111
|
+
function readArchiveJson(path) {
|
|
112
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Load valid archived findings and retain a reason for each rejected run. */
|
|
116
|
+
function archivedResults(entries, runsDir) {
|
|
117
|
+
const results = {};
|
|
118
|
+
const reasons = [];
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (typeof entry.runId !== 'string' || !RUN_ID.test(entry.runId)) {
|
|
121
|
+
reasons.push({ key: caseKey(entry), reason: 'unsafe or missing runId' });
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const dir = join(runsDir, entry.runId);
|
|
125
|
+
const evidence = normalizeArchivedEvidence(entry,
|
|
126
|
+
readArchiveJson(join(dir, 'run.json')),
|
|
127
|
+
readArchiveJson(join(dir, 'worktree.json')),
|
|
128
|
+
readArchiveJson(join(dir, 'findings.json')));
|
|
129
|
+
if (evidence.reason) reasons.push({ key: evidence.key, reason: evidence.reason });
|
|
130
|
+
else results[evidence.key] = evidence.findings;
|
|
131
|
+
}
|
|
132
|
+
return { results, reasons };
|
|
133
|
+
}
|
|
134
|
+
|
|
108
135
|
export function score(manifest, results) {
|
|
109
136
|
const rows = validateManifest(manifest).map((e) => {
|
|
110
137
|
const key = caseKey(e);
|
|
@@ -147,12 +174,24 @@ export function score(manifest, results) {
|
|
|
147
174
|
|
|
148
175
|
function main(argv) {
|
|
149
176
|
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
150
|
-
|
|
151
|
-
|
|
177
|
+
const results = flag('results');
|
|
178
|
+
const archiveIndex = argv.indexOf('--archive');
|
|
179
|
+
const archive = archiveIndex === -1 ? null : argv[archiveIndex + 1];
|
|
180
|
+
const resultsIndex = argv.indexOf('--results');
|
|
181
|
+
if (!flag('manifest') || (archiveIndex !== -1 && (!archive || archive.startsWith('--')))
|
|
182
|
+
|| (resultsIndex !== -1 && (!results || results.startsWith('--')))
|
|
183
|
+
|| Boolean(results) === Boolean(archive)) {
|
|
184
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> (--results <results.json> | --archive <runs-dir>) [--json]');
|
|
152
185
|
process.exit(2);
|
|
153
186
|
}
|
|
154
|
-
const
|
|
187
|
+
const manifest = JSON.parse(readFileSync(flag('manifest'), 'utf8'));
|
|
188
|
+
const source = archive
|
|
189
|
+
? archivedResults(validateArchiveRunIds(validateManifest(manifest)), archive)
|
|
190
|
+
: { results: JSON.parse(readFileSync(results, 'utf8')), reasons: null };
|
|
191
|
+
const out = score(manifest, source.results);
|
|
192
|
+
if (source.reasons) out.unscoredReasons = source.reasons;
|
|
155
193
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
194
|
+
for (const { key, reason } of source.reasons || []) console.error(`unscored ${key}: ${reason}`);
|
|
156
195
|
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
157
196
|
if (out.buggyRuns === undefined) {
|
|
158
197
|
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { collectGitEvidence } from './git-evidence.mjs';
|
|
5
|
+
import { validateLintReceipt } from './validate-lint-receipt.mjs';
|
|
6
|
+
|
|
7
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
const unavailableLint = reason => ({ captureStatus: 'unavailable', reason });
|
|
9
|
+
const unavailableBundle = reason => ({ schema: 1,
|
|
10
|
+
git: { schema: 1, status: 'unavailable', reason }, lint: unavailableLint(reason) });
|
|
11
|
+
|
|
12
|
+
export function collectEvidence(worktreeMetadata) {
|
|
13
|
+
if (!record(worktreeMetadata)) return unavailableBundle('worktree metadata is missing or malformed');
|
|
14
|
+
const git = collectGitEvidence({ worktreePath: worktreeMetadata.worktreePath,
|
|
15
|
+
mergeBase: worktreeMetadata.mergeBase, headSha: worktreeMetadata.headSha });
|
|
16
|
+
if (git.status !== 'available') return { schema: 1, git, lint: unavailableLint(`Git evidence unavailable: ${git.reason}`) };
|
|
17
|
+
if (typeof worktreeMetadata.eslintJson !== 'string' || !worktreeMetadata.eslintJson) return { schema: 1, git, lint: unavailableLint('lint report path unavailable') };
|
|
18
|
+
if (typeof worktreeMetadata.eslintProvenance !== 'string' || !worktreeMetadata.eslintProvenance) return { schema: 1, git, lint: unavailableLint('lint provenance path unavailable') };
|
|
19
|
+
let reportBytes, receipt;
|
|
20
|
+
try { reportBytes = readFileSync(worktreeMetadata.eslintJson); } catch { return { schema: 1, git, lint: unavailableLint('lint report could not be read') }; }
|
|
21
|
+
try { receipt = JSON.parse(readFileSync(worktreeMetadata.eslintProvenance, 'utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint provenance is missing or malformed') }; }
|
|
22
|
+
const provenance = validateLintReceipt(receipt, { baseSha: git.provenance.baseSha,
|
|
23
|
+
headSha: git.provenance.headSha, trackedClean: git.provenance.trackedCleanBefore && git.provenance.trackedCleanAfter }, reportBytes);
|
|
24
|
+
if (provenance.status !== 'available') return { schema: 1, git, lint: unavailableLint(provenance.reason) };
|
|
25
|
+
let report;
|
|
26
|
+
try { report = JSON.parse(reportBytes.toString('utf8')); } catch { return { schema: 1, git, lint: unavailableLint('lint report is malformed JSON') }; }
|
|
27
|
+
if (!Array.isArray(report)) return { schema: 1, git, lint: unavailableLint('lint report is not a JSON array') };
|
|
28
|
+
return { schema: 1, git, lint: { captureStatus: 'available', exitCode: provenance.exitCode,
|
|
29
|
+
reportSha256: provenance.reportSha256, report } };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function main(argv) {
|
|
33
|
+
const i = argv.indexOf('--worktree'), path = i < 0 ? null : argv[i + 1];
|
|
34
|
+
if (!path || path.startsWith('--')) { console.error('usage: evidence-bundle.mjs --worktree <worktree.json>'); process.exitCode = 2; return; }
|
|
35
|
+
try { console.log(JSON.stringify(collectEvidence(JSON.parse(readFileSync(path, 'utf8'))))); }
|
|
36
|
+
catch { console.log(JSON.stringify(unavailableBundle('worktree metadata could not be read'))); process.exitCode = 2; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const DIMENSIONS = {
|
|
4
|
+
deps: ['deps'],
|
|
5
|
+
scan: ['triage-quality', 'triage-logic', 'project-rules'],
|
|
6
|
+
medium: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
7
|
+
deep: ['quality', 'implementation', 'simplification', 'conventions', 'test-quality', 'project-rules'],
|
|
8
|
+
};
|
|
9
|
+
const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/i;
|
|
10
|
+
const HASH = /^[a-f0-9]{64}$/i;
|
|
11
|
+
const record = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
12
|
+
const count = value => Number.isSafeInteger(value) && value >= 0;
|
|
13
|
+
|
|
14
|
+
/** Validate a native completion receipt against run policy and exact findings bytes. */
|
|
15
|
+
export function validateNativeCompletion(receipt, expected, findingsBytes) {
|
|
16
|
+
const unavailable = reason => ({ available: false, reason });
|
|
17
|
+
if (!record(expected) || !record(expected.eligible) || !record(receipt)) return unavailable('receipt or expected inputs are malformed');
|
|
18
|
+
if (!['deps', 'scan', 'medium', 'deep'].includes(expected.depth) || typeof expected.worktreeAvailable !== 'boolean'
|
|
19
|
+
|| !count(expected.eligible.verify) || !count(expected.eligible.prove)) return unavailable('expected policy inputs are malformed');
|
|
20
|
+
if (typeof expected.runId !== 'string' || !expected.runId || typeof expected.headSha !== 'string' || !SHA.test(expected.headSha)) return unavailable('expected identity is malformed');
|
|
21
|
+
if (!(findingsBytes instanceof Uint8Array)) return unavailable('findings bytes are unavailable');
|
|
22
|
+
if (receipt.schema !== 1 || receipt.engine !== 'codex' || receipt.status !== 'complete') return unavailable('native completion is not complete');
|
|
23
|
+
if (receipt.runId !== expected.runId || receipt.depth !== expected.depth) return unavailable('run identity or depth mismatch');
|
|
24
|
+
if (typeof receipt.headSha !== 'string' || !SHA.test(receipt.headSha) || receipt.headSha.toLowerCase() !== expected.headSha.toLowerCase()) return unavailable('head SHA mismatch');
|
|
25
|
+
if (typeof receipt.findingsSha256 !== 'string' || !HASH.test(receipt.findingsSha256)
|
|
26
|
+
|| createHash('sha256').update(findingsBytes).digest('hex') !== receipt.findingsSha256.toLowerCase()) return unavailable('findings hash mismatch');
|
|
27
|
+
const required = DIMENSIONS[expected.depth];
|
|
28
|
+
if (!Array.isArray(receipt.dimensions) || receipt.dimensions.length !== required.length) return unavailable('required review dimensions are missing or duplicated');
|
|
29
|
+
const dimensions = Array.from(receipt.dimensions);
|
|
30
|
+
if (dimensions.some(d => !record(d) || !required.includes(d.name) || d.status !== 'completed')) return unavailable('review dimensions are missing, unknown, or incomplete');
|
|
31
|
+
if (new Set(dimensions.map(d => d.name)).size !== required.length) return unavailable('review dimensions are duplicated');
|
|
32
|
+
const verify = expected.depth === 'deps' ? 0 : expected.eligible.verify;
|
|
33
|
+
const prove = ['medium', 'deep'].includes(expected.depth) && expected.worktreeAvailable ? Math.min(5, expected.eligible.prove) : 0;
|
|
34
|
+
const validStage = (stage, scheduled) => record(stage) && count(stage.scheduled) && count(stage.completed)
|
|
35
|
+
&& count(stage.failed) && stage.scheduled === scheduled && stage.completed === scheduled && stage.failed === 0;
|
|
36
|
+
if (!record(receipt.stages) || !validStage(receipt.stages.verify, verify) || !validStage(receipt.stages.prove, prove)) return unavailable('verification or proof stage accounting is incomplete');
|
|
37
|
+
if (receipt.stages.critic !== (expected.depth === 'deep' ? 'completed' : 'not_required')) return unavailable('critic stage status does not match depth');
|
|
38
|
+
return { available: true, reason: null };
|
|
39
|
+
}
|