@optitransfer/proofline 0.0.0-stage → 0.2.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +32 -0
- package/CHANGELOG.md +56 -0
- package/CONTRIBUTING.md +29 -0
- package/LICENSE +154 -0
- package/NOTICE +10 -0
- package/PATENTS-NONASSERT.md +1 -0
- package/README.md +285 -2
- package/bin/proofline.mjs +292 -0
- package/docs/00-index.md +43 -0
- package/docs/01-scope.md +70 -0
- package/docs/02-requirements.md +64 -0
- package/docs/03-domain-model.md +43 -0
- package/docs/04-architecture.md +79 -0
- package/docs/05-lifecycle.md +47 -0
- package/docs/06-verification.md +57 -0
- package/docs/07-evidence.md +69 -0
- package/docs/08-adapters.md +61 -0
- package/docs/09-security.md +64 -0
- package/docs/10-conformance.md +59 -0
- package/docs/11-development-pipeline.md +50 -0
- package/docs/12-roadmap.md +104 -0
- package/docs/13-interdependencies.md +53 -0
- package/docs/14-metrics.md +40 -0
- package/docs/adr/0001-product-boundary.md +28 -0
- package/docs/adr/0002-evidence-and-gates.md +22 -0
- package/docs/adr/0003-standalone-first.md +18 -0
- package/docs/receipt.md +25 -0
- package/package.json +60 -4
- package/src/cache.mjs +81 -0
- package/src/checks.mjs +276 -0
- package/src/claims.mjs +60 -0
- package/src/contract.mjs +404 -0
- package/src/engine.mjs +378 -0
- package/src/evidence.mjs +45 -0
- package/src/forbidden.mjs +57 -0
- package/src/guided.mjs +240 -0
- package/src/hook-stop.mjs +105 -0
- package/src/index.mjs +431 -0
- package/src/infer.mjs +173 -0
- package/src/inference.json +20 -0
- package/src/isolate.mjs +123 -0
- package/src/mcp.mjs +70 -0
- package/src/receipt.mjs +17 -0
- package/src/signature.mjs +110 -0
- package/src/util.mjs +92 -0
- package/src/vault.mjs +109 -0
- package/src/verify-receipt.mjs +109 -0
package/package.json
CHANGED
|
@@ -1,6 +1,62 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@optitransfer/proofline",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0-next.1",
|
|
4
|
+
"description": "Independent V&V for coding agents. The agent says done; Proofline re-runs every check against a contract, rejects claims the evidence does not support, and gates the push.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ai-agents",
|
|
7
|
+
"llm",
|
|
8
|
+
"mcp",
|
|
9
|
+
"mcp-server",
|
|
10
|
+
"agentic-ai",
|
|
11
|
+
"claude-code",
|
|
12
|
+
"verification",
|
|
13
|
+
"validation",
|
|
14
|
+
"testing",
|
|
15
|
+
"test-automation",
|
|
16
|
+
"quality-gate",
|
|
17
|
+
"ci-cd",
|
|
18
|
+
"devops",
|
|
19
|
+
"code-review",
|
|
20
|
+
"git-hooks",
|
|
21
|
+
"audit",
|
|
22
|
+
"attestation",
|
|
23
|
+
"developer-tools",
|
|
24
|
+
"nodejs",
|
|
25
|
+
"cli"
|
|
26
|
+
],
|
|
27
|
+
"homepage": "https://github.com/mgillr/proofline",
|
|
28
|
+
"repository": {
|
|
29
|
+
"type": "git",
|
|
30
|
+
"url": "git+https://github.com/mgillr/proofline.git"
|
|
31
|
+
},
|
|
32
|
+
"bugs": {
|
|
33
|
+
"url": "https://github.com/mgillr/proofline/issues"
|
|
34
|
+
},
|
|
35
|
+
"author": "mgillr",
|
|
36
|
+
"license": "Apache-2.0",
|
|
37
|
+
"private": false,
|
|
38
|
+
"type": "module",
|
|
39
|
+
"bin": { "proofline": "bin/proofline.mjs" },
|
|
40
|
+
"engines": { "node": ">=22" },
|
|
41
|
+
"files": [
|
|
42
|
+
"bin",
|
|
43
|
+
"src",
|
|
44
|
+
"docs",
|
|
45
|
+
"AGENTS.md",
|
|
46
|
+
"README.md",
|
|
47
|
+
"LICENSE",
|
|
48
|
+
"NOTICE",
|
|
49
|
+
"CHANGELOG.md",
|
|
50
|
+
"CONTRIBUTING.md",
|
|
51
|
+
"PATENTS-NONASSERT.md"
|
|
52
|
+
],
|
|
53
|
+
"scripts": {
|
|
54
|
+
"test": "node --test test/*.test.mjs",
|
|
55
|
+
"demo": "node examples/demo/demo.mjs",
|
|
56
|
+
"self-gate": "node bin/proofline.mjs gate PL-SELF --skip-pushed",
|
|
57
|
+
"mcp": "node bin/proofline.mjs mcp"
|
|
58
|
+
},
|
|
59
|
+
"devDependencies": {
|
|
60
|
+
"hivevault": "0.3.3"
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/cache.mjs
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Result cache: a PASS of an expensive check (command, metric) on a clean tree runs once per HEAD and is reused.
|
|
2
|
+
// Opt-in per contract (defaults.reuse_results). The PASS is recorded in the evidence log; the cache file points at
|
|
3
|
+
// that entry, and a result is reused only from an entry that is still in an intact chain with the same hash.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { git, sha256, canon, readJson } from './util.mjs';
|
|
7
|
+
import { record, readLog, verifyChain } from './evidence.mjs';
|
|
8
|
+
|
|
9
|
+
/** The only verifiers whose PASS is reused. The others are cheap, or depend on the base. */
|
|
10
|
+
export const REUSABLE = ['command', 'metric'];
|
|
11
|
+
export const DEFAULT_MAX_AGE_H = 24;
|
|
12
|
+
export const cacheDir = (root) => path.join(root, '.proofline', 'cache');
|
|
13
|
+
const cacheFile = (root, key) => path.join(cacheDir(root), `${key}.json`);
|
|
14
|
+
|
|
15
|
+
/** A contract's reuse settings: on only when defaults.reuse_results is true. */
|
|
16
|
+
export function settings(c) {
|
|
17
|
+
const d = c.defaults || {};
|
|
18
|
+
const age = d.reuse_max_age_h;
|
|
19
|
+
return { on: d.reuse_results === true, maxAgeH: typeof age === 'number' && age > 0 ? age : DEFAULT_MAX_AGE_H, maxLoad: typeof d.max_load_per_cpu === 'number' ? d.max_load_per_cpu : undefined };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** The env vars a check names: `$VAR` and `${VAR}` in its command, `{VAR}` in a metric's output_file. */
|
|
23
|
+
export function namedEnv(v) {
|
|
24
|
+
const names = new Set();
|
|
25
|
+
for (const m of String(v.cmd ?? '').matchAll(/\$\{?([A-Za-z_]\w*)/g)) names.add(m[1]);
|
|
26
|
+
for (const m of String(v.output_file ?? '').matchAll(/\{(\w+)\}/g)) names.add(m[1]);
|
|
27
|
+
return [...names].sort();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The cache key of a criterion's verifier, or null when its result is neither reused nor stored: not a reusable
|
|
32
|
+
* verifier, no HEAD, or a tree that differs from HEAD outside .proofline/ (an uncommitted edit never reuses a result).
|
|
33
|
+
* The key holds the verifier (canonical JSON), HEAD, process.version and the values of the env vars the check names;
|
|
34
|
+
* for a metric also its baseline and the regression tolerance, which decide its verdict.
|
|
35
|
+
*/
|
|
36
|
+
export function keyFor(root, v, { baseline, tolerancePct } = {}) {
|
|
37
|
+
if (!REUSABLE.includes(v.type)) return null;
|
|
38
|
+
const head = git(['rev-parse', '--verify', '--quiet', 'HEAD'], root);
|
|
39
|
+
if (!head.ok || !head.out) return null;
|
|
40
|
+
// --untracked-files=normal: an untracked file counts even where status.showUntrackedFiles=no hides it.
|
|
41
|
+
const st = git(['status', '--porcelain', '--untracked-files=normal'], root);
|
|
42
|
+
if (!st.ok || st.out.split('\n').some((l) => l && !l.includes('.proofline/'))) return null;
|
|
43
|
+
const env = Object.fromEntries(namedEnv(v).map((n) => [n, v.env?.[n] ?? process.env[n] ?? null]));
|
|
44
|
+
const k = { verifier: v, head: head.out, node: process.version, env };
|
|
45
|
+
if (v.type === 'metric') Object.assign(k, { baseline: baseline?.[v.name || v.key] ?? null, tolerance_pct: tolerancePct ?? 20 });
|
|
46
|
+
return { key: sha256(canon(k)), head: head.out };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The evidence entry of a PASS that may be reused for `key`, or null (then the check runs). The cache file must name
|
|
51
|
+
* an entry that is in an intact chain with the hash it stored, records a PASS for this key, is no older than
|
|
52
|
+
* maxAgeH hours, and, for a metric, ran at a load per CPU within maxLoad when that is set.
|
|
53
|
+
*/
|
|
54
|
+
export function lookup(root, key, { maxAgeH = DEFAULT_MAX_AGE_H, maxLoad, now = Date.now() } = {}) {
|
|
55
|
+
const c = readJson(cacheFile(root, key));
|
|
56
|
+
if (!c || c.key !== key || c.status !== 'pass' || !Number.isInteger(c.entry)) return null;
|
|
57
|
+
let e;
|
|
58
|
+
try { if (!verifyChain(root).ok) return null; e = readLog(root)[c.entry]; } catch { return null; }
|
|
59
|
+
if (!e || e.hash !== c.entry_hash || e.type !== 'result' || e.data?.key !== key || e.data.status !== 'pass') return null;
|
|
60
|
+
const age = now - Date.parse(e.ts);
|
|
61
|
+
if (!(age >= 0 && age <= maxAgeH * 3600e3)) return null;
|
|
62
|
+
if (e.data.type === 'metric' && typeof maxLoad === 'number' && !(e.data.evidence?.machine?.load_per_cpu <= maxLoad)) return null;
|
|
63
|
+
return e;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Record a fresh PASS in the evidence log and point the cache file for its key at that entry. Returns the entry. */
|
|
67
|
+
export function store(root, { key, head }, task, a, r) {
|
|
68
|
+
const data = { task, criterion: a.id, type: a.verifier.type, key, head, status: r.status, detail: r.detail, evidence: r.evidence };
|
|
69
|
+
const e = record(root, 'result', data);
|
|
70
|
+
const f = cacheFile(root, key), tmp = `${f}.${process.pid}.tmp`;
|
|
71
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
72
|
+
fs.writeFileSync(tmp, JSON.stringify({ ...data, ts: e.ts, entry: e.seq, entry_hash: e.hash }, null, 2) + '\n');
|
|
73
|
+
fs.renameSync(tmp, f);
|
|
74
|
+
return e;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A criterion's result taken from the evidence entry that recorded its PASS. */
|
|
78
|
+
export function reusedResult(e) {
|
|
79
|
+
const from = `reused from ${e.ts} (evidence entry ${e.seq})`;
|
|
80
|
+
return { status: 'pass', detail: `${from}: ${e.data.detail}`, evidence: { ...e.data.evidence, reused_from: { ts: e.ts, entry: e.seq, hash: e.hash, head: e.data.head } }, reused: { ts: e.ts, entry: e.seq } };
|
|
81
|
+
}
|
package/src/checks.mjs
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
// Deterministic verifiers. Each returns { status: pass|fail|flaky|error|blocked, detail, evidence }.
|
|
2
|
+
// "error" means the check could not be run (missing tool/file/base) and leads to BLOCKED, never PASS.
|
|
3
|
+
// "blocked" means it must not be run now (a metric on a busy machine) and also leads to BLOCKED.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { run, git, tail, pick, lastJson, matchAny, onRemote } from './util.mjs';
|
|
8
|
+
|
|
9
|
+
/** The machine a metric runs on: 1-minute load average and CPU count. verifyTask takes a replacement as `machine`. */
|
|
10
|
+
export const readMachine = () => ({ load1: os.loadavg()[0], cpus: os.cpus().length });
|
|
11
|
+
|
|
12
|
+
/** `node --version` as the bench command would resolve it (same cwd and env), or null. */
|
|
13
|
+
function nodeVersion(root, env) {
|
|
14
|
+
const r = run('node --version', { cwd: root, timeoutS: 30, env });
|
|
15
|
+
return r.exit === 0 ? r.stdout.trim() : null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export const TEST_FILE_RE = /(^|\/)(test|tests|__tests__)\/|\.(test|spec)\.[mc]?[jt]sx?$/;
|
|
19
|
+
const sub = (s, vars) => s.replace(/\{(\w+)\}/g, (m, k) => (vars[k] ?? m));
|
|
20
|
+
|
|
21
|
+
export function resolveBase(root, ref) {
|
|
22
|
+
const r = git(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], root);
|
|
23
|
+
return r.ok ? r.out : null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Changed files between base and the working tree (tracked + untracked). */
|
|
27
|
+
export function changedFiles(root, base) {
|
|
28
|
+
const out = new Map();
|
|
29
|
+
const d = git(['diff', '--name-status', '-M', base, '--'], root);
|
|
30
|
+
for (const l of d.out.split('\n').filter(Boolean)) {
|
|
31
|
+
const [st, a, b] = l.split('\t');
|
|
32
|
+
out.set(b || a, st[0]);
|
|
33
|
+
}
|
|
34
|
+
for (const f of git(['ls-files', '--others', '--exclude-standard'], root).out.split('\n').filter(Boolean)) out.set(f, 'A');
|
|
35
|
+
for (const f of [...out.keys()]) if (f.startsWith('.proofline/')) out.delete(f);
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Parse a unified diff (-U0) into per-file hunks of added/removed lines. */
|
|
40
|
+
export function diffHunks(root, base) {
|
|
41
|
+
const d = git(['diff', '-U0', base, '--'], root).out;
|
|
42
|
+
const files = [];
|
|
43
|
+
let cur = null, hunk = null;
|
|
44
|
+
for (const l of d.split('\n')) {
|
|
45
|
+
if (l.startsWith('diff --git')) { cur = { file: l.split(' b/').pop(), hunks: [] }; files.push(cur); continue; }
|
|
46
|
+
if (!cur) continue;
|
|
47
|
+
if (l.startsWith('+++') || l.startsWith('---')) continue;
|
|
48
|
+
if (l.startsWith('@@')) { hunk = { add: [], del: [] }; cur.hunks.push(hunk); continue; }
|
|
49
|
+
if (!hunk) continue;
|
|
50
|
+
if (l[0] === '+') hunk.add.push(l.slice(1)); else if (l[0] === '-') hunk.del.push(l.slice(1));
|
|
51
|
+
}
|
|
52
|
+
// untracked files count as fully added
|
|
53
|
+
for (const f of git(['ls-files', '--others', '--exclude-standard'], root).out.split('\n').filter(Boolean)) {
|
|
54
|
+
if (f.startsWith('.proofline/')) continue;
|
|
55
|
+
let txt = ''; try { txt = fs.readFileSync(path.join(root, f), 'utf8'); } catch {}
|
|
56
|
+
files.push({ file: f, hunks: [{ add: txt.split('\n'), del: [] }] });
|
|
57
|
+
}
|
|
58
|
+
return files;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function runCounted(cmd, root, v, ctx) {
|
|
62
|
+
const r = ctx?.sandbox
|
|
63
|
+
? ctx.sandbox(cmd, { cwd: root, timeoutS: v.timeout_s || 900, env: v.env, writable: v.writable })
|
|
64
|
+
: run(cmd, { cwd: root, timeoutS: v.timeout_s || 900, env: v.env });
|
|
65
|
+
const txt = r.stdout + '\n' + r.stderr;
|
|
66
|
+
const num = (k) => { const m = txt.match(new RegExp(`^[#ℹ]\\s*${k}\\s+(\\d+)`, 'm')); return m ? +m[1] : undefined; };
|
|
67
|
+
r.counts = { tests: num('tests'), pass: num('pass'), fail: num('fail'), skipped: num('skipped'), todo: num('todo'), cancelled: num('cancelled') };
|
|
68
|
+
return r;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export const FAILING_CAP = 100;
|
|
72
|
+
const ANSI_RE = /\x1b\[[0-9;]*m/g;
|
|
73
|
+
// TAP: `not ok <n> - <name>`, subtests indented. node writes a `#` inside a name as `\#`, so an unescaped ` #` starts the directive.
|
|
74
|
+
const TAP_FAIL_RE = /^\s*not ok\s+\d+(?:\s+-)?(?:\s+(.*))?$/;
|
|
75
|
+
const TAP_DIRECTIVE_RE = /(?:^|\s)#\s*(\w*)/;
|
|
76
|
+
// spec and dot reporters: `✖ <name> (<ms>ms)`. The duration comes last, except for a todo or skip, which node marks with
|
|
77
|
+
// ` # <reason>` (or `# TODO`, `# SKIP`) after it. The `failing tests:` and `Failed tests:` summaries repeat these lines.
|
|
78
|
+
const SPEC_FAIL_RE = /^\s*\u2716\s+(.*)\s\(\d+(?:\.\d+)?ms\)(\s+#.*)?$/;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Names of the failed tests anywhere in a run's output, in order of first appearance, without repeats, todos or skips.
|
|
82
|
+
* At most `cap` names; `failing_total` is set only when there were more.
|
|
83
|
+
*/
|
|
84
|
+
export function failingTests(text, cap = FAILING_CAP) {
|
|
85
|
+
const seen = new Set();
|
|
86
|
+
for (const line of String(text).replace(ANSI_RE, '').split(/\r?\n/)) {
|
|
87
|
+
let m = line.match(TAP_FAIL_RE), name;
|
|
88
|
+
if (m) {
|
|
89
|
+
const rest = m[1] || '', d = rest.match(TAP_DIRECTIVE_RE);
|
|
90
|
+
if (!d || !/^(todo|skip)/i.test(d[1])) name = (d ? rest.slice(0, d.index) : rest).replace(/\\([\\#])/g, '$1');
|
|
91
|
+
} else if ((m = line.match(SPEC_FAIL_RE)) && !m[2]) name = m[1];
|
|
92
|
+
if (name && name.trim()) seen.add(name.trim());
|
|
93
|
+
}
|
|
94
|
+
const all = [...seen];
|
|
95
|
+
return all.length > cap ? { failing: all.slice(0, cap), failing_total: all.length } : { failing: all };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const ev = (r) => ({ cmd: r.cmd, exit: r.exit, ms: r.ms, counts: r.counts, timedOut: r.timedOut, ...failingTests(r.stdout + '\n' + r.stderr), out_tail: tail(r.stdout + (r.stderr ? '\n[stderr]\n' + r.stderr : ''), 1200) });
|
|
99
|
+
/** "12 tests failed: a, b, ... and 2 more" from a run's evidence (first 10 names), or '' when it names none. */
|
|
100
|
+
const named = (e) => {
|
|
101
|
+
const n = e.failing_total ?? e.failing.length, show = e.failing.slice(0, 10);
|
|
102
|
+
return n ? `${n} test${n === 1 ? '' : 's'} failed: ${show.join(', ')}${n > show.length ? ` and ${n - show.length} more` : ''}` : '';
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export const verifiers = {
|
|
106
|
+
command(v, ctx) {
|
|
107
|
+
const expect = v.expect_exit ?? 0;
|
|
108
|
+
let r = runCounted(v.cmd, ctx.root, v, ctx);
|
|
109
|
+
const first = ev(r), firstExit = r.exit;
|
|
110
|
+
if (r.exit === 127) return { status: 'error', detail: `command not found: ${v.cmd}`, evidence: first };
|
|
111
|
+
const tooMany = (r2) => {
|
|
112
|
+
const c = r2.counts; const p = [];
|
|
113
|
+
if ((c.cancelled || 0) > 0) p.push(`${c.cancelled} cancelled/hung`);
|
|
114
|
+
if (v.max_skipped !== undefined && (c.skipped || 0) + (c.todo || 0) > v.max_skipped) p.push(`${(c.skipped || 0) + (c.todo || 0)} skipped/todo > allowed ${v.max_skipped}`);
|
|
115
|
+
if (v.min_tests !== undefined && (c.tests ?? c.pass ?? 0) < v.min_tests) p.push(`only ${c.tests ?? c.pass ?? 0} tests ran, contract requires ≥ ${v.min_tests} (tests deleted or not discovered?)`);
|
|
116
|
+
return p;
|
|
117
|
+
};
|
|
118
|
+
if (r.exit === expect && !tooMany(r).length) return { status: 'pass', detail: `exit ${r.exit} in ${r.ms} ms`, evidence: first };
|
|
119
|
+
if (r.exit === expect) return { status: 'fail', detail: [...tooMany(r), named(first)].filter(Boolean).join('; '), evidence: first };
|
|
120
|
+
// Both runs keep their failing names: on a flaky result the first run's list is the tests that flapped.
|
|
121
|
+
for (let i = 0; i < (ctx.flakyReruns ?? 1); i++) {
|
|
122
|
+
r = runCounted(v.cmd, ctx.root, v, ctx);
|
|
123
|
+
if (r.exit === expect && !tooMany(r).length) return { status: 'flaky', detail: `failed (exit ${firstExit}${named(first) ? `; ${named(first)}` : ''}) then passed on re-run ${i + 1}: flaky, not a pass. Root-cause the non-determinism.`, evidence: { first, rerun: ev(r) } };
|
|
124
|
+
}
|
|
125
|
+
const rerun = ev(r), n = named(rerun.failing.length ? rerun : first);
|
|
126
|
+
return { status: 'fail', detail: `exit ${r.exit} (expected ${expect})${r.timedOut ? ', timed out' : ''}; failed again on re-run, so it is real${n ? `; ${n}` : ''}`, evidence: { first, rerun } };
|
|
127
|
+
},
|
|
128
|
+
|
|
129
|
+
file_exists(v, ctx) {
|
|
130
|
+
return fs.existsSync(path.join(ctx.root, v.path)) ? { status: 'pass', detail: `${v.path} exists` } : { status: 'fail', detail: `${v.path} does not exist in the repo` };
|
|
131
|
+
},
|
|
132
|
+
|
|
133
|
+
test_exists(v, ctx) {
|
|
134
|
+
const f = path.join(ctx.root, v.test_file);
|
|
135
|
+
if (!fs.existsSync(f)) return { status: 'fail', detail: `test file ${v.test_file} not found` };
|
|
136
|
+
if (v.test_name && !fs.readFileSync(f, 'utf8').includes(v.test_name)) return { status: 'fail', detail: `test "${v.test_name}" not found in ${v.test_file}` };
|
|
137
|
+
return { status: 'pass', detail: 'test present' };
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
/** The test must exist now, FAIL at the base revision (with the new test copied in), and PASS now. */
|
|
141
|
+
test_red_green(v, ctx) {
|
|
142
|
+
const exists = verifiers.test_exists(v, ctx);
|
|
143
|
+
if (exists.status !== 'pass') return exists;
|
|
144
|
+
if (!ctx.base) return { status: 'error', detail: 'base revision unresolved; cannot prove red-green' };
|
|
145
|
+
const wt = fs.mkdtempSync(path.join(os.tmpdir(), 'proofline-base-'));
|
|
146
|
+
try {
|
|
147
|
+
const add = git(['worktree', 'add', '--detach', '--force', wt, ctx.base], ctx.root);
|
|
148
|
+
if (!add.ok) return { status: 'error', detail: `git worktree add failed: ${add.err}` };
|
|
149
|
+
for (const f of [v.test_file, ...(v.support_files || [])]) {
|
|
150
|
+
fs.mkdirSync(path.dirname(path.join(wt, f)), { recursive: true });
|
|
151
|
+
fs.copyFileSync(path.join(ctx.root, f), path.join(wt, f));
|
|
152
|
+
}
|
|
153
|
+
if (v.setup) run(v.setup, { cwd: wt, timeoutS: 900 });
|
|
154
|
+
const cmd = sub(v.cmd, { test: v.test_file, name: v.test_name || '' });
|
|
155
|
+
const red = runCounted(cmd, wt, v, ctx);
|
|
156
|
+
const green = runCounted(cmd, ctx.root, v, ctx);
|
|
157
|
+
const evd = { base: ctx.base, red: ev(red), green: ev(green) };
|
|
158
|
+
if (red.exit === 127 || green.exit === 127) return { status: 'error', detail: 'test command not found', evidence: evd };
|
|
159
|
+
if (red.exit === 0) return { status: 'fail', detail: `test PASSES on base ${ctx.base.slice(0, 8)} without the change: it does not prove the change (tests how, not what?)`, evidence: evd };
|
|
160
|
+
if (green.exit !== 0) return { status: 'fail', detail: `test fails on the current tree (exit ${green.exit})`, evidence: evd };
|
|
161
|
+
if (v.test_name && !(green.stdout + green.stderr).includes(v.test_name)) return { status: 'fail', detail: `"${v.test_name}" did not appear in the run output: it was not executed`, evidence: evd };
|
|
162
|
+
return { status: 'pass', detail: `red on base (exit ${red.exit}), green now`, evidence: evd };
|
|
163
|
+
} finally {
|
|
164
|
+
git(['worktree', 'remove', '--force', wt], ctx.root);
|
|
165
|
+
fs.rmSync(wt, { recursive: true, force: true });
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
|
|
169
|
+
metric(v, ctx) {
|
|
170
|
+
// One bench run serves every metric criterion that uses the same command (benches are slow).
|
|
171
|
+
// The load is read before the run: a number taken on a busy machine is not evidence either way.
|
|
172
|
+
ctx.cmdCache ??= {};
|
|
173
|
+
const ck = v.cmd + JSON.stringify(v.env || {});
|
|
174
|
+
const max = ctx.contract?.defaults?.max_load_per_cpu;
|
|
175
|
+
const { r, machine } = ctx.cmdCache[ck] ??= (() => {
|
|
176
|
+
const m = (ctx.machine || readMachine)();
|
|
177
|
+
const machine = { load1: m.load1, cpus: m.cpus, load_per_cpu: +(m.load1 / Math.max(1, m.cpus)).toFixed(2), node: nodeVersion(ctx.root, v.env) };
|
|
178
|
+
if (typeof max === 'number' && m.load1 / Math.max(1, m.cpus) > max) return { r: null, machine };
|
|
179
|
+
const exec = ctx.sandbox
|
|
180
|
+
? (cmd, o) => ctx.sandbox(cmd, { ...o, writable: v.writable })
|
|
181
|
+
: (cmd, o) => run(cmd, o);
|
|
182
|
+
return { r: exec(v.cmd, { cwd: ctx.root, timeoutS: v.timeout_s || 1800, env: v.env }), machine };
|
|
183
|
+
})();
|
|
184
|
+
const on = `load ${machine.load1.toFixed(2)} / ${machine.cpus} CPUs, node ${machine.node || 'not found'}`;
|
|
185
|
+
if (!r) return { status: 'blocked', detail: `machine busy: load ${machine.load1.toFixed(2)} over ${machine.cpus} CPUs = ${machine.load_per_cpu.toFixed(2)} per CPU > max_load_per_cpu ${max}; the bench was not run. Re-run when the machine is idle.`, evidence: { cmd: v.cmd, machine } };
|
|
186
|
+
let data;
|
|
187
|
+
if (v.output_file) { try { data = JSON.parse(fs.readFileSync(path.resolve(ctx.root, sub(v.output_file, process.env)), 'utf8')); } catch {} }
|
|
188
|
+
else data = lastJson(r.stdout);
|
|
189
|
+
const e = { cmd: v.cmd, exit: r.exit, ms: r.ms, machine, out_tail: tail(r.stdout + r.stderr, 800) };
|
|
190
|
+
if (r.exit === 127) return { status: 'error', detail: `bench command not found: ${v.cmd} (is the bench rig committed in the repo?)`, evidence: e };
|
|
191
|
+
if (data === undefined) return { status: 'error', detail: `no JSON output from bench (exit ${r.exit})`, evidence: e };
|
|
192
|
+
const val = pick(data, v.key);
|
|
193
|
+
e.key = v.key; e.measured = val; e.threshold = `${v.op} ${v.value}`;
|
|
194
|
+
if (typeof val !== 'number') return { status: 'error', detail: `metric ${v.key} missing from bench output`, evidence: e };
|
|
195
|
+
const ok = { '<': val < v.value, '<=': val <= v.value, '>': val > v.value, '>=': val >= v.value, '==': val === v.value }[v.op];
|
|
196
|
+
const name = v.name || v.key;
|
|
197
|
+
ctx.measured[name] = val;
|
|
198
|
+
const b = ctx.baseline?.[name];
|
|
199
|
+
let reg = '';
|
|
200
|
+
if (typeof b === 'number' && b !== 0) {
|
|
201
|
+
const tol = (ctx.tolerancePct ?? 20) / 100;
|
|
202
|
+
const worse = v.op.startsWith('<') ? val > b * (1 + tol) : v.op.startsWith('>') ? val < b * (1 - tol) : false;
|
|
203
|
+
e.baseline = b;
|
|
204
|
+
if (worse) reg = ` REGRESSION vs baseline ${b} (> ${ctx.tolerancePct ?? 20}%)`;
|
|
205
|
+
}
|
|
206
|
+
if (!ok) return { status: 'fail', detail: `${name} = ${val}, needs ${v.op} ${v.value}${reg} (${on})`, evidence: e };
|
|
207
|
+
if (reg) return { status: 'fail', detail: `${v.key} = ${val} meets threshold but${reg} (${on})`, evidence: e };
|
|
208
|
+
return { status: 'pass', detail: `${v.key} = ${val} ${v.op} ${v.value} (${on})`, evidence: e };
|
|
209
|
+
},
|
|
210
|
+
|
|
211
|
+
grep(v, ctx) {
|
|
212
|
+
const f = path.join(ctx.root, v.file);
|
|
213
|
+
if (!fs.existsSync(f)) return { status: 'fail', detail: `${v.file} not found` };
|
|
214
|
+
const txt = fs.readFileSync(f, 'utf8');
|
|
215
|
+
const re = new RegExp(v.pattern, v.flags || 'm');
|
|
216
|
+
const m = txt.match(re);
|
|
217
|
+
const want = (v.must || 'present') === 'present';
|
|
218
|
+
if (!!m === want) return { status: 'pass', detail: `/${v.pattern}/ ${want ? 'present' : 'absent'} in ${v.file}`, evidence: m ? { line: txt.slice(0, m.index).split('\n').length, text: m[0] } : undefined };
|
|
219
|
+
return { status: 'fail', detail: `/${v.pattern}/ ${want ? 'NOT found' : 'still present'} in ${v.file}${v.why ? ` (${v.why})` : ''}` };
|
|
220
|
+
},
|
|
221
|
+
|
|
222
|
+
/** The symbol must be referenced (not just defined) in a file reachable by static imports from the default entry point. */
|
|
223
|
+
wiring(v, ctx) {
|
|
224
|
+
const entry = path.join(ctx.root, v.entry);
|
|
225
|
+
if (!fs.existsSync(entry)) return { status: 'fail', detail: `entry ${v.entry} not found` };
|
|
226
|
+
const seen = new Map([[entry, null]]); const q = [entry];
|
|
227
|
+
const exts = ['', '.mjs', '.js', '.cjs', '.ts', '/index.mjs', '/index.js', '/index.ts'];
|
|
228
|
+
const defRe = new RegExp(`(function\\*?|class|const|let|var)\\s+${v.symbol}\\b|^\\s*(async\\s+)?${v.symbol}\\s*\\([^)]*\\)\\s*\\{`, 'm');
|
|
229
|
+
const useRe = new RegExp(`\\b${v.symbol}\\b`);
|
|
230
|
+
while (q.length) {
|
|
231
|
+
const f = q.shift(); let src = '';
|
|
232
|
+
try { src = fs.readFileSync(f, 'utf8'); } catch { continue; }
|
|
233
|
+
const uses = src.split('\n').filter((l) => useRe.test(l) && !defRe.test(l) && !/^\s*(\/\/|\*|import\b|export\s*\{)/.test(l));
|
|
234
|
+
if (uses.length) {
|
|
235
|
+
const chain = []; for (let x = f; x; x = seen.get(x)) chain.unshift(path.relative(ctx.root, x));
|
|
236
|
+
return { status: 'pass', detail: `${v.symbol} used on the default path: ${chain.join(' → ')}`, evidence: { chain, line: uses[0].trim().slice(0, 200) } };
|
|
237
|
+
}
|
|
238
|
+
for (const m of src.matchAll(/(?:import|export)[^'"]*?from\s*['"](\.[^'"]+)['"]|import\s*\(\s*['"](\.[^'"]+)['"]\s*\)|require\(\s*['"](\.[^'"]+)['"]\s*\)|^\s*import\s+['"](\.[^'"]+)['"]/gm)) {
|
|
239
|
+
const spec = m[1] || m[2] || m[3] || m[4];
|
|
240
|
+
for (const e of exts) {
|
|
241
|
+
const p = path.resolve(path.dirname(f), spec + e);
|
|
242
|
+
if (fs.existsSync(p) && fs.statSync(p).isFile()) { if (!seen.has(p)) { seen.set(p, f); q.push(p); } break; }
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return { status: 'fail', detail: `${v.symbol} is not referenced from the default path (${v.entry}; ${seen.size} files reachable). Built but not wired in.` };
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
scope(v, ctx) {
|
|
250
|
+
if (!ctx.base) return { status: 'error', detail: 'base unresolved; cannot compute scope' };
|
|
251
|
+
const allow = v.allow || ctx.task.scope?.allow || ['**'];
|
|
252
|
+
const out = [...ctx.changed.keys()].filter((f) => !matchAny(f, allow));
|
|
253
|
+
return out.length ? { status: 'fail', detail: `changes outside the approved scope: ${out.join(', ')}`, evidence: { outside: out, allow } } : { status: 'pass', detail: `${ctx.changed.size} changed files, all in scope` };
|
|
254
|
+
},
|
|
255
|
+
|
|
256
|
+
docs(v, ctx) {
|
|
257
|
+
const r = run(v.cmd, { cwd: ctx.root, timeoutS: v.timeout_s || 300 });
|
|
258
|
+
if (r.exit !== 0) return { status: r.exit === 127 ? 'error' : 'fail', detail: `docs generator exit ${r.exit}`, evidence: ev(r) };
|
|
259
|
+
const st = git(['status', '--porcelain', '--', ...v.paths], ctx.root).out;
|
|
260
|
+
return st ? { status: 'fail', detail: `generated docs differ from what is committed:\n${st}`, evidence: { changed: st } } : { status: 'pass', detail: 'docs regenerated, no diff' };
|
|
261
|
+
},
|
|
262
|
+
|
|
263
|
+
pushed(v, ctx) {
|
|
264
|
+
const branch = v.branch || ctx.contract.defaults?.branch || 'main';
|
|
265
|
+
const remote = v.remote || ctx.contract.defaults?.remote || 'origin';
|
|
266
|
+
const dirty = git(['status', '--porcelain'], ctx.root).out.split('\n').filter((l) => l && !l.includes('.proofline/'));
|
|
267
|
+
if (dirty.length) return { status: 'fail', detail: `working tree not clean (${dirty.length} paths); work held locally`, evidence: { dirty: dirty.slice(0, 20) } };
|
|
268
|
+
const head = git(['rev-parse', 'HEAD'], ctx.root).out;
|
|
269
|
+
// HEAD passes as the branch's tip or as a commit under it: a task whose commit sits inside a pushed stack is pushed
|
|
270
|
+
const o = onRemote(ctx.root, head, remote, branch);
|
|
271
|
+
if (!o.ok) return { status: 'error', detail: `cannot reach ${remote}: ${o.err}` };
|
|
272
|
+
if (o.at) return { status: 'pass', detail: `HEAD ${head.slice(0, 8)} is ${remote}/${branch}`, evidence: { tip: o.tip, behind: 0 } };
|
|
273
|
+
if (o.contains) return { status: 'pass', detail: `HEAD ${head.slice(0, 8)} is on ${remote}/${branch}, ${o.behind} commit${o.behind === 1 ? '' : 's'} under its tip ${o.tip.slice(0, 8)}`, evidence: { tip: o.tip, behind: o.behind } };
|
|
274
|
+
return { status: 'fail', detail: `HEAD ${head.slice(0, 8)} not pushed: ${remote}/${branch} is ${(o.tip || 'missing').slice(0, 8)}${o.unknown ? ', which could not be fetched to see whether it holds HEAD' : ''}`, evidence: { tip: o.tip || null, unknown: o.unknown } };
|
|
275
|
+
},
|
|
276
|
+
};
|
package/src/claims.mjs
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Parse an agent's status report and check every claim against evidence. Claims are never trusted.
|
|
2
|
+
import { git, onRemote } from './util.mjs';
|
|
3
|
+
|
|
4
|
+
const DONE = /^(done|complete(d)?|verified|pass(ed)?|fixed|finished|green)$/i;
|
|
5
|
+
const HEDGE = /\b(should work|likely|probably|untested|not tested|later|follow[- ]?up|out of scope|defer(red)?|todo|next session|in a future)\b/i;
|
|
6
|
+
|
|
7
|
+
/** Lines: TASK-ID | status | commit | tests | measured vs target | missing */
|
|
8
|
+
export function parseClaims(text, taskIds) {
|
|
9
|
+
const out = [];
|
|
10
|
+
for (const raw of String(text || '').split('\n')) {
|
|
11
|
+
if (!raw.includes('|')) continue;
|
|
12
|
+
const cells = raw.split('|').map((s) => s.trim()).filter((s, i, a) => !(s === '' && (i === 0 || i === a.length - 1)));
|
|
13
|
+
if (cells.length < 2) continue;
|
|
14
|
+
const id = cells[0].replace(/[*`]/g, '');
|
|
15
|
+
if (taskIds && !taskIds.includes(id)) continue;
|
|
16
|
+
out.push({ raw: raw.trim(), id, status: cells[1] || '', commit: cells[2] || '', tests: cells[3] || '', numbers: cells[4] || '', missing: cells[5] || '' });
|
|
17
|
+
}
|
|
18
|
+
return out;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** results: per-task { decision, measured: {key: val}, metricLabels: {key: [aliases]} } */
|
|
22
|
+
export function checkClaim(root, claim, res, { requirePushed = false, remote = 'origin', branch = 'main' } = {}) {
|
|
23
|
+
const issues = [];
|
|
24
|
+
const claimedDone = DONE.test(claim.status.replace(/[^a-z]/gi, ''));
|
|
25
|
+
if (claimedDone && res.decision !== 'PASS') issues.push(`claimed "${claim.status}" but the independent gate says ${res.decision}: ${res.failing.join('; ') || 'see report'}`);
|
|
26
|
+
if (HEDGE.test(claim.raw)) issues.push(`hedged or deferred wording ("${claim.raw.match(HEDGE)[0]}"): not an acceptable status; never defer, report blocked with what you need`);
|
|
27
|
+
const sha = (claim.commit.match(/\b[0-9a-f]{7,40}\b/i) || [])[0];
|
|
28
|
+
if (claimedDone && !sha) issues.push('claimed done without a commit hash');
|
|
29
|
+
if (sha) {
|
|
30
|
+
const full = git(['rev-parse', '--verify', '--quiet', `${sha}^{commit}`], root);
|
|
31
|
+
if (!full.ok) issues.push(`commit ${sha} does not exist`);
|
|
32
|
+
else {
|
|
33
|
+
if (!git(['merge-base', '--is-ancestor', full.out, 'HEAD'], root).ok) issues.push(`commit ${sha} is not in HEAD's history`);
|
|
34
|
+
if (requirePushed) {
|
|
35
|
+
const o = onRemote(root, full.out, remote, branch);
|
|
36
|
+
if (!o.ok) issues.push(`cannot check that commit ${sha} is on ${remote}/${branch}: ${o.err}`);
|
|
37
|
+
else if (!o.contains) issues.push(`commit ${sha} is not on ${remote}/${branch}${o.unknown ? ` (its tip ${o.tip.slice(0, 8)} could not be fetched)` : ''}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const names = claim.tests.split(/[,;]+/).map((s) => s.trim().replace(/^`|`$/g, '')).filter((s) => s && !/^(-|n\/a|none)$/i.test(s));
|
|
42
|
+
if (claimedDone && !names.length) issues.push('claimed done without naming the test(s) that prove it');
|
|
43
|
+
for (const n of names) {
|
|
44
|
+
const base = n.replace(/\s*\(.*\)$/, '').split(/\s+/)[0];
|
|
45
|
+
if (!git(['grep', '-q', '-F', base], root).ok && !git(['ls-files', '--', `*${base}*`], root).out) issues.push(`named test "${base}" not found in the repo`);
|
|
46
|
+
}
|
|
47
|
+
const keys = Object.keys(res.measured || {});
|
|
48
|
+
if (claimedDone && keys.length && !/\d/.test(claim.numbers)) issues.push(`task has metric targets (${keys.join(', ')}) but the claim gives no measured numbers`);
|
|
49
|
+
for (const k of keys) {
|
|
50
|
+
const aliases = [k, ...(res.metricLabels?.[k] || [])];
|
|
51
|
+
for (const a of aliases) {
|
|
52
|
+
const m = claim.numbers.match(new RegExp(a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\D{0,12}?(\\d+(?:\\.\\d+)?)', 'i'));
|
|
53
|
+
if (!m) continue;
|
|
54
|
+
const said = +m[1], got = res.measured[k];
|
|
55
|
+
if (Math.abs(said - got) > Math.max(0.15 * Math.abs(got), 1)) issues.push(`claimed ${a} ${said} but measured ${got}`);
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return { claim: claim.raw, id: claim.id, ok: issues.length === 0, issues };
|
|
60
|
+
}
|