ai-engineering-loop 2.10.1 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +2 -0
- package/.agents/skills/ael/references/loop-deliver.md +1 -0
- package/.agents/skills/ael/references/loop-goal.md +1 -0
- package/.agents/skills/ael/references/loop-risky.md +1 -0
- package/.claude/agents/devil-advocate.md +2 -1
- package/.claude/agents/judge.md +2 -0
- package/.grok/agents/devil-advocate.md +2 -1
- package/.grok/agents/judge.md +2 -0
- package/CHANGELOG.md +24 -0
- package/README.md +80 -75
- package/agents/shared/devil-advocate.body.md +2 -1
- package/agents/shared/judge.body.md +2 -0
- package/bin/ai-engineering-loop.js +153 -5
- package/core/goal-contract.md +4 -0
- package/core/iteration-policy.md +4 -0
- package/core/run-isolation.md +55 -0
- package/core/verification-loop.md +25 -0
- package/docs/images/ael-hero.svg +19 -0
- package/docs/images/readme-banner.png +0 -0
- package/docs/images/social-preview.png +0 -0
- package/docs/images/studio-hero-orange.svg +136 -0
- package/lib/ablation.js +1 -1
- package/lib/cli-args.js +5 -2
- package/lib/finding-proofs.js +72 -0
- package/lib/gates.js +21 -0
- package/lib/harness-audit.js +20 -0
- package/lib/interventions.js +56 -0
- package/lib/measurements.js +51 -0
- package/lib/orchestration.js +1 -0
- package/lib/recap-html.js +2 -2
- package/lib/recap.js +2 -0
- package/lib/rules.js +51 -1
- package/lib/run-isolation.js +293 -0
- package/lib/run-state.js +2 -1
- package/lib/safety-fact.js +64 -0
- package/lib/stats.js +9 -0
- package/lib/verification-lanes.js +23 -0
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +4 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { spawnSync } = require('child_process');
|
|
6
|
+
const { destructiveFindings } = require('./guard.js');
|
|
7
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
8
|
+
|
|
9
|
+
const PROOF_TIMEOUT_MS = 120000;
|
|
10
|
+
const TAIL = 2000;
|
|
11
|
+
|
|
12
|
+
const tail = (text) => redactHomePaths(redactSecrets(String(text || '')).text).slice(-TAIL);
|
|
13
|
+
|
|
14
|
+
function runProof(rootDir, proof, timeoutMs) {
|
|
15
|
+
const command = String(proof.command || '').trim();
|
|
16
|
+
if (!command) return { status: 'refused', reason: 'proof.command is empty' };
|
|
17
|
+
const destructive = destructiveFindings(command);
|
|
18
|
+
if (destructive.length) return { status: 'refused', reason: destructive.map((item) => item.reason).join('; ') };
|
|
19
|
+
const started = Date.now();
|
|
20
|
+
const result = proof.shell === false
|
|
21
|
+
? spawnSync(command.split(/\s+/)[0], command.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs })
|
|
22
|
+
: spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
|
|
23
|
+
const durationMs = Date.now() - started;
|
|
24
|
+
if (result.error?.code === 'ETIMEDOUT' || (result.signal && durationMs >= timeoutMs - 50)) {
|
|
25
|
+
return { status: 'timed_out', durationMs };
|
|
26
|
+
}
|
|
27
|
+
if (result.error) return { status: 'refused', reason: result.error.message, durationMs };
|
|
28
|
+
return {
|
|
29
|
+
status: result.status === 0 ? 'not_reproduced' : 'reproduced',
|
|
30
|
+
exitCode: result.status,
|
|
31
|
+
durationMs,
|
|
32
|
+
stdoutTail: tail(result.stdout),
|
|
33
|
+
stderrTail: tail(result.stderr)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function runFindingProofs(rootDir, run, { timeoutMs = PROOF_TIMEOUT_MS } = {}) {
|
|
38
|
+
const runDir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
|
|
39
|
+
const ledgerPath = path.join(runDir, 'findings.json');
|
|
40
|
+
const counts = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
41
|
+
let ledger;
|
|
42
|
+
try {
|
|
43
|
+
ledger = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'));
|
|
44
|
+
} catch {
|
|
45
|
+
return counts;
|
|
46
|
+
}
|
|
47
|
+
if (!Array.isArray(ledger.findings)) return counts;
|
|
48
|
+
const records = [];
|
|
49
|
+
let changed = false;
|
|
50
|
+
ledger.findings = ledger.findings.map((finding) => {
|
|
51
|
+
const next = { ...finding };
|
|
52
|
+
if ('proofStatus' in next) {
|
|
53
|
+
delete next.proofStatus;
|
|
54
|
+
changed = true;
|
|
55
|
+
}
|
|
56
|
+
const runnable = next.validity === 'VALID' && ['BLOCKER', 'HIGH'].includes(next.severity) && next.proof && typeof next.proof === 'object';
|
|
57
|
+
if (!runnable) return next;
|
|
58
|
+
const outcome = runProof(rootDir, next.proof, timeoutMs);
|
|
59
|
+
next.proofStatus = outcome.status;
|
|
60
|
+
counts[outcome.status] += 1;
|
|
61
|
+
records.push({ id: next.id, command: tail(next.proof.command), ...outcome });
|
|
62
|
+
changed = true;
|
|
63
|
+
return next;
|
|
64
|
+
});
|
|
65
|
+
if (changed) fs.writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, { mode: 0o600 });
|
|
66
|
+
if (records.length) {
|
|
67
|
+
fs.writeFileSync(path.join(runDir, 'proofs.json'), `${JSON.stringify({ schemaVersion: 1, runId: run.runId, diffHash: ledger.diffHash, proofs: records }, null, 2)}\n`, { mode: 0o600 });
|
|
68
|
+
}
|
|
69
|
+
return counts;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
module.exports = { PROOF_TIMEOUT_MS, runFindingProofs };
|
package/lib/gates.js
CHANGED
|
@@ -686,6 +686,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
686
686
|
mergeImpactDrift(rootDir, run);
|
|
687
687
|
mergeVerificationIntegrity(rootDir, run);
|
|
688
688
|
mergeCommentJustifications(rootDir, run);
|
|
689
|
+
if (!require('./ablation.js').ablated(rootDir, 'proofs', run.runId)) require('./finding-proofs.js').runFindingProofs(rootDir, run);
|
|
689
690
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
690
691
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
691
692
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
|
@@ -733,6 +734,14 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
733
734
|
}
|
|
734
735
|
const blocking = expected.blockingFindings || [];
|
|
735
736
|
const cause = value.cause || inferCause(expected.verdict, blocking);
|
|
737
|
+
if (expected.verdict === 'ITERATE' && value.cause) {
|
|
738
|
+
const previous = require('./no-progress.js').readIterations(rootDir, run.runId).filter((entry) => entry.verdict === 'ITERATE').pop();
|
|
739
|
+
const premise = value.premise || {};
|
|
740
|
+
if (previous && !previous.causeInferred && previous.cause === cause && !(String(premise.statement || '').trim() && String(premise.test || '').trim())) {
|
|
741
|
+
throw gateError(`Judge ITERATE repeats cause "${cause}": state and test the shared premise first (PREMISE_REQUIRED)`, [`iteration ${previous.iteration || run.iteration - 1} also failed on ${cause}`],
|
|
742
|
+
{ why: 'When two fixes that share one premise fail the same gate, the premise is the likely bug; a third fix on the same premise usually fails too.', fix: 'Add premise: { statement: "<the one sentence every failed fix assumed>", test: "<the observation that would differ if it were false, and what it showed>" } to verdict.json, then retry `gate judge`.' });
|
|
743
|
+
}
|
|
744
|
+
}
|
|
736
745
|
recordIteration(rootDir, run, {
|
|
737
746
|
verdict: expected.verdict,
|
|
738
747
|
diffHash: run.artifacts?.diff?.sha256,
|
|
@@ -747,6 +756,18 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
747
756
|
throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
|
|
748
757
|
{ why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
|
|
749
758
|
}
|
|
759
|
+
if (tierOf(run) === 'T2') {
|
|
760
|
+
const safety = require('./safety-fact.js').safetyFactProblem(rootDir, run);
|
|
761
|
+
if (safety) {
|
|
762
|
+
throw gateError('Judge PASS refused: T2 needs a proven safety fact (SAFETY_FACT_UNPROVEN)', [safety],
|
|
763
|
+
{ why: 'A high-risk change is safe because of one fact; a writeup that sounds right proves nothing until real code runs.', fix: `Run \`safety record --run ${run.runId} --fact "<the one fact>" --cmd "<script or test that passes only if the fact holds>"\`, then retry \`gate judge\`.` });
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const measured = require('./measurements.js').measurementProblems(rootDir, run, loadFrozenGoal(rootDir, run));
|
|
767
|
+
if (measured.length) {
|
|
768
|
+
throw gateError('Judge PASS refused: a performance number is not explained (MEASUREMENT_UNEXPLAINED)', measured,
|
|
769
|
+
{ why: 'A single run or a gap smaller than the noise says nothing; the limiter tells you whether the number measured the change at all.', fix: `Measure at least 3 runs, find what bounds the result, then \`measurement record --run ${run.runId} --ac <AC-n> --runs <n> --spread <pct> --effect <pct> --limiter "<what bounds it>"\` and retry.` });
|
|
770
|
+
}
|
|
750
771
|
const unproven = unprovenCriteria(rootDir, run);
|
|
751
772
|
if (unproven.length) {
|
|
752
773
|
throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
|
package/lib/harness-audit.js
CHANGED
|
@@ -92,6 +92,7 @@ function auditRules(rootDir, now = new Date()) {
|
|
|
92
92
|
else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
|
|
93
93
|
else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
|
|
94
94
|
}
|
|
95
|
+
if (!rule.provenBy) items.push({ id: 'rule-unproven', level: 'info', detail: `${rule.id} was never shown to catch a real mistake. FIX: re-promote it with \`learn promote ${rule.from || '<learning>'} ... --prove <commit that introduced the mistake>\`.` });
|
|
95
96
|
const age = (now - new Date(rule.createdAt || 0)) / 86400000;
|
|
96
97
|
if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
|
|
97
98
|
}
|
|
@@ -127,10 +128,29 @@ function auditFeatureMap(rootDir) {
|
|
|
127
128
|
return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
|
|
128
129
|
}
|
|
129
130
|
|
|
131
|
+
function auditFeatureMapDrift(rootDir) {
|
|
132
|
+
const entries = require('./feature-map.js').loadFeatureMap(rootDir);
|
|
133
|
+
if (!entries.length) return [];
|
|
134
|
+
let files = [];
|
|
135
|
+
try {
|
|
136
|
+
files = require('child_process').execFileSync('git', ['ls-files'], { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).split('\n').filter(Boolean);
|
|
137
|
+
} catch {
|
|
138
|
+
return [];
|
|
139
|
+
}
|
|
140
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
141
|
+
const items = [];
|
|
142
|
+
for (const entry of entries) {
|
|
143
|
+
const dead = entry.paths.filter((glob) => !files.some((file) => matchesGlob(file, glob)));
|
|
144
|
+
if (dead.length) items.push({ id: 'feature-map-drift', level: 'warn', detail: `Feature "${entry.id}" lists paths that match no tracked file: ${dead.join(', ')}. FIX: update the Paths column in .ai-engineering-loop/feature-map.md, or remove the feature if it is gone.` });
|
|
145
|
+
}
|
|
146
|
+
return items;
|
|
147
|
+
}
|
|
148
|
+
|
|
130
149
|
function auditHarness(rootDir, { budget = true } = {}) {
|
|
131
150
|
const items = [
|
|
132
151
|
...auditEntryFile(rootDir),
|
|
133
152
|
...auditFeatureMap(rootDir),
|
|
153
|
+
...auditFeatureMapDrift(rootDir),
|
|
134
154
|
...auditLearnings(rootDir),
|
|
135
155
|
...auditRules(rootDir),
|
|
136
156
|
...(budget ? auditBudget(rootDir) : [])
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const lines = (file) => {
|
|
7
|
+
try {
|
|
8
|
+
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
|
|
9
|
+
try {
|
|
10
|
+
return [JSON.parse(line)];
|
|
11
|
+
} catch {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
} catch {
|
|
16
|
+
return [];
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const readJson = (file) => {
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
23
|
+
} catch {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
function runInterventions(runDir) {
|
|
29
|
+
const answers = lines(path.join(runDir, 'interactions.jsonl')).filter((entry) => entry.type === 'ANSWER').length;
|
|
30
|
+
const state = readJson(path.join(runDir, 'state.json'));
|
|
31
|
+
const escalated = state?.state === 'ESCALATED' || (state?.history || []).some((entry) => entry.to === 'ESCALATED') ? 1 : 0;
|
|
32
|
+
const outcomes = lines(path.join(runDir, 'outcomes.jsonl')).length;
|
|
33
|
+
const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
34
|
+
for (const proof of readJson(path.join(runDir, 'proofs.json'))?.proofs || []) {
|
|
35
|
+
if (Object.hasOwn(proofs, proof.status)) proofs[proof.status] += 1;
|
|
36
|
+
}
|
|
37
|
+
return { answers, escalated, outcomes, total: answers + escalated + outcomes, proofs };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function summarizeInterventions(perRun) {
|
|
41
|
+
const sum = (key) => perRun.reduce((n, item) => n + item[key], 0);
|
|
42
|
+
const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
43
|
+
for (const item of perRun) for (const key of Object.keys(proofs)) proofs[key] += item.proofs[key];
|
|
44
|
+
const ran = proofs.reproduced + proofs.not_reproduced;
|
|
45
|
+
return {
|
|
46
|
+
runs: perRun.length,
|
|
47
|
+
answers: sum('answers'),
|
|
48
|
+
escalations: sum('escalated'),
|
|
49
|
+
outcomes: sum('outcomes'),
|
|
50
|
+
perRun: perRun.length ? Math.round((sum('total') / perRun.length) * 100) / 100 : null,
|
|
51
|
+
proofs,
|
|
52
|
+
reproducedRate: ran ? Math.round((proofs.reproduced / ran) * 1000) / 1000 : null
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { runInterventions, summarizeInterventions };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const FILE = 'measurements.json';
|
|
7
|
+
const MIN_RUNS = 3;
|
|
8
|
+
|
|
9
|
+
const fail = (message) => Object.assign(new Error(message), { code: 'INVALID_USAGE' });
|
|
10
|
+
const filePath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
|
|
11
|
+
const percent = (value, flag) => {
|
|
12
|
+
const number = Number(String(value ?? '').replace(/%$/, ''));
|
|
13
|
+
if (!Number.isFinite(number) || number < 0) throw fail(`${flag} must be a non-negative percentage, such as 4 or 4%`);
|
|
14
|
+
return number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function loadMeasurements(rootDir, runId) {
|
|
18
|
+
try {
|
|
19
|
+
return JSON.parse(fs.readFileSync(filePath(rootDir, runId), 'utf8')).measurements || [];
|
|
20
|
+
} catch {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function recordMeasurement(rootDir, runId, { ac, runs, spread, effect, limiter, now = new Date() }) {
|
|
26
|
+
if (!/^AC-\d+$/.test(String(ac || ''))) throw fail('--ac must be an AC id such as AC-2');
|
|
27
|
+
const count = Number(runs);
|
|
28
|
+
if (!Number.isInteger(count) || count < 1) throw fail('--runs must be a whole number of measured runs');
|
|
29
|
+
const reason = String(limiter || '').trim();
|
|
30
|
+
if (!reason) throw fail('--limiter is required: the resource or code path that bounds the number, from a profile or counters');
|
|
31
|
+
const entry = { ac, runs: count, spread: percent(spread, '--spread'), effect: percent(effect, '--effect'), limiter: reason, recordedAt: now.toISOString() };
|
|
32
|
+
const measurements = loadMeasurements(rootDir, runId).filter((item) => item.ac !== ac).concat(entry);
|
|
33
|
+
fs.writeFileSync(filePath(rootDir, runId), `${JSON.stringify({ schemaVersion: 1, runId, measurements }, null, 2)}\n`, { mode: 0o600 });
|
|
34
|
+
return entry;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function measurementProblems(rootDir, run, goal) {
|
|
38
|
+
const needed = (goal?.acceptanceCriteria || []).filter((ac) => ac.measure === true);
|
|
39
|
+
if (!needed.length) return [];
|
|
40
|
+
const recorded = new Map(loadMeasurements(rootDir, run.runId).map((item) => [item.ac, item]));
|
|
41
|
+
const problems = [];
|
|
42
|
+
for (const ac of needed) {
|
|
43
|
+
const m = recorded.get(ac.id);
|
|
44
|
+
if (!m) problems.push(`${ac.id}: no measurement recorded`);
|
|
45
|
+
else if (m.runs < MIN_RUNS) problems.push(`${ac.id}: ${m.runs} run(s); at least ${MIN_RUNS} are needed to see the noise`);
|
|
46
|
+
else if (m.spread >= m.effect) problems.push(`${ac.id}: run-to-run spread ${m.spread}% is not smaller than the effect ${m.effect}%`);
|
|
47
|
+
}
|
|
48
|
+
return problems;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
module.exports = { MIN_RUNS, loadMeasurements, measurementProblems, recordMeasurement };
|
package/lib/orchestration.js
CHANGED
|
@@ -151,6 +151,7 @@ function validateFindingLedger(ledger) {
|
|
|
151
151
|
function isBlockingFinding(finding) {
|
|
152
152
|
if (!finding || finding.validity !== 'VALID') return false;
|
|
153
153
|
if (finding.severity !== 'BLOCKER' && finding.severity !== 'HIGH') return false;
|
|
154
|
+
if (finding.proofStatus === 'not_reproduced') return false;
|
|
154
155
|
const axis = finding.axis || 'spec';
|
|
155
156
|
if (axis === 'standards' && finding.hardConvention !== true) return false;
|
|
156
157
|
return true;
|
package/lib/recap-html.js
CHANGED
|
@@ -195,11 +195,11 @@ function renderRecap(data, { audience = 'me', now = new Date(), lang = null } =
|
|
|
195
195
|
const contentLang = lang ? ` lang="${esc(lang)}"` : '';
|
|
196
196
|
const technical = runs.filter((run) => run.delivered && run.clientTextSource === 'objective');
|
|
197
197
|
const empty = `<p class="empty">No runs in the last ${esc(data.window.days)} days. Try a longer window: <code>ael recap --days ${esc(data.window.days * 4)}</code></p>`;
|
|
198
|
-
const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}</div>
|
|
198
|
+
const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Times you had to step in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>
|
|
199
199
|
<div class="panel"><h2>Activity per day</h2>${heatmap(data)}</div>
|
|
200
200
|
<div class="panel"><h2>Where the time went</h2>${stageBars(runs)}</div>
|
|
201
201
|
<div class="panel"><h2>Why runs had to iterate</h2>${causes(metrics)}</div>` : `<div class="panel">${empty}</div>`;
|
|
202
|
-
const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
|
|
202
|
+
const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}${metric('Times a human stepped in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
|
|
203
203
|
const tab = (view, text) => `<button type="button" aria-controls="${view}" data-view="${view}" aria-pressed="${view === audience}">${text}</button>`;
|
|
204
204
|
const section = (view, html) => `<section class="view${view === audience ? ' on' : ''}" id="${view}" aria-label="${esc({ me: 'For me', lead: 'For the lead', client: 'For the client' }[view])}">${html}</section>`;
|
|
205
205
|
const range = `${fmtDate(data.window.from)} – ${fmtDate(new Date(data.window.to.getTime() - 1))}`;
|
package/lib/recap.js
CHANGED
|
@@ -110,6 +110,7 @@ function loadRun(projectPath, runId) {
|
|
|
110
110
|
outOfMap: Array.isArray(receipt.impact?.outOfMap) ? receipt.impact.outOfMap : [],
|
|
111
111
|
openFindings: Array.isArray(summary.openFindings) ? summary.openFindings.length : 0,
|
|
112
112
|
stages: stageDurations(history),
|
|
113
|
+
interventions: require('./interventions.js').runInterventions(dir),
|
|
113
114
|
files
|
|
114
115
|
};
|
|
115
116
|
}
|
|
@@ -146,6 +147,7 @@ function metricsOf(runs) {
|
|
|
146
147
|
firstPassRate: judged.length ? Math.round((judged.filter((run) => run.iterations[0].verdict === 'PASS').length / judged.length) * 100) : null,
|
|
147
148
|
medianCycleMs: median(runs.filter((run) => run.delivered).map(cycle).filter(Boolean)),
|
|
148
149
|
t2: runs.filter((run) => run.tier === 'T2').length,
|
|
150
|
+
interventionsPerRun: require('./interventions.js').summarizeInterventions(runs.map((run) => run.interventions)).perRun,
|
|
149
151
|
causes
|
|
150
152
|
};
|
|
151
153
|
}
|
package/lib/rules.js
CHANGED
|
@@ -36,7 +36,7 @@ function loadLearningEntries(rootDir) {
|
|
|
36
36
|
|
|
37
37
|
const SCOPES = Object.freeze(['repo', 'diff']);
|
|
38
38
|
|
|
39
|
-
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
|
|
39
|
+
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, prove = null, now = new Date() } = {}) {
|
|
40
40
|
if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
|
|
41
41
|
// C-20 built-in kinds: no hand-written check, always judged on added lines.
|
|
42
42
|
let builtin = null;
|
|
@@ -79,10 +79,59 @@ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shel
|
|
|
79
79
|
createdBy: String(by).trim(),
|
|
80
80
|
createdAt: now.toISOString()
|
|
81
81
|
};
|
|
82
|
+
if (prove) rule.provenBy = proveRule(rootDir, rule, prove, { now });
|
|
82
83
|
atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
|
|
83
84
|
return rule;
|
|
84
85
|
}
|
|
85
86
|
|
|
87
|
+
function proveRule(rootDir, rule, commit, { now = new Date() } = {}) {
|
|
88
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
89
|
+
const os = require('os');
|
|
90
|
+
const git = (...args) => execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
91
|
+
let sha;
|
|
92
|
+
try {
|
|
93
|
+
sha = git('rev-parse', '--verify', `${commit}^{commit}`).trim();
|
|
94
|
+
} catch {
|
|
95
|
+
throw rulesError(`--prove ${commit} is not a commit in this repository.`, 'INVALID_USAGE');
|
|
96
|
+
}
|
|
97
|
+
let failed;
|
|
98
|
+
if (rule.scope === 'diff') {
|
|
99
|
+
const { parseDiff } = require('./verification-integrity.js');
|
|
100
|
+
const files = parseDiff(git('show', '--format=', '--no-color', sha, '--', '.', ':(exclude).ai-engineering-loop')).filter((file) => !file.deleted);
|
|
101
|
+
const lines = files.flatMap((file) => file.added.map((added) => ({ file: file.path, line: added.line, text: added.text })));
|
|
102
|
+
if (rule.kind) {
|
|
103
|
+
failed = require('./rule-check.js').checkRule(rule, lines).length > 0;
|
|
104
|
+
} else {
|
|
105
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-'));
|
|
106
|
+
const addedPath = path.join(tmp, 'added-lines.txt');
|
|
107
|
+
fs.writeFileSync(addedPath, lines.map((l) => `${l.file}:${l.line}:${l.text}`).join('\n'));
|
|
108
|
+
const env = { ...process.env, AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
|
|
109
|
+
const result = rule.shell !== false
|
|
110
|
+
? spawnSync(rule.check, { cwd: rootDir, shell: true, env, encoding: 'utf8' })
|
|
111
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, env, encoding: 'utf8' });
|
|
112
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
113
|
+
failed = result.status !== 0;
|
|
114
|
+
}
|
|
115
|
+
} else {
|
|
116
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-wt-'));
|
|
117
|
+
const tree = path.join(tmp, 'tree');
|
|
118
|
+
git('worktree', 'add', '--detach', '--quiet', tree, sha);
|
|
119
|
+
try {
|
|
120
|
+
const result = rule.shell !== false
|
|
121
|
+
? spawnSync(rule.check, { cwd: tree, shell: true, encoding: 'utf8' })
|
|
122
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: tree, encoding: 'utf8' });
|
|
123
|
+
failed = result.status !== 0;
|
|
124
|
+
} finally {
|
|
125
|
+
git('worktree', 'remove', '--force', tree);
|
|
126
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (!failed) {
|
|
130
|
+
throw rulesError(`The check did not fail on ${sha.slice(0, 12)}, so it would not have caught that mistake. Pick the commit that introduced the mistake, or fix the check.`, 'RULE_NOT_PROVEN');
|
|
131
|
+
}
|
|
132
|
+
return { commit: sha, at: now.toISOString() };
|
|
133
|
+
}
|
|
134
|
+
|
|
86
135
|
// Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
|
|
87
136
|
function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
|
|
88
137
|
const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
|
|
@@ -189,6 +238,7 @@ function missingRules(rootDir, bundle) {
|
|
|
189
238
|
}
|
|
190
239
|
|
|
191
240
|
module.exports = {
|
|
241
|
+
proveRule,
|
|
192
242
|
SCOPES,
|
|
193
243
|
commandForRule,
|
|
194
244
|
findRule,
|