ai-engineering-loop 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +4 -3
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +1 -1
- package/.agents/skills/ael/references/loop-maker.md +1 -0
- package/.claude/agents/devil-advocate.md +1 -1
- package/.grok/agents/devil-advocate.md +1 -1
- package/CHANGELOG.md +22 -0
- package/agents/shared/devil-advocate.body.md +1 -1
- package/bin/ai-engineering-loop.js +69 -6
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +18 -0
- package/core/verification-loop.md +38 -0
- package/lib/ablation.js +1 -1
- package/lib/ac-status.js +39 -7
- package/lib/budget.js +9 -1
- package/lib/calibration.js +17 -0
- package/lib/claimed-vs-reality.js +1 -1
- package/lib/cli-args.js +1 -1
- package/lib/comment-findings.js +187 -0
- package/lib/feature-map.js +57 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +23 -0
- package/lib/harness-audit.js +17 -0
- package/lib/learn.js +24 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +99 -6
- package/lib/stats.js +20 -0
- package/lib/verification-lanes.js +39 -7
- package/lib/verification-recorder.js +3 -3
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +22 -3
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { isTestFile, parseDiff } = require('./verification-integrity.js');
|
|
6
|
+
|
|
7
|
+
const SOURCE = 'comment-justification';
|
|
8
|
+
const NO_COMMENTS_SOURCE = 'no-comments';
|
|
9
|
+
const SKIP_FILE = /\.(md|mdx|txt|rst|json|jsonc|lock|ya?ml|toml|ini|cfg|conf|env|csv|svg|snap|xml|plist)$|(^|\/)(CHANGELOG|LICENSE|NOTICE|README)[^/]*$/i;
|
|
10
|
+
|
|
11
|
+
const SLASH = /\.(c|h|cc|cpp|hpp|cs|java|kt|kts|scala|swift|go|rs|js|jsx|mjs|cjs|ts|tsx|mts|cts|dart|php|css|scss|less|groovy|proto|zig|sol)$/i;
|
|
12
|
+
const HASH = /\.(py|pyi|sh|bash|zsh|fish|rb|pl|pm|r|ps1|tf|hcl|nix|cmake|mk)$|(^|\/)(Makefile|Dockerfile|Gemfile|Rakefile|BUILD|WORKSPACE)$/i;
|
|
13
|
+
const MARKUP = /\.(html?|vue|svelte|astro)$/i;
|
|
14
|
+
const DASH = /\.(sql|lua|hs|elm)$/i;
|
|
15
|
+
|
|
16
|
+
const ALLOWED = /^#!|SPDX-License-Identifier|\bcopyright\b|@license|@preserve|@generated|^\/\/\/\s*<reference|^\/\/go:(build|generate|embed)|^\/\/\s*\+build|-\*-\s*coding|^#\s*(region|endregion)\b|^\/\/\s*#(region|endregion)\b|webpackChunkName|@vite-ignore/i;
|
|
17
|
+
const SUPPRESS = [
|
|
18
|
+
{ pattern: /eslint-disable(?:-next-line|-line)?/, reason: /eslint-disable\S*(?:\s.*?)?\s--\s+(.{10,})/ },
|
|
19
|
+
{ pattern: /@ts-ignore/, reason: /@ts-ignore\s+(.{10,})/ },
|
|
20
|
+
{ pattern: /@ts-expect-error/, reason: /@ts-expect-error\s+(.{10,})/ },
|
|
21
|
+
{ pattern: /#\s*type:\s*ignore/, reason: /type:\s*ignore\S*\s+#?\s*(.{10,})/ },
|
|
22
|
+
{ pattern: /#\s*noqa/, reason: /noqa\S*\s+#?\s*(.{10,})/ },
|
|
23
|
+
{ pattern: /\/\/\s*nolint/, reason: /nolint\S*\s+\/\/\s*(.{10,})/ },
|
|
24
|
+
{ pattern: /NOSONAR/, reason: /NOSONAR\s+(.{10,})/ },
|
|
25
|
+
{ pattern: /istanbul ignore|c8 ignore/, reason: /ignore \w+\s+(.{10,})/ },
|
|
26
|
+
{ pattern: /prettier-ignore|biome-ignore|stylelint-disable|rubocop:disable|pylint:\s*disable/, reason: /(?:prettier-ignore|biome-ignore\S*|stylelint-disable\S*|rubocop:disable\S*|pylint:\s*disable\S*)\s+.*?(?:--|:)\s*(.{10,})/ }
|
|
27
|
+
];
|
|
28
|
+
const JUSTIFICATION = /\b(work-?around|quick[- ]?fix|hack(y)?|temporary|temp fix|for now|band-?aid|kludge|until we|proper fix later|fix (this )?later)\b/i;
|
|
29
|
+
|
|
30
|
+
function commentIndex(text, marker) {
|
|
31
|
+
let quote = null;
|
|
32
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
33
|
+
const ch = text[i];
|
|
34
|
+
if (quote) {
|
|
35
|
+
if (ch === '\\') i += 1;
|
|
36
|
+
else if (ch === quote) quote = null;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
40
|
+
quote = ch;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (text.startsWith(marker, i)) {
|
|
44
|
+
if (marker === '#' && i > 0 && !/\s/.test(text[i - 1])) continue;
|
|
45
|
+
return i;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return -1;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function commentOf(file, text) {
|
|
52
|
+
if (SLASH.test(file)) {
|
|
53
|
+
if (/^\s*\*(?!\*\/)/.test(text) && !/^\s*\*\s*[\w(]+\s*[=;)]/.test(text)) return text.trim();
|
|
54
|
+
for (const marker of ['//', '/*']) {
|
|
55
|
+
const at = commentIndex(text, marker);
|
|
56
|
+
if (at !== -1) return text.slice(at).trim();
|
|
57
|
+
}
|
|
58
|
+
const jsx = text.indexOf('{/*');
|
|
59
|
+
return jsx !== -1 ? text.slice(jsx).trim() : null;
|
|
60
|
+
}
|
|
61
|
+
if (HASH.test(file)) {
|
|
62
|
+
const at = commentIndex(text, '#');
|
|
63
|
+
return at !== -1 ? text.slice(at).trim() : null;
|
|
64
|
+
}
|
|
65
|
+
if (MARKUP.test(file)) {
|
|
66
|
+
const at = text.indexOf('<!--');
|
|
67
|
+
if (at !== -1) return text.slice(at).trim();
|
|
68
|
+
const slash = commentIndex(text, '//');
|
|
69
|
+
return slash !== -1 && /^\s*\/\//.test(text) ? text.slice(slash).trim() : null;
|
|
70
|
+
}
|
|
71
|
+
if (DASH.test(file)) {
|
|
72
|
+
const at = commentIndex(text, '--');
|
|
73
|
+
return at !== -1 ? text.slice(at).trim() : null;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function loadProjectRules(rootDir) {
|
|
79
|
+
try {
|
|
80
|
+
return JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'rules.json'), 'utf8'));
|
|
81
|
+
} catch {
|
|
82
|
+
return {};
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function loadExtraPatterns(rootDir) {
|
|
87
|
+
return (loadProjectRules(rootDir).justificationPatterns || []).map((text) => new RegExp(text, 'i'));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function scanComments(diffText, { extraPatterns = [], noComments = true } = {}) {
|
|
91
|
+
const findings = [];
|
|
92
|
+
let n = 0;
|
|
93
|
+
for (const file of parseDiff(diffText)) {
|
|
94
|
+
if (file.deleted || isTestFile(file.path) || SKIP_FILE.test(file.path)) continue;
|
|
95
|
+
const plain = [];
|
|
96
|
+
for (const added of file.added) {
|
|
97
|
+
const comment = commentOf(file.path, added.text);
|
|
98
|
+
if (!comment) continue;
|
|
99
|
+
const location = `${file.path}:${added.line}`;
|
|
100
|
+
const suppress = SUPPRESS.find((item) => item.pattern.test(comment));
|
|
101
|
+
if (suppress) {
|
|
102
|
+
const reasoned = suppress.reason.test(comment);
|
|
103
|
+
findings.push({
|
|
104
|
+
id: `COMMENT-${++n}`,
|
|
105
|
+
axis: 'standards',
|
|
106
|
+
location,
|
|
107
|
+
failureScenario: reasoned
|
|
108
|
+
? 'A lint/type/coverage suppression was added; it hides the signal instead of fixing what the check found'
|
|
109
|
+
: 'A lint/type/coverage suppression was added with no reason; the next agent will copy it as the established pattern',
|
|
110
|
+
evidence: `added line: ${added.text.trim().slice(0, 160)}`,
|
|
111
|
+
severity: reasoned ? 'MEDIUM' : 'HIGH',
|
|
112
|
+
validity: 'VALID',
|
|
113
|
+
disposition: 'WEAK',
|
|
114
|
+
concreteAlternativeDiff: `- ${added.text.trim()}\n+ (remove the suppression and fix the code the check flagged)`,
|
|
115
|
+
source: SOURCE
|
|
116
|
+
});
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (ALLOWED.test(comment)) continue;
|
|
120
|
+
if (JUSTIFICATION.test(comment) || extraPatterns.some((pattern) => pattern.test(comment))) {
|
|
121
|
+
findings.push({
|
|
122
|
+
id: `COMMENT-${++n}`,
|
|
123
|
+
axis: 'standards',
|
|
124
|
+
location,
|
|
125
|
+
failureScenario: 'A comment justifies a workaround instead of fixing the root cause; comments cannot be enforced and the workaround spreads',
|
|
126
|
+
evidence: `added line: ${added.text.trim().slice(0, 160)}`,
|
|
127
|
+
severity: 'MEDIUM',
|
|
128
|
+
validity: 'VALID',
|
|
129
|
+
disposition: 'WEAK',
|
|
130
|
+
source: SOURCE
|
|
131
|
+
});
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
plain.push(added);
|
|
135
|
+
}
|
|
136
|
+
if (noComments && plain.length) {
|
|
137
|
+
const lines = plain.map((added) => added.line);
|
|
138
|
+
findings.push({
|
|
139
|
+
id: `NOCOMMENT-${++n}`,
|
|
140
|
+
axis: 'standards',
|
|
141
|
+
hardConvention: true,
|
|
142
|
+
location: `${file.path}:${lines[0]}`,
|
|
143
|
+
failureScenario: `${plain.length} comment line(s) added in product code; comments go stale, cannot be enforced, and agents copy them as license to explain instead of fix`,
|
|
144
|
+
evidence: `comment lines ${lines.slice(0, 12).join(', ')}${lines.length > 12 ? ', ...' : ''}; first: ${plain[0].text.trim().slice(0, 120)}`,
|
|
145
|
+
severity: 'HIGH',
|
|
146
|
+
validity: 'VALID',
|
|
147
|
+
disposition: 'WEAK',
|
|
148
|
+
concreteAlternativeDiff: `- ${plain[0].text.trim()}\n+ (delete it: put intent in names, types, tests or a lint rule; put rationale in the commit message or docs/)`,
|
|
149
|
+
source: NO_COMMENTS_SOURCE
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return findings;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function scanRunComments(rootDir, run) {
|
|
157
|
+
const rules = loadProjectRules(rootDir);
|
|
158
|
+
try {
|
|
159
|
+
return scanComments(fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8'), {
|
|
160
|
+
extraPatterns: (rules.justificationPatterns || []).map((text) => new RegExp(text, 'i')),
|
|
161
|
+
noComments: rules.comments !== 'allow'
|
|
162
|
+
});
|
|
163
|
+
} catch {
|
|
164
|
+
return [];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function mergeCommentFindings(ledger, findings) {
|
|
169
|
+
const errors = [];
|
|
170
|
+
const sources = new Set([SOURCE, NO_COMMENTS_SOURCE]);
|
|
171
|
+
const existing = new Map((ledger.findings || []).filter((item) => sources.has(item.source)).map((item) => [`${item.source}|${item.location}`, item]));
|
|
172
|
+
const additions = [];
|
|
173
|
+
for (const item of findings) {
|
|
174
|
+
const present = existing.get(`${item.source}|${item.location}`);
|
|
175
|
+
if (!present) {
|
|
176
|
+
additions.push(item);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const downgraded = present.validity !== 'VALID' || present.severity !== item.severity || (item.hardConvention && present.hardConvention !== true);
|
|
180
|
+
if (downgraded && !String(present.dismissal?.evidence || '').trim()) {
|
|
181
|
+
errors.push(`${present.id} (${present.location}) was downgraded without dismissal.evidence. FIX: restore it, or add dismissal: {by, evidence} explaining why this comment or suppression must stay.`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return { ledger: { ...ledger, findings: [...(ledger.findings || []), ...additions] }, added: additions.length, errors };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
module.exports = { NO_COMMENTS_SOURCE, SOURCE, commentOf, loadExtraPatterns, mergeCommentFindings, scanComments, scanRunComments };
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Feature map (C-22, "All Code Is Slop (Except Mine)" 10:06-10:46): a
|
|
4
|
+
// site map of what users can do, so agents stop guessing what "the thingy in
|
|
5
|
+
// the left sidebar" means and runtime verification has a shared vocabulary.
|
|
6
|
+
// .ai-engineering-loop/feature-map.md holds one Markdown table:
|
|
7
|
+
//
|
|
8
|
+
// | Feature | Where (UI or endpoint) | Paths | How to reach | Driver |
|
|
9
|
+
// |---|---|---|---|---|
|
|
10
|
+
// | export-csv | Reports > Export button | src/reports/**, src/export/** | open Reports, click Export | npm run drive -- export |
|
|
11
|
+
//
|
|
12
|
+
// `Paths` are globs (comma separated) used to pick the entries a diff
|
|
13
|
+
// touches; `Driver` is an optional reusable command that runs the flow.
|
|
14
|
+
|
|
15
|
+
const fs = require('fs');
|
|
16
|
+
const path = require('path');
|
|
17
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
18
|
+
|
|
19
|
+
const FEATURE_MAP_FILE = path.join('.ai-engineering-loop', 'feature-map.md');
|
|
20
|
+
|
|
21
|
+
function parseFeatureMap(text) {
|
|
22
|
+
const entries = [];
|
|
23
|
+
for (const raw of String(text || '').split('\n')) {
|
|
24
|
+
const line = raw.trim();
|
|
25
|
+
if (!line.startsWith('|') || /^\|\s*-/.test(line)) continue;
|
|
26
|
+
const cells = line.split('|').slice(1, -1).map((cell) => cell.trim());
|
|
27
|
+
if (cells.length < 4 || /^feature$/i.test(cells[0])) continue;
|
|
28
|
+
entries.push({
|
|
29
|
+
id: cells[0],
|
|
30
|
+
where: cells[1],
|
|
31
|
+
paths: cells[2].split(',').map((glob) => glob.trim().replace(/`/g, '')).filter(Boolean),
|
|
32
|
+
how: cells[3],
|
|
33
|
+
driver: (cells[4] || '').replace(/`/g, '').trim() || null
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return entries;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function loadFeatureMap(rootDir) {
|
|
40
|
+
try {
|
|
41
|
+
return parseFeatureMap(fs.readFileSync(path.join(rootDir, FEATURE_MAP_FILE), 'utf8'));
|
|
42
|
+
} catch {
|
|
43
|
+
return [];
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function featuresForFiles(rootDir, files) {
|
|
48
|
+
return loadFeatureMap(rootDir).filter((entry) => files.some((file) => entry.paths.some((glob) => matchesGlob(file, glob))));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function renderForPack(entries) {
|
|
52
|
+
if (!entries.length) return '';
|
|
53
|
+
return ['', '## Features this change touches (feature map)', '',
|
|
54
|
+
...entries.map((entry) => `- ${entry.id}: ${entry.where}. Reach: ${entry.how}.${entry.driver ? ` Driver: \`${entry.driver}\`` : ''}`)].join('\n');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
module.exports = { FEATURE_MAP_FILE, featuresForFiles, loadFeatureMap, parseFeatureMap, renderForPack };
|
package/lib/flow.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Goal `flow` (C-21, "All Code Is Slop (Except Mine)" 12:46): a small state
|
|
4
|
+
// machine from the user's point of view, so a human can check that the agent
|
|
5
|
+
// understood the intent before any code exists, and review it again next to
|
|
6
|
+
// the verification artifacts. Plain Markdown tables (no diagram engine), so
|
|
7
|
+
// it renders on every host and in every PR.
|
|
8
|
+
//
|
|
9
|
+
// "flow": [ { "from": "List", "event": "click Export", "to": "Exporting" }, ... ]
|
|
10
|
+
|
|
11
|
+
function flowProblems(flow) {
|
|
12
|
+
if (flow == null) return [];
|
|
13
|
+
if (!Array.isArray(flow) || !flow.length) return ['flow must be a non-empty array of {from, event, to}'];
|
|
14
|
+
const problems = [];
|
|
15
|
+
flow.forEach((step, index) => {
|
|
16
|
+
for (const field of ['from', 'event', 'to']) {
|
|
17
|
+
if (!String(step?.[field] || '').trim()) problems.push(`flow[${index}].${field} is required`);
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
return problems;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Warnings a reviewer should look at: states nothing leads to (besides the
|
|
24
|
+
// first), and states with no way out that are not marked terminal.
|
|
25
|
+
function flowWarnings(flow) {
|
|
26
|
+
if (flowProblems(flow).length || !flow) return [];
|
|
27
|
+
const start = flow[0].from;
|
|
28
|
+
const sources = new Set(flow.map((step) => step.from));
|
|
29
|
+
const targets = new Set(flow.map((step) => step.to));
|
|
30
|
+
const states = [...new Set(flow.flatMap((step) => [step.from, step.to]))];
|
|
31
|
+
const warnings = [];
|
|
32
|
+
for (const state of states) {
|
|
33
|
+
if (state !== start && !targets.has(state)) warnings.push(`"${state}" is never reached from another state`);
|
|
34
|
+
if (!sources.has(state) && !/^(done|end|closed|success|error|failed|cancel(l)?ed)$/i.test(state)) {
|
|
35
|
+
warnings.push(`"${state}" has no way out; mark it terminal (Done/Error) or add the next step`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return warnings;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const cell = (value) => String(value ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|');
|
|
42
|
+
|
|
43
|
+
function renderFlow(flow) {
|
|
44
|
+
if (!Array.isArray(flow) || !flow.length) return '';
|
|
45
|
+
return ['| From | Event | To |', '|---|---|---|', ...flow.map((step) => `| ${cell(step.from)} | ${cell(step.event)} | ${cell(step.to)} |`)].join('\n');
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// C-25: the same flow as a Mermaid stateDiagram-v2 (GitHub renders it in
|
|
49
|
+
// PRs). States get safe ids (s0, s1, ...) with the reader's label attached,
|
|
50
|
+
// so any text in a state name keeps the diagram valid. Visualisation only:
|
|
51
|
+
// model checking stays with the `formal` key (e.g. pstack /statechart).
|
|
52
|
+
const TERMINAL = /^(done|end|closed|success|error|failed|cancel(l)?ed)$/i;
|
|
53
|
+
|
|
54
|
+
function mermaidText(value) {
|
|
55
|
+
return String(value ?? '')
|
|
56
|
+
.replace(/[\r\n]+/g, ' ')
|
|
57
|
+
.replace(/"/g, "'")
|
|
58
|
+
.replace(/;/g, ',')
|
|
59
|
+
.replace(/#/g, 'no. ')
|
|
60
|
+
.replace(/[{}<>]/g, '')
|
|
61
|
+
.trim();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function renderMermaid(flow) {
|
|
65
|
+
if (!Array.isArray(flow) || !flow.length || flowProblems(flow).length) return '';
|
|
66
|
+
const ids = new Map();
|
|
67
|
+
const idOf = (state) => {
|
|
68
|
+
if (!ids.has(state)) ids.set(state, `s${ids.size}`);
|
|
69
|
+
return ids.get(state);
|
|
70
|
+
};
|
|
71
|
+
for (const step of flow) {
|
|
72
|
+
idOf(step.from);
|
|
73
|
+
idOf(step.to);
|
|
74
|
+
}
|
|
75
|
+
const sources = new Set(flow.map((step) => step.from));
|
|
76
|
+
const lines = ['stateDiagram-v2'];
|
|
77
|
+
for (const [state, id] of ids) lines.push(` state "${mermaidText(state)}" as ${id}`);
|
|
78
|
+
lines.push(` [*] --> ${idOf(flow[0].from)}`);
|
|
79
|
+
for (const step of flow) lines.push(` ${idOf(step.from)} --> ${idOf(step.to)}: ${mermaidText(step.event)}`);
|
|
80
|
+
for (const [state, id] of ids) {
|
|
81
|
+
if (!sources.has(state) && TERMINAL.test(state)) lines.push(` ${id} --> [*]`);
|
|
82
|
+
}
|
|
83
|
+
return lines.join('\n');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
module.exports = { flowProblems, flowWarnings, renderFlow, renderMermaid };
|
package/lib/gates.js
CHANGED
|
@@ -131,6 +131,7 @@ function validateGoalContract(contract) {
|
|
|
131
131
|
const errors = [];
|
|
132
132
|
if (!contract || typeof contract !== 'object') return { valid: false, errors: ['contract must be an object'] };
|
|
133
133
|
if (!String(contract.objective || '').trim()) errors.push('objective is required');
|
|
134
|
+
errors.push(...require('./flow.js').flowProblems(contract.flow)); // C-21
|
|
134
135
|
if (!Array.isArray(contract.acceptanceCriteria) || contract.acceptanceCriteria.length === 0) {
|
|
135
136
|
errors.push('acceptanceCriteria must contain at least one item');
|
|
136
137
|
} else {
|
|
@@ -517,6 +518,27 @@ function mergeVerificationIntegrity(rootDir, run) {
|
|
|
517
518
|
return merged.added;
|
|
518
519
|
}
|
|
519
520
|
|
|
521
|
+
// C-18: suppressions and justification comments added by the change.
|
|
522
|
+
function mergeCommentJustifications(rootDir, run) {
|
|
523
|
+
if (require('./ablation.js').ablated(rootDir, 'comments', run.runId)) return 0;
|
|
524
|
+
const { scanRunComments, mergeCommentFindings } = require('./comment-findings.js');
|
|
525
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
526
|
+
if (!fs.existsSync(filePath)) return 0;
|
|
527
|
+
let ledger;
|
|
528
|
+
try {
|
|
529
|
+
ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
530
|
+
} catch {
|
|
531
|
+
return 0;
|
|
532
|
+
}
|
|
533
|
+
const merged = mergeCommentFindings(ledger, scanRunComments(rootDir, run));
|
|
534
|
+
if (merged.errors.length) {
|
|
535
|
+
throw gateError('Review gate failed (comment or no-comments finding dropped)', merged.errors,
|
|
536
|
+
{ why: 'Comments and suppressions are how agents justify skipping the root cause, and new comments in product code are banned by default (C-26); dropping the finding silently hides that decision.', fix: 'Restore the finding, or record dismissal: {by, evidence} for each one.' });
|
|
537
|
+
}
|
|
538
|
+
if (merged.added) fs.writeFileSync(filePath, `${JSON.stringify(merged.ledger, null, 2)}\n`, { mode: 0o600 });
|
|
539
|
+
return merged.added;
|
|
540
|
+
}
|
|
541
|
+
|
|
520
542
|
function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
521
543
|
if (!Object.hasOwn(GATE_FILES, gate)) {
|
|
522
544
|
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
|
|
@@ -644,6 +666,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
644
666
|
}
|
|
645
667
|
mergeImpactDrift(rootDir, run);
|
|
646
668
|
mergeVerificationIntegrity(rootDir, run);
|
|
669
|
+
mergeCommentJustifications(rootDir, run);
|
|
647
670
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
648
671
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
649
672
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
package/lib/harness-audit.js
CHANGED
|
@@ -86,6 +86,12 @@ function auditRules(rootDir, now = new Date()) {
|
|
|
86
86
|
const items = [];
|
|
87
87
|
for (const rule of rules) {
|
|
88
88
|
if (!rule.from || !rule.why || !rule.fix) items.push({ id: 'rule-provenance', level: 'warn', detail: `${rule.id} lacks a source (from), why or fix. FIX: record where it came from and how to repair a violation.` });
|
|
89
|
+
if (rule.scope === 'diff') {
|
|
90
|
+
const backlog = require('./rules.js').legacyBacklog(rootDir, rule);
|
|
91
|
+
if (backlog.error) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id}: ${backlog.error}` });
|
|
92
|
+
else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
|
|
93
|
+
else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
|
|
94
|
+
}
|
|
89
95
|
const age = (now - new Date(rule.createdAt || 0)) / 86400000;
|
|
90
96
|
if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
|
|
91
97
|
}
|
|
@@ -111,9 +117,20 @@ function auditBudget(rootDir) {
|
|
|
111
117
|
}
|
|
112
118
|
}
|
|
113
119
|
|
|
120
|
+
// C-22: runtime verification without a feature map makes agents guess what
|
|
121
|
+
// to exercise.
|
|
122
|
+
function auditFeatureMap(rootDir) {
|
|
123
|
+
const lanes = require('./verification-lanes.js');
|
|
124
|
+
const commands = lanes.verificationCommands(rootDir);
|
|
125
|
+
if (!commands.e2e && !commands.smoke) return [];
|
|
126
|
+
if (require('./feature-map.js').loadFeatureMap(rootDir).length) return [];
|
|
127
|
+
return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
|
|
128
|
+
}
|
|
129
|
+
|
|
114
130
|
function auditHarness(rootDir, { budget = true } = {}) {
|
|
115
131
|
const items = [
|
|
116
132
|
...auditEntryFile(rootDir),
|
|
133
|
+
...auditFeatureMap(rootDir),
|
|
117
134
|
...auditLearnings(rootDir),
|
|
118
135
|
...auditRules(rootDir),
|
|
119
136
|
...(budget ? auditBudget(rootDir) : [])
|
package/lib/learn.js
CHANGED
|
@@ -255,6 +255,25 @@ function relevance(entry, files) {
|
|
|
255
255
|
return score;
|
|
256
256
|
}
|
|
257
257
|
|
|
258
|
+
// C-19 ("All Code Is Slop (Except Mine)", 06:46): agents over-rotate on a
|
|
259
|
+
// single correction and treat it as a hard rule. Confidence follows how many
|
|
260
|
+
// Runs produced the lesson; only rule-candidates read as firm guidance.
|
|
261
|
+
const PROMOTE_AFTER_RUNS = 3;
|
|
262
|
+
|
|
263
|
+
function confidenceOf(entry) {
|
|
264
|
+
const runs = (entry.runIds || []).length;
|
|
265
|
+
if (runs >= PROMOTE_AFTER_RUNS) return 'rule-candidate';
|
|
266
|
+
if (runs === 2) return 'recurring';
|
|
267
|
+
return 'tentative';
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Wording that matches the confidence: a one-off lesson is a hint, never a rule.
|
|
271
|
+
function guidanceFor(confidence, lesson) {
|
|
272
|
+
if (confidence === 'tentative') return `Consider (seen in 1 Run, not a rule): ${lesson}`;
|
|
273
|
+
if (confidence === 'recurring') return `Recurring (2 Runs), weigh it: ${lesson}`;
|
|
274
|
+
return `Repeated in ${PROMOTE_AFTER_RUNS}+ Runs (promote it to a rule): ${lesson}`;
|
|
275
|
+
}
|
|
276
|
+
|
|
258
277
|
// Top-K learnings for a context pack, greedily kept inside `budgetTokens`.
|
|
259
278
|
// Entries relevant to the pack's files rank first; project-wide recurring
|
|
260
279
|
// lessons (2+ runs) fill the rest.
|
|
@@ -269,11 +288,13 @@ function selectLearnings(rootDir, { files = [], budgetTokens = 0, limit = MAX_CO
|
|
|
269
288
|
let tokens = 0;
|
|
270
289
|
for (const { entry } of ranked) {
|
|
271
290
|
if (selected.length >= limit) break;
|
|
291
|
+
const confidence = confidenceOf(entry);
|
|
272
292
|
const item = {
|
|
273
293
|
id: entry.id,
|
|
274
294
|
axis: entry.axis,
|
|
275
295
|
severity: entry.severity,
|
|
276
|
-
|
|
296
|
+
confidence,
|
|
297
|
+
lesson: guidanceFor(confidence, safeText(entry.lesson)),
|
|
277
298
|
locations: entry.locations.slice(0, 5),
|
|
278
299
|
occurrences: entry.occurrences
|
|
279
300
|
};
|
|
@@ -291,6 +312,8 @@ module.exports = {
|
|
|
291
312
|
LEARNINGS_BUDGET_SHARE,
|
|
292
313
|
DEFAULT_MAX_AGE_RUNS,
|
|
293
314
|
isLesson,
|
|
315
|
+
confidenceOf,
|
|
316
|
+
guidanceFor,
|
|
294
317
|
learn,
|
|
295
318
|
loadLearnings,
|
|
296
319
|
selectLearnings,
|
package/lib/receipt.js
CHANGED
|
@@ -134,10 +134,27 @@ function coverageSummary(coverage) {
|
|
|
134
134
|
return numbers.length ? Object.fromEntries(numbers.sort(([a], [b]) => a.localeCompare(b))) : null;
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
// C-21: ACs whose proof is visual or crosses components need an artifact
|
|
138
|
+
// (video, screenshot, diagram) a reviewer can look at instead of the diff.
|
|
139
|
+
function needsArtifact(criterion) {
|
|
140
|
+
return criterion.crossComponent === true || (Array.isArray(criterion.rubric) && criterion.rubric.length > 0);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function artifactKind(url) {
|
|
144
|
+
const value = String(url).toLowerCase();
|
|
145
|
+
if (/\.(mp4|webm|mov|gif)(\?|$)|loom\.com|youtu/.test(value)) return 'video';
|
|
146
|
+
if (/\.(png|jpe?g|webp)(\?|$)/.test(value)) return 'screenshot';
|
|
147
|
+
if (/\.(svg|drawio)(\?|$)|mermaid|excalidraw|figma\.com/.test(value)) return 'diagram';
|
|
148
|
+
return 'link';
|
|
149
|
+
}
|
|
150
|
+
|
|
137
151
|
function criterionEntry(criterion, { commands, claims, links }) {
|
|
138
152
|
const claim = claims.find((row) => row.ac === criterion.id);
|
|
139
153
|
const evidence = [];
|
|
140
154
|
const missing = [];
|
|
155
|
+
if (needsArtifact(criterion) && !(links[criterion.id] || []).length) {
|
|
156
|
+
missing.push('no verification artifact (video, screenshot or diagram); pass --evidence ' + criterion.id + '=<https-url>');
|
|
157
|
+
}
|
|
141
158
|
if (claim && String(claim.reality || '').trim()) {
|
|
142
159
|
evidence.push({ type: 'claim', claimed: safeText(claim.claimed), reality: safeText(claim.reality) });
|
|
143
160
|
} else {
|
|
@@ -146,7 +163,7 @@ function criterionEntry(criterion, { commands, claims, links }) {
|
|
|
146
163
|
if (!commands.length) missing.push('no recorded verification command');
|
|
147
164
|
else if (commands.some((item) => !item.passed)) missing.push('a verification command did not pass');
|
|
148
165
|
evidence.push(...commands);
|
|
149
|
-
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', url });
|
|
166
|
+
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', kind: artifactKind(url), url });
|
|
150
167
|
return {
|
|
151
168
|
id: criterion.id,
|
|
152
169
|
statement: safeText(criterion.statement),
|
|
@@ -199,7 +216,8 @@ function buildReceipt(rootDir, { runId, host, model, head, evidenceLinks = [], a
|
|
|
199
216
|
sha256: hashBytes(fs.readFileSync(goalFile)),
|
|
200
217
|
contentHash: sha256(goal),
|
|
201
218
|
version: run.goal?.version || 1,
|
|
202
|
-
objective: safeText(goal.objective)
|
|
219
|
+
objective: safeText(goal.objective),
|
|
220
|
+
...(Array.isArray(goal.flow) && goal.flow.length ? { flow: goal.flow.map((step) => ({ from: safeText(step.from), event: safeText(step.event), to: safeText(step.to) })) } : {})
|
|
203
221
|
},
|
|
204
222
|
acceptanceCriteria: (goal.acceptanceCriteria || []).map((criterion) => criterionEntry(criterion, { commands, claims, links })),
|
|
205
223
|
verification: {
|
|
@@ -259,6 +277,17 @@ function renderReceiptMarkdown(receipt) {
|
|
|
259
277
|
`Run \`${receipt.runId}\` · Goal v${receipt.goal.version} \`${short(receipt.goal.contentHash)}\` · base \`${short(receipt.base.sha, 7)}\` → head \`${short(receipt.head.sha, 7)}\` · digest \`${short(receipt.digest)}\``,
|
|
260
278
|
''
|
|
261
279
|
];
|
|
280
|
+
// C-21: review artifacts first (talk 09:26: review shrinks to looking at
|
|
281
|
+
// videos, screenshots and diagrams), then the intended flow, then the rest.
|
|
282
|
+
const artifacts = receipt.acceptanceCriteria.flatMap((criterion) => criterion.evidence
|
|
283
|
+
.filter((item) => item.type === 'link')
|
|
284
|
+
.map((item) => `- **${criterion.id}** ${item.kind || 'link'}: [${item.kind || 'evidence'}](${item.url})`));
|
|
285
|
+
if (artifacts.length) lines.push('### Verification artifacts', '', ...artifacts, '');
|
|
286
|
+
if (receipt.goal.flow) {
|
|
287
|
+
const flowLib = require('./flow.js');
|
|
288
|
+
lines.push('### Intended flow', '', '```mermaid', flowLib.renderMermaid(receipt.goal.flow), '```', '', flowLib.renderFlow(receipt.goal.flow), '');
|
|
289
|
+
}
|
|
290
|
+
lines.push('### Acceptance criteria', '');
|
|
262
291
|
for (const criterion of receipt.acceptanceCriteria) {
|
|
263
292
|
const parts = [];
|
|
264
293
|
for (const item of criterion.evidence) {
|
|
@@ -358,6 +387,8 @@ function verifyReceipt(rootDir, { runId, receiptPath, head } = {}) {
|
|
|
358
387
|
}
|
|
359
388
|
|
|
360
389
|
module.exports = {
|
|
390
|
+
criterionEntry,
|
|
391
|
+
needsArtifact,
|
|
361
392
|
RECEIPT_FILE,
|
|
362
393
|
RECEIPT_MARKDOWN_FILE,
|
|
363
394
|
buildReceipt,
|
package/lib/reverts.js
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Revert detection (C-24, "All Code Is Slop (Except Mine)" 13:26-14:46):
|
|
4
|
+
// the revert rate of delivered work is a quality anchor that needs no human
|
|
5
|
+
// input. A git `Revert "<subject>"` commit is matched to a DELIVERED Run when
|
|
6
|
+
// it reverts that Run's receipt head (by sha) or a commit with the same
|
|
7
|
+
// subject (squash merges change the sha but keep the subject).
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { execFileSync } = require('child_process');
|
|
12
|
+
const { runsDir } = require('./run-state.js');
|
|
13
|
+
|
|
14
|
+
function git(rootDir, args) {
|
|
15
|
+
try {
|
|
16
|
+
return execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 16 * 1024 * 1024 });
|
|
17
|
+
} catch {
|
|
18
|
+
return '';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function deliveredRuns(rootDir) {
|
|
23
|
+
let names = [];
|
|
24
|
+
try {
|
|
25
|
+
names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
26
|
+
} catch {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
const runs = [];
|
|
30
|
+
for (const runId of names) {
|
|
31
|
+
const dir = path.join(runsDir(rootDir), runId);
|
|
32
|
+
let state = null;
|
|
33
|
+
let receipt = null;
|
|
34
|
+
try {
|
|
35
|
+
state = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
|
36
|
+
} catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (state.state !== 'DELIVERED') continue;
|
|
40
|
+
try {
|
|
41
|
+
receipt = JSON.parse(fs.readFileSync(path.join(dir, 'receipt.json'), 'utf8'));
|
|
42
|
+
} catch {
|
|
43
|
+
receipt = null;
|
|
44
|
+
}
|
|
45
|
+
const head = receipt?.head?.sha || null;
|
|
46
|
+
const subject = head ? git(rootDir, ['log', '-1', '--format=%s', head]).trim() : '';
|
|
47
|
+
runs.push({ runId, head, subject });
|
|
48
|
+
}
|
|
49
|
+
return runs;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function revertCommits(rootDir) {
|
|
53
|
+
const out = git(rootDir, ['log', '--all', '-E', '--grep=^Revert "', '--format=%H%x1f%s%x1f%b%x1e']);
|
|
54
|
+
return out.split('\x1e').map((chunk) => chunk.trim()).filter(Boolean).map((chunk) => {
|
|
55
|
+
const [sha, subject, body] = chunk.split('\x1f');
|
|
56
|
+
const reverted = (body || '').match(/This reverts commit ([0-9a-f]{7,40})/);
|
|
57
|
+
const original = (subject || '').match(/^Revert "(.+)"$/);
|
|
58
|
+
return { sha, revertedSha: reverted ? reverted[1] : null, originalSubject: original ? original[1] : null };
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function detectReverts(rootDir) {
|
|
63
|
+
const runs = deliveredRuns(rootDir);
|
|
64
|
+
const reverts = revertCommits(rootDir);
|
|
65
|
+
const reverted = [];
|
|
66
|
+
for (const run of runs) {
|
|
67
|
+
const hit = reverts.find((revert) => (run.head && revert.revertedSha && (run.head.startsWith(revert.revertedSha) || revert.revertedSha.startsWith(run.head)))
|
|
68
|
+
|| (run.subject && revert.originalSubject === run.subject));
|
|
69
|
+
if (hit) {
|
|
70
|
+
const files = run.head ? git(rootDir, ['show', '--name-only', '--format=', run.head]).split('\n').filter(Boolean) : [];
|
|
71
|
+
reverted.push({ runId: run.runId, revertSha: hit.sha, location: files[0] || run.runId });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return { delivered: runs.length, reverted, rate: runs.length ? Math.round((reverted.length / runs.length) * 1000) / 1000 : 0 };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { detectReverts };
|