ai-engineering-loop 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +4 -3
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +1 -1
- package/.agents/skills/ael/references/loop-maker.md +1 -0
- package/.claude/agents/devil-advocate.md +1 -1
- package/.grok/agents/devil-advocate.md +1 -1
- package/CHANGELOG.md +22 -0
- package/agents/shared/devil-advocate.body.md +1 -1
- package/bin/ai-engineering-loop.js +69 -6
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +18 -0
- package/core/verification-loop.md +38 -0
- package/lib/ablation.js +1 -1
- package/lib/ac-status.js +39 -7
- package/lib/budget.js +9 -1
- package/lib/calibration.js +17 -0
- package/lib/claimed-vs-reality.js +1 -1
- package/lib/cli-args.js +1 -1
- package/lib/comment-findings.js +187 -0
- package/lib/feature-map.js +57 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +23 -0
- package/lib/harness-audit.js +17 -0
- package/lib/learn.js +24 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +99 -6
- package/lib/stats.js +20 -0
- package/lib/verification-lanes.js +39 -7
- package/lib/verification-recorder.js +3 -3
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +22 -3
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Built-in rule kinds (C-20, "All Code Is Slop (Except Mine)" 05:24-06:46):
|
|
4
|
+
// the two corrections teams make most often, without hand-written grep.
|
|
5
|
+
// - forbid-import "<fromGlob> -> <toGlob>": files under fromGlob may not
|
|
6
|
+
// import modules under toGlob (relative paths are resolved; bare names
|
|
7
|
+
// such as `fs` or `react` are matched as written).
|
|
8
|
+
// - forbid-api "<regex>" except "<glob>": the API may only appear inside
|
|
9
|
+
// the excepted paths (for example a framework folder).
|
|
10
|
+
// Both judge only lines the change adds (the C-17 ratchet env).
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
15
|
+
|
|
16
|
+
const IMPORT_SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s+)['"]([^'"]+)['"]/g;
|
|
17
|
+
const EXTENSIONS = ['', '.js', '.ts', '.tsx', '.jsx', '.mjs', '.cjs', '/index.js', '/index.ts'];
|
|
18
|
+
|
|
19
|
+
function parseImportSpec(spec) {
|
|
20
|
+
const parts = String(spec || '').split('->').map((part) => part.trim());
|
|
21
|
+
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
22
|
+
throw Object.assign(new Error('--forbid-import needs "<fromGlob> -> <toGlob>", e.g. "src/renderer/** -> src/main/**"'), { code: 'INVALID_USAGE' });
|
|
23
|
+
}
|
|
24
|
+
return { from: parts[0], to: parts[1] };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function readAddedLines(env = process.env) {
|
|
28
|
+
const file = env.AEL_ADDED_LINES_FILE;
|
|
29
|
+
if (!file) return [];
|
|
30
|
+
let text = '';
|
|
31
|
+
try {
|
|
32
|
+
text = fs.readFileSync(file, 'utf8');
|
|
33
|
+
} catch {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return text.split('\n').filter(Boolean).flatMap((raw) => {
|
|
37
|
+
const match = raw.match(/^(.+?):(\d+):(.*)$/);
|
|
38
|
+
return match ? [{ file: match[1].replace(/\\/g, '/'), line: Number(match[2]), text: match[3] }] : [];
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function importTargets(file, text) {
|
|
43
|
+
const targets = [];
|
|
44
|
+
for (const match of text.matchAll(IMPORT_SPECIFIER)) {
|
|
45
|
+
const spec = match[1];
|
|
46
|
+
if (spec.startsWith('.')) {
|
|
47
|
+
const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(file), spec));
|
|
48
|
+
targets.push({ spec, candidates: EXTENSIONS.map((ext) => `${resolved}${ext}`) });
|
|
49
|
+
} else {
|
|
50
|
+
targets.push({ spec, candidates: [spec] });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return targets;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// [{ file, line, message }]
|
|
57
|
+
function checkRule(rule, lines) {
|
|
58
|
+
const violations = [];
|
|
59
|
+
if (rule.kind === 'forbid-import') {
|
|
60
|
+
const { from, to } = parseImportSpec(rule.spec);
|
|
61
|
+
for (const added of lines) {
|
|
62
|
+
if (!matchesGlob(added.file, from)) continue;
|
|
63
|
+
for (const target of importTargets(added.file, added.text)) {
|
|
64
|
+
if (target.candidates.some((candidate) => matchesGlob(candidate, to) || candidate === to)) {
|
|
65
|
+
violations.push({ file: added.file, line: added.line, message: `imports "${target.spec}" (${to}), forbidden from ${from}` });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
} else if (rule.kind === 'forbid-api') {
|
|
70
|
+
const pattern = new RegExp(rule.pattern);
|
|
71
|
+
for (const added of lines) {
|
|
72
|
+
if (rule.except && matchesGlob(added.file, rule.except)) continue;
|
|
73
|
+
if (pattern.test(added.text)) {
|
|
74
|
+
violations.push({ file: added.file, line: added.line, message: `uses /${rule.pattern}/ outside ${rule.except || 'the allowed paths'}` });
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
} else {
|
|
78
|
+
throw Object.assign(new Error(`Rule ${rule.id} has no built-in kind`), { code: 'INVALID_RULE' });
|
|
79
|
+
}
|
|
80
|
+
return violations;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Shell text that runs `rules check <id>` with this very CLI, so a rules.json
|
|
84
|
+
// shared across machines never stores a machine-specific path.
|
|
85
|
+
function builtinCommand(rule) {
|
|
86
|
+
const bin = path.join(__dirname, '..', 'bin', 'ai-engineering-loop.js');
|
|
87
|
+
return `"${process.execPath}" "${bin}" rules check ${rule.id}`;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = { builtinCommand, checkRule, importTargets, parseImportSpec, readAddedLines };
|
package/lib/rules.js
CHANGED
|
@@ -34,8 +34,30 @@ function loadLearningEntries(rootDir) {
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
const SCOPES = Object.freeze(['repo', 'diff']);
|
|
38
|
+
|
|
39
|
+
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
|
|
40
|
+
if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
|
|
41
|
+
// C-20 built-in kinds: no hand-written check, always judged on added lines.
|
|
42
|
+
let builtin = null;
|
|
43
|
+
if (forbidImport && forbidApi) throw rulesError('Use one of --forbid-import or --forbid-api per rule.', 'INVALID_USAGE');
|
|
44
|
+
if (forbidImport) {
|
|
45
|
+
require('./rule-check.js').parseImportSpec(forbidImport);
|
|
46
|
+
builtin = { kind: 'forbid-import', spec: String(forbidImport).trim() };
|
|
47
|
+
} else if (forbidApi) {
|
|
48
|
+
try {
|
|
49
|
+
new RegExp(forbidApi);
|
|
50
|
+
} catch (cause) {
|
|
51
|
+
throw rulesError(`--forbid-api is not a valid regex: ${cause.message}`, 'INVALID_USAGE');
|
|
52
|
+
}
|
|
53
|
+
builtin = { kind: 'forbid-api', pattern: String(forbidApi), ...(except ? { except: String(except).trim() } : {}) };
|
|
54
|
+
}
|
|
55
|
+
if (builtin) {
|
|
56
|
+
check = `ael rules check R-${learningId.replace(/^L-/, '')}`;
|
|
57
|
+
scope = 'diff';
|
|
58
|
+
shell = true;
|
|
59
|
+
}
|
|
60
|
+
if (!String(check || '').trim()) throw rulesError('--check "<command>" (or --forbid-import / --forbid-api) is required: the check must exit non-zero on a violation.', 'INVALID_USAGE');
|
|
39
61
|
if (!String(fix || '').trim()) throw rulesError('--fix "<instruction>" is required: tell the agent exactly how to repair a violation.', 'INVALID_USAGE');
|
|
40
62
|
if (!String(by || '').trim()) throw rulesError('--by <name> is required: promoting a rule is a human decision.', 'INVALID_USAGE');
|
|
41
63
|
const learning = loadLearningEntries(rootDir).find((entry) => entry.id === learningId);
|
|
@@ -52,6 +74,8 @@ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shel
|
|
|
52
74
|
why: why ? String(why).trim() : learning.lesson,
|
|
53
75
|
fix: String(fix).trim(),
|
|
54
76
|
layer: 'L1',
|
|
77
|
+
scope,
|
|
78
|
+
...(builtin || {}),
|
|
55
79
|
createdBy: String(by).trim(),
|
|
56
80
|
createdAt: now.toISOString()
|
|
57
81
|
};
|
|
@@ -67,9 +91,69 @@ function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
|
|
|
67
91
|
.map((entry) => ({ id: entry.id, runs: entry.runIds.length, lesson: entry.lesson }));
|
|
68
92
|
}
|
|
69
93
|
|
|
70
|
-
//
|
|
71
|
-
|
|
72
|
-
|
|
94
|
+
// C-17 ratchet: a `diff` rule judges only what this change adds. Its check
|
|
95
|
+
// receives AEL_CHANGED_FILES (one path per line) and AEL_ADDED_LINES_FILE
|
|
96
|
+
// (`path:line:text` per added line, the format of `grep -n`), so
|
|
97
|
+
// `! grep -E "<pattern>" "$AEL_ADDED_LINES_FILE"` flags new occurrences only.
|
|
98
|
+
function writeAddedLines(rootDir, run) {
|
|
99
|
+
const { parseDiff } = require('./verification-integrity.js');
|
|
100
|
+
let diffText = '';
|
|
101
|
+
try {
|
|
102
|
+
diffText = fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8');
|
|
103
|
+
} catch {
|
|
104
|
+
diffText = '';
|
|
105
|
+
}
|
|
106
|
+
const files = parseDiff(diffText).filter((file) => !file.deleted);
|
|
107
|
+
const lines = files.flatMap((file) => file.added.map((added) => `${file.path}:${added.line}:${added.text}`));
|
|
108
|
+
const dir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
|
|
109
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
110
|
+
const addedPath = path.join(dir, 'rule-added-lines.txt');
|
|
111
|
+
fs.writeFileSync(addedPath, lines.length ? `${lines.join('\n')}\n` : '');
|
|
112
|
+
return { AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Built-in kinds run through this CLI (`rules check <id>`); others run their
|
|
116
|
+
// stored check.
|
|
117
|
+
function commandForRule(rule) {
|
|
118
|
+
return rule.kind ? require('./rule-check.js').builtinCommand(rule) : rule.check;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Default-plan steps for every rule (L1, shell by default). `run` enables
|
|
122
|
+
// the diff env for `scope: "diff"` rules.
|
|
123
|
+
function rulePlan(rootDir, run = null) {
|
|
124
|
+
const rules = loadRules(rootDir).rules;
|
|
125
|
+
const diffEnv = run && rules.some((rule) => rule.scope === 'diff') ? writeAddedLines(rootDir, run) : null;
|
|
126
|
+
return rules.map((rule) => ({
|
|
127
|
+
key: `rule:${rule.id}`,
|
|
128
|
+
layer: 'L1',
|
|
129
|
+
text: commandForRule(rule),
|
|
130
|
+
shell: rule.kind ? true : rule.shell !== false,
|
|
131
|
+
rule,
|
|
132
|
+
...(rule.scope === 'diff' ? { env: diffEnv || { AEL_CHANGED_FILES: '', AEL_ADDED_LINES_FILE: '' } } : {})
|
|
133
|
+
}));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Legacy backlog of a `diff` rule: run its check against every tracked line
|
|
137
|
+
// (as if all were added). Exit 0 = clean, ready to switch to `repo`.
|
|
138
|
+
function legacyBacklog(rootDir, rule, { maxBytes = 32 * 1024 * 1024 } = {}) {
|
|
139
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
140
|
+
let dump = '';
|
|
141
|
+
try {
|
|
142
|
+
dump = execFileSync('git', ['grep', '-n', '-I', '-e', '', '--', '.', ':(exclude).ai-engineering-loop'], { cwd: rootDir, encoding: 'utf8', maxBuffer: maxBytes });
|
|
143
|
+
} catch (cause) {
|
|
144
|
+
if (cause.status !== 1) return { rule: rule.id, error: `git grep failed: ${cause.message.split('\n')[0]}` };
|
|
145
|
+
}
|
|
146
|
+
const os = require('os');
|
|
147
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-backlog-'));
|
|
148
|
+
const addedPath = path.join(tmp, 'all-lines.txt');
|
|
149
|
+
fs.writeFileSync(addedPath, dump);
|
|
150
|
+
const files = [...new Set(dump.split('\n').filter(Boolean).map((line) => line.split(':')[0]))];
|
|
151
|
+
const result = rule.kind || rule.shell !== false
|
|
152
|
+
? spawnSync(commandForRule(rule), { cwd: rootDir, shell: true, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath }, maxBuffer: maxBytes })
|
|
153
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath } });
|
|
154
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
155
|
+
const output = `${result.stdout || ''}${result.stderr || ''}`.split('\n').filter((line) => line.trim());
|
|
156
|
+
return { rule: rule.id, clean: result.status === 0, outputLines: output.length, sample: output.slice(0, 3) };
|
|
73
157
|
}
|
|
74
158
|
|
|
75
159
|
// Shell commands are recorded JSON-quoted; compare on the unquoted text.
|
|
@@ -87,7 +171,11 @@ function plainCommand(text) {
|
|
|
87
171
|
|
|
88
172
|
function ruleForCommand(rootDir, commandText) {
|
|
89
173
|
const value = plainCommand(commandText);
|
|
90
|
-
return loadRules(rootDir).rules.find((rule) => rule.check === value) || null;
|
|
174
|
+
return loadRules(rootDir).rules.find((rule) => rule.check === value || commandForRule(rule) === value) || null;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function findRule(rootDir, id) {
|
|
178
|
+
return loadRules(rootDir).rules.find((rule) => rule.id === id) || null;
|
|
91
179
|
}
|
|
92
180
|
|
|
93
181
|
// Rules with no passing record in a full-scope bundle.
|
|
@@ -101,6 +189,11 @@ function missingRules(rootDir, bundle) {
|
|
|
101
189
|
}
|
|
102
190
|
|
|
103
191
|
module.exports = {
|
|
192
|
+
SCOPES,
|
|
193
|
+
commandForRule,
|
|
194
|
+
findRule,
|
|
195
|
+
legacyBacklog,
|
|
196
|
+
writeAddedLines,
|
|
104
197
|
PROMOTE_AFTER_RUNS,
|
|
105
198
|
RULES_FILE,
|
|
106
199
|
loadRules,
|
package/lib/stats.js
CHANGED
|
@@ -60,7 +60,25 @@ function computeStats(rootDir, { last = null } = {}) {
|
|
|
60
60
|
const rate = (n, d) => (d ? Math.round((n / d) * 1000) / 1000 : 0);
|
|
61
61
|
for (const bucket of Object.values(tiers)) bucket.falseVictoryRate = rate(bucket.falseVictories, bucket.judged);
|
|
62
62
|
const topCauses = Object.entries(causes).sort((a, b) => b[1] - a[1]).slice(0, 2).map(([cause, count]) => ({ cause, count }));
|
|
63
|
+
// C-24: revert rate from git, token share per role from the usage ledger.
|
|
64
|
+
const reverts = require('./reverts.js').detectReverts(rootDir);
|
|
65
|
+
const tokensByRole = {};
|
|
66
|
+
try {
|
|
67
|
+
for (const entry of require('./budget.js').readAllUsage(rootDir)) {
|
|
68
|
+
const role = entry.role || 'unlabeled';
|
|
69
|
+
tokensByRole[role] = (tokensByRole[role] || 0) + entry.totalTokens;
|
|
70
|
+
}
|
|
71
|
+
} catch {
|
|
72
|
+
// Unreadable ledger: report no token split rather than fail stats.
|
|
73
|
+
}
|
|
74
|
+
const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
|
|
75
|
+
const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
|
|
63
76
|
return {
|
|
77
|
+
revertRate: reverts.rate,
|
|
78
|
+
reverted: reverts.reverted,
|
|
79
|
+
delivered: reverts.delivered,
|
|
80
|
+
tokensByRole,
|
|
81
|
+
verificationTokenShare: labeled ? rate(checking, labeled) : null,
|
|
64
82
|
runs: runs.length,
|
|
65
83
|
judged,
|
|
66
84
|
falseVictories,
|
|
@@ -80,6 +98,8 @@ function formatStats(stats) {
|
|
|
80
98
|
`Iterations to PASS (avg): ${stats.iterationsToPass ?? 'n/a'}`,
|
|
81
99
|
`Failure causes: ${Object.entries(stats.causes).map(([cause, count]) => `${cause} ${count}`).join(', ') || 'none'}`
|
|
82
100
|
];
|
|
101
|
+
lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
|
|
102
|
+
if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
|
|
83
103
|
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
84
104
|
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
85
105
|
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
@@ -73,20 +73,50 @@ function relatedCommand(rootDir, run, { split, shell = false } = {}) {
|
|
|
73
73
|
return { ...parsed, args, files };
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
// Verification
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
76
|
+
// Verification catalog (#110, C-23): every kind of proof a project may
|
|
77
|
+
// configure in verification.md, the layer it proves, and whether the default
|
|
78
|
+
// plan runs it. AEL does not ship these tools; it recognises them, orders
|
|
79
|
+
// them, binds their evidence, and lets an AC require one (`verify.key`).
|
|
80
|
+
// Layers: L1 static, L2 runtime, L3 end-to-end, LF formal (model checking,
|
|
81
|
+
// proofs; slow, so never in the default plan).
|
|
82
|
+
const VERIFICATION_CATALOG = Object.freeze({
|
|
83
|
+
build: { layer: 'L1', plan: true, what: 'compiles or bundles' },
|
|
84
|
+
typecheck: { layer: 'L1', plan: true, what: 'static types' },
|
|
85
|
+
lint: { layer: 'L1', plan: true, what: 'lint and static analysis' },
|
|
86
|
+
security: { layer: 'L1', plan: false, what: 'dependency or static security scan' },
|
|
87
|
+
test_unit: { layer: 'L2', plan: true, what: 'unit tests' },
|
|
88
|
+
test_related: { layer: 'L2', plan: false, what: 'tests related to changed files' },
|
|
89
|
+
test_all: { layer: 'L2', plan: false, what: 'full regression suite' },
|
|
90
|
+
property: { layer: 'L2', plan: true, what: 'property-based tests' },
|
|
91
|
+
fuzz: { layer: 'L2', plan: false, what: 'fuzz tests' },
|
|
92
|
+
mutation: { layer: 'L2', plan: false, what: 'mutation tests (do the tests catch broken code?)' },
|
|
93
|
+
coverage: { layer: 'L2', plan: false, what: 'coverage threshold' },
|
|
94
|
+
benchmark: { layer: 'L2', plan: false, what: 'benchmark or performance tests' },
|
|
95
|
+
smoke: { layer: 'L2', plan: false, what: 'start, probe readiness, stop' },
|
|
96
|
+
contract: { layer: 'L3', plan: false, what: 'API or consumer contract tests' },
|
|
97
|
+
e2e: { layer: 'L3', plan: false, what: 'end-to-end user flows' },
|
|
98
|
+
pentest: { layer: 'L3', plan: false, what: 'penetration or DAST scan' },
|
|
99
|
+
formal: { layer: 'LF', plan: false, what: 'formal verification (TLA+, Lean, model checking)' }
|
|
81
100
|
});
|
|
101
|
+
const LAYER_OF_KEY = Object.freeze(Object.fromEntries(Object.entries(VERIFICATION_CATALOG).map(([key, entry]) => [key, entry.layer])));
|
|
82
102
|
// Cheapest-first default plan (templates/repo-config/verification.md
|
|
83
|
-
// "Default order"): L1 static checks, then L2 unit tests.
|
|
84
|
-
const DEFAULT_PLAN_KEYS = Object.freeze(
|
|
103
|
+
// "Default order"): L1 static checks, then L2 unit and property tests.
|
|
104
|
+
const DEFAULT_PLAN_KEYS = Object.freeze(Object.keys(VERIFICATION_CATALOG).filter((key) => VERIFICATION_CATALOG[key].plan));
|
|
85
105
|
|
|
86
106
|
function layerOfKey(key) {
|
|
87
107
|
return LAYER_OF_KEY[key] || null;
|
|
88
108
|
}
|
|
89
109
|
|
|
110
|
+
// verification.md key of a recorded command string (null when unmapped).
|
|
111
|
+
function keyOfCommand(rootDir, commandText) {
|
|
112
|
+
const value = String(commandText || '').trim();
|
|
113
|
+
for (const [key, text] of Object.entries(verificationCommands(rootDir))) {
|
|
114
|
+
const base = text.replace(FILES_TOKEN, '').trim();
|
|
115
|
+
if (value === text || (base && value.startsWith(base))) return key;
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
90
120
|
// Ordered [{ key, layer, text }] for the configured default plan. A command
|
|
91
121
|
// marked `# Optional` is still run when present.
|
|
92
122
|
// Commands that rewrite files (`eslint --fix`, `prettier --write`) would
|
|
@@ -163,6 +193,8 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
163
193
|
}
|
|
164
194
|
|
|
165
195
|
module.exports = {
|
|
196
|
+
VERIFICATION_CATALOG,
|
|
197
|
+
keyOfCommand,
|
|
166
198
|
DEFAULT_PLAN_KEYS,
|
|
167
199
|
LAYER_OF_KEY,
|
|
168
200
|
defaultCommandPlan,
|
|
@@ -134,7 +134,7 @@ function displayCommand(command, args) {
|
|
|
134
134
|
return redactHomePaths(redactSecrets(rendered).text);
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
-
function runCommand(command, args, { cwd, timeoutMs, shell }) {
|
|
137
|
+
function runCommand(command, args, { cwd, timeoutMs, shell, env: extraEnv = null }) {
|
|
138
138
|
return new Promise((resolve) => {
|
|
139
139
|
const started = new Date();
|
|
140
140
|
let timedOut = false;
|
|
@@ -147,7 +147,7 @@ function runCommand(command, args, { cwd, timeoutMs, shell }) {
|
|
|
147
147
|
const detached = process.platform !== 'win32';
|
|
148
148
|
const child = spawn(target.command, target.args, {
|
|
149
149
|
cwd,
|
|
150
|
-
env: process.env,
|
|
150
|
+
env: extraEnv ? { ...process.env, ...extraEnv } : process.env,
|
|
151
151
|
shell: Boolean(shell),
|
|
152
152
|
detached,
|
|
153
153
|
windowsHide: true,
|
|
@@ -326,7 +326,7 @@ async function recordVerificationCommands(rootDir, {
|
|
|
326
326
|
const previous = fresh ? null : bundle.commands.find((record) => record.command === display && passedRecord(record)
|
|
327
327
|
&& (record.scope || 'full') === (scope || 'full'));
|
|
328
328
|
if (previous) return { ...previous, reused: true };
|
|
329
|
-
const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell) });
|
|
329
|
+
const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell), env: item.env || null });
|
|
330
330
|
const tagged = { ...result, ...(item.layer ? { layer: item.layer } : {}), ...(item.key ? { key: item.key } : {}) };
|
|
331
331
|
return scope ? { ...tagged, scope } : tagged;
|
|
332
332
|
};
|
package/package.json
CHANGED
|
@@ -66,12 +66,17 @@
|
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
68
|
"verify": {
|
|
69
|
-
"description": "Optional (C-07).
|
|
69
|
+
"description": "Optional (C-07, C-23). What proves this AC: an exact `command`, a verification catalog `key` (e.g. pentest, property, formal; default verify runs it when configured), or a `layer` (any passing command of that layer). Gate judge refuses PASS until it passed on the gated diff.",
|
|
70
70
|
"type": "object",
|
|
71
|
-
"
|
|
71
|
+
"anyOf": [
|
|
72
|
+
{ "required": ["command"] },
|
|
73
|
+
{ "required": ["key"] },
|
|
74
|
+
{ "required": ["layer"] }
|
|
75
|
+
],
|
|
72
76
|
"properties": {
|
|
73
77
|
"command": { "type": "string", "minLength": 1 },
|
|
74
|
-
"
|
|
78
|
+
"key": { "enum": ["build", "typecheck", "lint", "security", "test_unit", "test_related", "test_all", "property", "fuzz", "mutation", "coverage", "benchmark", "smoke", "contract", "e2e", "pentest", "formal"] },
|
|
79
|
+
"layer": { "enum": ["L1", "L2", "L3", "LF"] }
|
|
75
80
|
},
|
|
76
81
|
"additionalProperties": false
|
|
77
82
|
},
|
|
@@ -130,6 +135,20 @@
|
|
|
130
135
|
},
|
|
131
136
|
"additionalProperties": false
|
|
132
137
|
},
|
|
138
|
+
"flow": {
|
|
139
|
+
"description": "Optional (C-21): the intended user flow as transitions. goal draft renders it as a From | Event | To table for the human to check before freeze; the receipt shows it next to the verification artifacts.",
|
|
140
|
+
"type": "array",
|
|
141
|
+
"minItems": 1,
|
|
142
|
+
"items": {
|
|
143
|
+
"type": "object",
|
|
144
|
+
"required": ["from", "event", "to"],
|
|
145
|
+
"properties": {
|
|
146
|
+
"from": { "type": "string", "minLength": 1 },
|
|
147
|
+
"event": { "type": "string", "minLength": 1 },
|
|
148
|
+
"to": { "type": "string", "minLength": 1 }
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
},
|
|
133
152
|
"featureId": {
|
|
134
153
|
"description": "Optional (C-15): id of the feature in .ai-engineering-loop/features.json this Goal delivers. Goal freeze marks it active; delivery marks it passing with this Run as evidence.",
|
|
135
154
|
"type": "string",
|