ai-engineering-loop 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +47 -11
- package/.claude/commands/ai-engineering-loop.md +1 -1
- package/.claude/skills/ai-engineering-loop/SKILL.md +26 -44
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +20 -0
- package/.claude/skills/ai-engineering-loop/references/loop-deliver.md +8 -0
- package/.claude/skills/ai-engineering-loop/references/loop-goal.md +10 -0
- package/.claude/skills/ai-engineering-loop/references/loop-maker.md +8 -0
- package/.claude/skills/ai-engineering-loop/references/loop-review.md +18 -0
- package/.claude/skills/ai-engineering-loop/references/loop-risky.md +9 -0
- package/.claude/skills/ai-engineering-loop/references/loop-verify.md +20 -0
- package/.gemini/skills/ai-engineering-loop/SKILL.md +24 -37
- package/.gemini/skills/ai-engineering-loop/references/loop-commands.md +20 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-deliver.md +8 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-goal.md +10 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-maker.md +8 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-review.md +18 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-risky.md +9 -0
- package/.gemini/skills/ai-engineering-loop/references/loop-verify.md +20 -0
- package/.grok/skills/ai-engineering-loop/SKILL.md +29 -61
- package/.grok/skills/ai-engineering-loop/references/loop-commands.md +20 -0
- package/.grok/skills/ai-engineering-loop/references/loop-deliver.md +8 -0
- package/.grok/skills/ai-engineering-loop/references/loop-goal.md +10 -0
- package/.grok/skills/ai-engineering-loop/references/loop-maker.md +8 -0
- package/.grok/skills/ai-engineering-loop/references/loop-review.md +18 -0
- package/.grok/skills/ai-engineering-loop/references/loop-risky.md +9 -0
- package/.grok/skills/ai-engineering-loop/references/loop-verify.md +20 -0
- package/CHANGELOG.md +69 -0
- package/README.md +52 -1
- package/README.npm.md +6 -2
- package/THIRD_PARTY_NOTICES.md +29 -0
- package/adapters/dot/skills/dot-dev-skill-router/SKILL.md +24 -16
- package/adapters/project-onboarding/SKILL.md +27 -0
- package/adapters/project-onboarding/references/output.md +29 -0
- package/adapters/quality/skills/ael-code-simplifier/SKILL.md +43 -0
- package/adapters/quality/skills/ael-receiving-code-review/SKILL.md +37 -0
- package/bin/ai-engineering-loop.js +1137 -78
- package/core/artifact-gates.md +32 -1
- package/core/root-cause-analysis.md +6 -0
- package/core/runtime-safety.md +32 -4
- package/core/verification-loop.md +6 -0
- package/evals/cases/21-tier-t0-docs.json +34 -0
- package/evals/cases/22-tier-t1-small-pass.json +30 -0
- package/evals/cases/23-tier-t1-blocker.json +26 -0
- package/evals/cases/24-tier-t1-auth-escalates.json +30 -0
- package/lib/agent-artifacts.js +250 -0
- package/lib/auto-update.js +486 -0
- package/lib/budget.js +31 -1
- package/lib/claimed-vs-reality.js +134 -0
- package/lib/cli-args.js +84 -0
- package/lib/docs.js +91 -0
- package/lib/doctor.js +7 -1
- package/lib/gates.js +114 -1
- package/lib/host-skill-build.js +121 -0
- package/lib/migrations.js +84 -0
- package/lib/project-registry.js +148 -0
- package/lib/run-state.js +6 -3
- package/lib/safe-context.js +62 -4
- package/lib/sync-hosts.js +356 -28
- package/lib/tier.js +134 -0
- package/lib/update-manager.js +17 -1
- package/lib/update-notice.js +192 -0
- package/lib/user-state.js +80 -0
- package/lib/verification-recorder.js +268 -0
- package/lib/workflow-runtime.js +26 -1
- package/migrations/README.md +21 -0
- package/package.json +7 -2
- package/policies/review-budget.md +1 -1
- package/schemas/verification.schema.json +2 -2
package/lib/cli-args.js
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Side-effect-free argument checks that run before any command handler.
|
|
4
|
+
// Contract: `--help`/`-h` never executes a command, and commands that write
|
|
5
|
+
// files reject unknown flags before touching the filesystem.
|
|
6
|
+
|
|
7
|
+
const HELP_FLAGS = new Set(['-h', '--help']);
|
|
8
|
+
|
|
9
|
+
// Commands whose handlers write files (repository, home, or package installs).
|
|
10
|
+
// `flags` take no value; `valued` consume the next token; `positionals` lists
|
|
11
|
+
// accepted bare words (null = free text allowed).
|
|
12
|
+
const WRITER_COMMANDS = Object.freeze({
|
|
13
|
+
init: { flags: [], valued: [], positionals: [] },
|
|
14
|
+
refresh: { flags: ['--non-destructive', '--json'], valued: [], positionals: [] },
|
|
15
|
+
'sync-hosts': { flags: ['--dry-run', '--prune', '--json'], valued: ['--pack'], positionals: [] },
|
|
16
|
+
update: { flags: ['--json', '--yes', '--background', '--force', '--no-pin'], valued: ['--channel', '--pin'], positionals: ['check', 'plan', 'apply', 'auto', 'policy', 'rollback', ...['off', 'notify', 'patch', 'minor', 'all']] },
|
|
17
|
+
projects: { flags: ['--json'], valued: ['--root', '--depth'], positionals: null },
|
|
18
|
+
migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
|
|
19
|
+
verify: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
20
|
+
'review-pack': { flags: ['--json'], valued: ['--run', '--base', '--profile'], positionals: [] },
|
|
21
|
+
deliver: { flags: ['--json', '--approve'], valued: ['--run', '--destination', '--summary', '--by'], positionals: [] },
|
|
22
|
+
'generate-adapter': { flags: [], valued: ['--type'], positionals: [] },
|
|
23
|
+
'generate-workflow': { flags: ['--write'], valued: [], positionals: [] }
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
function beforeSeparator(args) {
|
|
27
|
+
const index = args.indexOf('--');
|
|
28
|
+
return index === -1 ? args : args.slice(0, index);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function wantsCommandHelp(command, rest) {
|
|
32
|
+
const scoped = beforeSeparator(rest);
|
|
33
|
+
// `run` takes free-text task words; only a leading help flag counts.
|
|
34
|
+
if (command === 'run') return scoped.length > 0 && HELP_FLAGS.has(scoped[0]);
|
|
35
|
+
return scoped.some((arg) => HELP_FLAGS.has(arg));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function unknownWriterArgs(command, rest) {
|
|
39
|
+
const spec = WRITER_COMMANDS[command];
|
|
40
|
+
if (!spec) return [];
|
|
41
|
+
const scoped = beforeSeparator(rest);
|
|
42
|
+
const unknown = [];
|
|
43
|
+
for (let index = 0; index < scoped.length; index += 1) {
|
|
44
|
+
const arg = scoped[index];
|
|
45
|
+
if (arg.startsWith('-')) {
|
|
46
|
+
const [name] = arg.split('=');
|
|
47
|
+
if (spec.valued.includes(name)) {
|
|
48
|
+
if (!arg.includes('=')) index += 1;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (spec.flags.includes(arg)) continue;
|
|
52
|
+
unknown.push(arg);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (spec.positionals === null) continue;
|
|
56
|
+
if (!spec.positionals.includes(arg)) unknown.push(arg);
|
|
57
|
+
}
|
|
58
|
+
return unknown;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function commandHelp(helpText, command) {
|
|
62
|
+
const lines = String(helpText).split('\n');
|
|
63
|
+
const start = lines.findIndex((line) => {
|
|
64
|
+
const match = line.match(/^ (\S+)/);
|
|
65
|
+
return match && match[1] === command;
|
|
66
|
+
});
|
|
67
|
+
if (start === -1) return null;
|
|
68
|
+
const block = [lines[start]];
|
|
69
|
+
for (let index = start + 1; index < lines.length; index += 1) {
|
|
70
|
+
const line = lines[index];
|
|
71
|
+
if (/^ \S/.test(line) || /^\S/.test(line) || line.trim() === '') break;
|
|
72
|
+
block.push(line);
|
|
73
|
+
}
|
|
74
|
+
return block.join('\n');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = {
|
|
78
|
+
HELP_FLAGS,
|
|
79
|
+
WRITER_COMMANDS,
|
|
80
|
+
beforeSeparator,
|
|
81
|
+
wantsCommandHelp,
|
|
82
|
+
unknownWriterArgs,
|
|
83
|
+
commandHelp
|
|
84
|
+
};
|
package/lib/docs.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Read-only access to packaged specs. Host skills cite `ai-engineering-loop docs <name>`
|
|
4
|
+
// instead of repository-relative paths such as `core/grill-policy.md`, because
|
|
5
|
+
// target repositories never contain the package's core/ or policies/ (issue #27).
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
|
|
10
|
+
// Loop stage references are generated host-neutral copies (issue #35).
|
|
11
|
+
const DOC_ROOTS = Object.freeze(['core', 'policies', 'profiles', 'agents', 'adapters', '.claude/skills/ai-engineering-loop/references']);
|
|
12
|
+
|
|
13
|
+
function packageRoot() {
|
|
14
|
+
return path.join(__dirname, '..');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function walk(dir, out) {
|
|
18
|
+
let entries;
|
|
19
|
+
try {
|
|
20
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
21
|
+
} catch {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
for (const entry of entries) {
|
|
25
|
+
const abs = path.join(dir, entry.name);
|
|
26
|
+
if (entry.isDirectory()) walk(abs, out);
|
|
27
|
+
else if (entry.isFile() && entry.name.endsWith('.md')) out.push(abs);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function listDocs(root = packageRoot()) {
|
|
32
|
+
const files = [];
|
|
33
|
+
for (const dir of DOC_ROOTS) walk(path.join(root, dir), files);
|
|
34
|
+
const counts = new Map();
|
|
35
|
+
const docs = files.map((abs) => {
|
|
36
|
+
const rel = path.relative(root, abs).split(path.sep).join('/');
|
|
37
|
+
const base = path.basename(rel, '.md');
|
|
38
|
+
counts.set(base, (counts.get(base) || 0) + 1);
|
|
39
|
+
return { rel, base, bytes: fs.statSync(abs).size };
|
|
40
|
+
});
|
|
41
|
+
return docs
|
|
42
|
+
.map((doc) => ({ ...doc, name: counts.get(doc.base) === 1 && doc.base !== 'SKILL' ? doc.base : doc.rel.replace(/\.md$/, '') }))
|
|
43
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function resolveDoc(name, root = packageRoot()) {
|
|
47
|
+
const wanted = String(name || '').replace(/\.md$/, '').replace(/\\/g, '/');
|
|
48
|
+
if (!wanted || wanted.includes('..')) return null;
|
|
49
|
+
return listDocs(root).find((doc) => doc.name === wanted || doc.rel.replace(/\.md$/, '') === wanted) || null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function extractSection(markdown, heading) {
|
|
53
|
+
const lines = String(markdown).split('\n');
|
|
54
|
+
const needle = String(heading).trim().toLowerCase();
|
|
55
|
+
const start = lines.findIndex((line) => /^#{1,6}\s/.test(line) && line.replace(/^#{1,6}\s+/, '').trim().toLowerCase().includes(needle));
|
|
56
|
+
if (start === -1) return null;
|
|
57
|
+
const level = lines[start].match(/^#+/)[0].length;
|
|
58
|
+
const out = [lines[start]];
|
|
59
|
+
for (let index = start + 1; index < lines.length; index += 1) {
|
|
60
|
+
const match = lines[index].match(/^(#{1,6})\s/);
|
|
61
|
+
if (match && match[1].length <= level) break;
|
|
62
|
+
out.push(lines[index]);
|
|
63
|
+
}
|
|
64
|
+
return out.join('\n').replace(/\s+$/, '');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function readDoc(name, { section = null, root = packageRoot() } = {}) {
|
|
68
|
+
const doc = resolveDoc(name, root);
|
|
69
|
+
if (!doc) {
|
|
70
|
+
const err = new Error(`Unknown doc "${name}". Run "ai-engineering-loop docs list".`);
|
|
71
|
+
err.code = 'UNKNOWN_DOC';
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
const text = fs.readFileSync(path.join(root, doc.rel), 'utf8').replace(/\r\n?/g, '\n');
|
|
75
|
+
if (!section) return { ...doc, text };
|
|
76
|
+
const part = extractSection(text, section);
|
|
77
|
+
if (part == null) {
|
|
78
|
+
const err = new Error(`Section "${section}" not found in ${doc.rel}`);
|
|
79
|
+
err.code = 'UNKNOWN_SECTION';
|
|
80
|
+
throw err;
|
|
81
|
+
}
|
|
82
|
+
return { ...doc, section, text: part };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = {
|
|
86
|
+
DOC_ROOTS,
|
|
87
|
+
listDocs,
|
|
88
|
+
resolveDoc,
|
|
89
|
+
readDoc,
|
|
90
|
+
extractSection
|
|
91
|
+
};
|
package/lib/doctor.js
CHANGED
|
@@ -8,6 +8,10 @@ const { listRecipes, loadRecipe, validateRecipe } = require('./recipe.js');
|
|
|
8
8
|
|
|
9
9
|
const REQUIRED_HOST_ASSETS = [
|
|
10
10
|
'.claude/skills/ai-engineering-loop/SKILL.md',
|
|
11
|
+
'adapters/project-onboarding/SKILL.md',
|
|
12
|
+
'adapters/quality/skills/ael-code-simplifier/SKILL.md',
|
|
13
|
+
'adapters/quality/skills/ael-receiving-code-review/SKILL.md',
|
|
14
|
+
'.claude/skills/ai-engineering-loop/references/loop-review.md',
|
|
11
15
|
'.claude/skills/backend-development/SKILL.md',
|
|
12
16
|
'.claude/skills/emil-design-eng/SKILL.md',
|
|
13
17
|
'.claude/skills/animate/SKILL.md',
|
|
@@ -15,6 +19,7 @@ const REQUIRED_HOST_ASSETS = [
|
|
|
15
19
|
'.claude/agents/devil-advocate.md',
|
|
16
20
|
'.claude/agents/judge.md',
|
|
17
21
|
'.grok/skills/ai-engineering-loop/SKILL.md',
|
|
22
|
+
'.grok/skills/ai-engineering-loop/references/loop-review.md',
|
|
18
23
|
'.grok/skills/backend-development/SKILL.md',
|
|
19
24
|
'.grok/skills/emil-design-eng/SKILL.md',
|
|
20
25
|
'.grok/skills/animate/SKILL.md',
|
|
@@ -22,6 +27,7 @@ const REQUIRED_HOST_ASSETS = [
|
|
|
22
27
|
'.grok/agents/devil-advocate.md',
|
|
23
28
|
'.grok/agents/judge.md',
|
|
24
29
|
'.gemini/skills/ai-engineering-loop/SKILL.md',
|
|
30
|
+
'.gemini/skills/ai-engineering-loop/references/loop-review.md',
|
|
25
31
|
'.gemini/skills/backend-development/SKILL.md',
|
|
26
32
|
'.gemini/skills/emil-design-eng/SKILL.md',
|
|
27
33
|
'.gemini/skills/animate/SKILL.md',
|
|
@@ -108,7 +114,7 @@ function runDoctor(rootDir = path.join(__dirname, '..')) {
|
|
|
108
114
|
|
|
109
115
|
try {
|
|
110
116
|
const fixtures = loadEvaluationCases(path.join(rootDir, 'evals', 'cases'));
|
|
111
|
-
checks.push(check('evaluation-catalog', fixtures.length ===
|
|
117
|
+
checks.push(check('evaluation-catalog', fixtures.length === 24, `${fixtures.length} valid cases; required 24`));
|
|
112
118
|
} catch (cause) {
|
|
113
119
|
checks.push(check('evaluation-catalog', false, cause.message));
|
|
114
120
|
}
|
package/lib/gates.js
CHANGED
|
@@ -21,8 +21,14 @@ const {
|
|
|
21
21
|
assertModeAllowed
|
|
22
22
|
} = require('./runtime-policy.js');
|
|
23
23
|
const {
|
|
24
|
+
MODEL_TIERS,
|
|
25
|
+
assessRunEscalation,
|
|
24
26
|
enforceVerdictTier
|
|
25
27
|
} = require('./escalation.js');
|
|
28
|
+
const { raiseTierFromDiff } = require('./tier.js');
|
|
29
|
+
const { readyForDa } = require('./claimed-vs-reality.js');
|
|
30
|
+
|
|
31
|
+
const CLAIMED_VS_REALITY_PATH = '.ai-engineering-loop/tasks/claimed-vs-reality.md';
|
|
26
32
|
|
|
27
33
|
const GATE_FILES = Object.freeze({
|
|
28
34
|
goal: 'goal-contract.json',
|
|
@@ -181,6 +187,104 @@ function resolveRun(rootDir, runId) {
|
|
|
181
187
|
return current;
|
|
182
188
|
}
|
|
183
189
|
|
|
190
|
+
function tierOf(run) {
|
|
191
|
+
return run?.tier?.current || null;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function claimedVsRealityReady(rootDir) {
|
|
195
|
+
try {
|
|
196
|
+
return readyForDa(fs.readFileSync(path.join(rootDir, CLAIMED_VS_REALITY_PATH), 'utf8'));
|
|
197
|
+
} catch {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// T0 has no Devil's Advocate. The CLI writes an explicit empty, diff-bound
|
|
203
|
+
// ledger so the review and Judge gates stay in place, but only after
|
|
204
|
+
// claimed-vs-reality has a Reality cell for every Claimed row.
|
|
205
|
+
function writeT0FindingLedger(rootDir, run) {
|
|
206
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
207
|
+
if (fs.existsSync(filePath)) return false;
|
|
208
|
+
if (!claimedVsRealityReady(rootDir)) {
|
|
209
|
+
throw gateError('T0 review needs a completed claimed-vs-reality file', [
|
|
210
|
+
`Fill every Claimed and Reality cell in ${CLAIMED_VS_REALITY_PATH} (scaffold: context claimed-vs-reality)`
|
|
211
|
+
]);
|
|
212
|
+
}
|
|
213
|
+
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
214
|
+
schemaVersion: 1,
|
|
215
|
+
runId: run.runId,
|
|
216
|
+
diffHash: run.artifacts.diff.sha256,
|
|
217
|
+
reviewer: 'none (tier T0: no Devil\'s Advocate)',
|
|
218
|
+
tier: 'T0',
|
|
219
|
+
findings: []
|
|
220
|
+
}, null, 2)}\n`, { mode: 0o600 });
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Whether the Judge subagent can be replaced by the deterministic verdict.
|
|
225
|
+
// Never for T2 or tierless (legacy) runs. T0/T1 qualify only with zero
|
|
226
|
+
// BLOCKER/HIGH findings of any validity, a clean escalation assessment,
|
|
227
|
+
// green diff-bound verification, a ready claimed-vs-reality file, and an
|
|
228
|
+
// iteration under the cap.
|
|
229
|
+
function judgeRequirement(rootDir, run) {
|
|
230
|
+
const tier = tierOf(run);
|
|
231
|
+
const reasons = [];
|
|
232
|
+
if (!tier) return { required: true, tier: null, reasons: ['NO_TIER (legacy run): use the Judge subagent'] };
|
|
233
|
+
if (tier === 'T2') return { required: true, tier, reasons: ['TIER_T2: Judge on STRONG_MODEL is mandatory'] };
|
|
234
|
+
let findings = [];
|
|
235
|
+
try {
|
|
236
|
+
findings = readJsonArtifact(rootDir, run.runId, GATE_FILES.review).value.findings || [];
|
|
237
|
+
} catch {
|
|
238
|
+
reasons.push('FINDINGS_MISSING');
|
|
239
|
+
}
|
|
240
|
+
const serious = findings.filter((f) => f && (f.severity === 'BLOCKER' || f.severity === 'HIGH'));
|
|
241
|
+
if (serious.length) reasons.push(`BLOCKER_OR_HIGH_FINDINGS: ${serious.map((f) => f.id).join(', ')}`);
|
|
242
|
+
if (run.iteration >= 3) reasons.push('ITERATION_CEILING');
|
|
243
|
+
let escalation = null;
|
|
244
|
+
try {
|
|
245
|
+
escalation = assessRunEscalation(rootDir, { runId: run.runId });
|
|
246
|
+
if (escalation.requiredTier !== MODEL_TIERS.CHEAP_MODEL) {
|
|
247
|
+
reasons.push(`ESCALATION_${escalation.requiredTier}: ${escalation.reasons.map((r) => r.code).join(', ')}`);
|
|
248
|
+
}
|
|
249
|
+
} catch (cause) {
|
|
250
|
+
reasons.push(`ESCALATION_UNAVAILABLE: ${cause.message}`);
|
|
251
|
+
}
|
|
252
|
+
if (!claimedVsRealityReady(rootDir)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
|
|
253
|
+
return { required: reasons.length > 0, tier, reasons };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function writeDeterministicVerdictIfEligible(rootDir, run) {
|
|
257
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.judge);
|
|
258
|
+
if (fs.existsSync(filePath)) return false;
|
|
259
|
+
const tier = tierOf(run);
|
|
260
|
+
if (!tier) return false;
|
|
261
|
+
const requirement = judgeRequirement(rootDir, run);
|
|
262
|
+
if (requirement.required) {
|
|
263
|
+
throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons);
|
|
264
|
+
}
|
|
265
|
+
const verification = readJsonArtifact(rootDir, run.runId, GATE_FILES.verification).value;
|
|
266
|
+
const findings = readJsonArtifact(rootDir, run.runId, GATE_FILES.review).value;
|
|
267
|
+
const goalContract = readJsonArtifact(rootDir, run.runId, GATE_FILES.goal).value;
|
|
268
|
+
const expected = computeJudgeVerdict({
|
|
269
|
+
goalContract,
|
|
270
|
+
verificationEvidence: verificationForJudge(verification),
|
|
271
|
+
findingLedger: findings,
|
|
272
|
+
activeIteration: run.iteration,
|
|
273
|
+
maxIterations: 3
|
|
274
|
+
});
|
|
275
|
+
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
276
|
+
schemaVersion: 1,
|
|
277
|
+
runId: run.runId,
|
|
278
|
+
verdict: expected.verdict,
|
|
279
|
+
reason: expected.reason,
|
|
280
|
+
action: expected.action,
|
|
281
|
+
modelTier: MODEL_TIERS.CHEAP_MODEL,
|
|
282
|
+
decidedBy: 'deterministic-gate',
|
|
283
|
+
tier
|
|
284
|
+
}, null, 2)}\n`, { mode: 0o600 });
|
|
285
|
+
return true;
|
|
286
|
+
}
|
|
287
|
+
|
|
184
288
|
function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
185
289
|
if (!Object.hasOwn(GATE_FILES, gate)) {
|
|
186
290
|
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`);
|
|
@@ -254,9 +358,11 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
254
358
|
if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors);
|
|
255
359
|
artifacts.sandboxEvidence = descriptor(rootDir, sandboxFile);
|
|
256
360
|
}
|
|
361
|
+
const raisedTier = raiseTierFromDiff(run.tier, { task: run.task, diff });
|
|
257
362
|
return transitionRun(rootDir, run.runId, RUN_STATES.MAKER_COMPLETE, {
|
|
258
363
|
gate,
|
|
259
|
-
artifacts
|
|
364
|
+
artifacts,
|
|
365
|
+
...(raisedTier ? { statePatch: { tier: raisedTier }, historyMetadata: { tier: raisedTier.current } } : {})
|
|
260
366
|
});
|
|
261
367
|
}
|
|
262
368
|
|
|
@@ -279,6 +385,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
279
385
|
assertArtifactCurrent(rootDir, run, 'goalContract');
|
|
280
386
|
assertArtifactCurrent(rootDir, run, 'diff');
|
|
281
387
|
assertArtifactCurrent(rootDir, run, 'verification');
|
|
388
|
+
if (tierOf(run) === 'T0') writeT0FindingLedger(rootDir, run);
|
|
282
389
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
283
390
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
284
391
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
|
@@ -297,9 +404,14 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
297
404
|
assertArtifactCurrent(rootDir, run, 'diff');
|
|
298
405
|
assertArtifactCurrent(rootDir, run, 'verification');
|
|
299
406
|
assertArtifactCurrent(rootDir, run, 'findings');
|
|
407
|
+
writeDeterministicVerdictIfEligible(rootDir, run);
|
|
300
408
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.judge);
|
|
301
409
|
const verdictCheck = validateVerdictDocument(value, run);
|
|
302
410
|
if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors);
|
|
411
|
+
if (tierOf(run) === 'T2' && value.verdict === 'PASS'
|
|
412
|
+
&& ![MODEL_TIERS.STRONG_MODEL, MODEL_TIERS.HUMAN].includes(value.modelTier)) {
|
|
413
|
+
throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE']);
|
|
414
|
+
}
|
|
303
415
|
try {
|
|
304
416
|
enforceVerdictTier(rootDir, run, value);
|
|
305
417
|
} catch (cause) {
|
|
@@ -366,5 +478,6 @@ module.exports = {
|
|
|
366
478
|
validateGoalContract,
|
|
367
479
|
validateVerificationBundle,
|
|
368
480
|
validateVerdictDocument,
|
|
481
|
+
judgeRequirement,
|
|
369
482
|
applyGate
|
|
370
483
|
};
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Host skill build (issue #35). One template plus one overlay per host produces
|
|
4
|
+
// the four ai-engineering-loop host skills and their references/. The generated
|
|
5
|
+
// files are committed (they ship in the npm package); a drift test fails when a
|
|
6
|
+
// generated file is edited by hand instead of the source in skill-src/.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
|
|
11
|
+
const SOURCE_DIR = 'skill-src/ai-engineering-loop';
|
|
12
|
+
const HOSTS = Object.freeze(['claude', 'grok', 'gemini', 'antigravity']);
|
|
13
|
+
const SKILL_CHAR_LIMIT = 4000;
|
|
14
|
+
const GENERATED_NOTE = '<!-- Generated from skill-src/ by npm run build:skills; do not edit. -->';
|
|
15
|
+
|
|
16
|
+
function readText(file) {
|
|
17
|
+
return fs.readFileSync(file, 'utf8').replace(/\r\n?/g, '\n');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Overlay format: `@@ key: value` (one line) or `@@ key` followed by a block
|
|
21
|
+
// that runs until the next `@@` line.
|
|
22
|
+
function parseOverlay(text) {
|
|
23
|
+
const values = {};
|
|
24
|
+
let current = null;
|
|
25
|
+
for (const line of text.split('\n')) {
|
|
26
|
+
const header = line.match(/^@@ ([a-zA-Z]+)(?::\s?(.*))?$/);
|
|
27
|
+
if (header) {
|
|
28
|
+
current = null;
|
|
29
|
+
if (header[2] !== undefined) values[header[1]] = header[2];
|
|
30
|
+
else {
|
|
31
|
+
current = header[1];
|
|
32
|
+
values[current] = [];
|
|
33
|
+
}
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (current) values[current].push(line);
|
|
37
|
+
}
|
|
38
|
+
for (const [key, value] of Object.entries(values)) {
|
|
39
|
+
if (Array.isArray(value)) values[key] = value.join('\n').replace(/\s+$/, '');
|
|
40
|
+
}
|
|
41
|
+
return values;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function render(template, values, label) {
|
|
45
|
+
return template.replace(/\{\{([a-zA-Z]+)\}\}/g, (_, key) => {
|
|
46
|
+
if (!Object.hasOwn(values, key)) throw new Error(`${label}: missing overlay value "${key}"`);
|
|
47
|
+
return values[key];
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function insertNote(content) {
|
|
52
|
+
const match = content.match(/^---\n[\s\S]*?\n---\n/);
|
|
53
|
+
if (!match) return `${GENERATED_NOTE}\n${content}`;
|
|
54
|
+
return `${match[0]}${GENERATED_NOTE}\n${content.slice(match[0].length)}`;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Returns [{ path, content }] relative to the package root.
|
|
58
|
+
function buildHostSkills(root = path.join(__dirname, '..')) {
|
|
59
|
+
const src = path.join(root, SOURCE_DIR);
|
|
60
|
+
const template = readText(path.join(src, 'SKILL.template.md'));
|
|
61
|
+
const refDir = path.join(src, 'references');
|
|
62
|
+
const references = fs.readdirSync(refDir).filter((f) => f.endsWith('.md')).sort()
|
|
63
|
+
.map((name) => ({ name, content: readText(path.join(refDir, name)) }));
|
|
64
|
+
const outputs = [];
|
|
65
|
+
for (const host of HOSTS) {
|
|
66
|
+
const overlay = parseOverlay(readText(path.join(src, 'hosts', `${host}.md`)));
|
|
67
|
+
if (!overlay.output) throw new Error(`${host}: overlay needs "@@ output: <path>"`);
|
|
68
|
+
const body = render(template, overlay, host).replace(/\n{3,}/g, '\n\n').replace(/\s*$/, '\n');
|
|
69
|
+
outputs.push({ host, path: overlay.output, content: insertNote(body), kind: 'skill' });
|
|
70
|
+
if (overlay.references) {
|
|
71
|
+
for (const ref of references) {
|
|
72
|
+
outputs.push({ host, path: path.posix.join(overlay.references, ref.name), content: ref.content, kind: 'reference' });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return outputs;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function checkHostSkills(root = path.join(__dirname, '..')) {
|
|
80
|
+
const drift = [];
|
|
81
|
+
for (const out of buildHostSkills(root)) {
|
|
82
|
+
let actual = null;
|
|
83
|
+
try {
|
|
84
|
+
actual = readText(path.join(root, out.path));
|
|
85
|
+
} catch {
|
|
86
|
+
// Missing file counts as drift.
|
|
87
|
+
}
|
|
88
|
+
if (actual !== out.content) drift.push(out.path);
|
|
89
|
+
}
|
|
90
|
+
return drift;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function writeHostSkills(root = path.join(__dirname, '..')) {
|
|
94
|
+
const written = [];
|
|
95
|
+
for (const out of buildHostSkills(root)) {
|
|
96
|
+
const abs = path.join(root, out.path);
|
|
97
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
98
|
+
let actual = null;
|
|
99
|
+
try {
|
|
100
|
+
actual = readText(abs);
|
|
101
|
+
} catch {
|
|
102
|
+
// New file.
|
|
103
|
+
}
|
|
104
|
+
if (actual !== out.content) {
|
|
105
|
+
fs.writeFileSync(abs, out.content);
|
|
106
|
+
written.push(out.path);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return written;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
module.exports = {
|
|
113
|
+
SOURCE_DIR,
|
|
114
|
+
HOSTS,
|
|
115
|
+
SKILL_CHAR_LIMIT,
|
|
116
|
+
GENERATED_NOTE,
|
|
117
|
+
parseOverlay,
|
|
118
|
+
buildHostSkills,
|
|
119
|
+
checkHostSkills,
|
|
120
|
+
writeHostSkills
|
|
121
|
+
};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Major-version migrations (decision 6 on #37). A release that crosses a major
|
|
4
|
+
// may ship `migrations/<major>.js`:
|
|
5
|
+
//
|
|
6
|
+
// module.exports = {
|
|
7
|
+
// major: 2,
|
|
8
|
+
// description: 'one line',
|
|
9
|
+
// migrateHome({ home }) {}, // optional, user-level state
|
|
10
|
+
// migrateProject({ projectDir }) {} // optional, once per registered project
|
|
11
|
+
// };
|
|
12
|
+
//
|
|
13
|
+
// `update apply` runs `ai-engineering-loop migrate --from <old>` with the NEW
|
|
14
|
+
// version after a major install. Each step must be idempotent; applied steps
|
|
15
|
+
// are also recorded in ~/.ai-engineering-loop/migrations.json so re-runs skip
|
|
16
|
+
// them (projects registered later are still migrated).
|
|
17
|
+
|
|
18
|
+
const fs = require('fs');
|
|
19
|
+
const path = require('path');
|
|
20
|
+
const { parseVersion } = require('./update-manager.js');
|
|
21
|
+
const { homeDir, userStatePath, readJsonFile, writeJsonFile } = require('./user-state.js');
|
|
22
|
+
|
|
23
|
+
const MIGRATIONS_DIR = path.join(__dirname, '..', 'migrations');
|
|
24
|
+
|
|
25
|
+
function loadMigrations(dir = MIGRATIONS_DIR) {
|
|
26
|
+
let names = [];
|
|
27
|
+
try {
|
|
28
|
+
names = fs.readdirSync(dir).filter((name) => /^\d+\.js$/.test(name));
|
|
29
|
+
} catch {
|
|
30
|
+
return [];
|
|
31
|
+
}
|
|
32
|
+
return names
|
|
33
|
+
.map((name) => {
|
|
34
|
+
const major = Number(name.replace(/\.js$/, ''));
|
|
35
|
+
const migration = require(path.join(dir, name));
|
|
36
|
+
if (!migration || migration.major !== major) {
|
|
37
|
+
throw Object.assign(new Error(`migrations/${name} must export { major: ${major}, ... }`), { code: 'INVALID_MIGRATION' });
|
|
38
|
+
}
|
|
39
|
+
return migration;
|
|
40
|
+
})
|
|
41
|
+
.sort((a, b) => a.major - b.major);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function pendingMigrations({ from, to, dir = MIGRATIONS_DIR }) {
|
|
45
|
+
const fromMajor = parseVersion(from)[0];
|
|
46
|
+
const toMajor = parseVersion(to)[0];
|
|
47
|
+
return loadMigrations(dir).filter((m) => m.major > fromMajor && m.major <= toMajor);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function crossesMajor(from, to) {
|
|
51
|
+
return parseVersion(to)[0] > parseVersion(from)[0];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function runMigrations({ from, to, projects = [], home = homeDir(), dir = MIGRATIONS_DIR, dryRun = false, now = new Date() }) {
|
|
55
|
+
const statePath = userStatePath('migrations.json', home);
|
|
56
|
+
const state = readJsonFile(statePath, { schemaVersion: 1, applied: {} });
|
|
57
|
+
const applied = state.applied || {};
|
|
58
|
+
const report = [];
|
|
59
|
+
for (const migration of pendingMigrations({ from, to, dir })) {
|
|
60
|
+
const key = String(migration.major);
|
|
61
|
+
const record = applied[key] || { home: false, projects: [] };
|
|
62
|
+
const step = { major: migration.major, description: migration.description || '', home: 'skipped', projects: [] };
|
|
63
|
+
if (typeof migration.migrateHome === 'function' && !record.home) {
|
|
64
|
+
if (!dryRun) migration.migrateHome({ home });
|
|
65
|
+
record.home = true;
|
|
66
|
+
step.home = dryRun ? 'would-run' : 'ran';
|
|
67
|
+
}
|
|
68
|
+
if (typeof migration.migrateProject === 'function') {
|
|
69
|
+
for (const project of projects) {
|
|
70
|
+
const projectDir = project.path || project;
|
|
71
|
+
if (record.projects.includes(projectDir)) continue;
|
|
72
|
+
if (!dryRun) migration.migrateProject({ projectDir, home });
|
|
73
|
+
record.projects.push(projectDir);
|
|
74
|
+
step.projects.push(projectDir);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
if (!dryRun) applied[key] = { ...record, at: record.at || now.toISOString() };
|
|
78
|
+
report.push(step);
|
|
79
|
+
}
|
|
80
|
+
if (!dryRun && report.length) writeJsonFile(statePath, { schemaVersion: 1, applied });
|
|
81
|
+
return { from, to, migrations: report };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
module.exports = { MIGRATIONS_DIR, loadMigrations, pendingMigrations, crossesMajor, runMigrations };
|