ai-engineering-loop 1.16.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.agents/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.agents/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.agents/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.claude/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.claude/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.claude/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/CHANGELOG.md +69 -0
- package/README.md +24 -22
- package/adapters/dot/gitlab.md +6 -0
- package/adapters/github/pull-request.md +5 -0
- package/adapters/gitlab/merge-request.md +5 -0
- package/adapters/quality/skills/task-impact-inquiry/SKILL.md +2 -0
- package/bin/ai-engineering-loop.js +514 -19
- package/core/bench.md +77 -0
- package/core/blast-radius-map.md +37 -0
- package/core/configuration-precedence.md +11 -1
- package/core/learnings.md +24 -0
- package/core/local-workflow-studio.md +26 -68
- package/core/personal-workflow.md +78 -0
- package/core/pr-receipt.md +35 -0
- package/core/runtime-safety.md +24 -0
- package/core/workflow-recipes.md +2 -0
- package/evals/README.md +2 -0
- package/evals/cases/25-no-task-onboarding.json +22 -0
- package/evals/scenarios/t0-docs-typo.json +31 -0
- package/evals/scenarios/t1-empty-state.json +31 -0
- package/evals/scenarios/t1-list-pagination.json +33 -0
- package/evals/scenarios/t2-auth-session.json +31 -0
- package/evals/scenarios/t2-payment-idempotency.json +31 -0
- package/evals/scenarios/trap-destructive-reset.json +33 -0
- package/evals/scenarios/trap-out-of-scope.json +34 -0
- package/evals/scenarios/trap-prompt-injection.json +39 -0
- package/lib/bench-targets.js +194 -0
- package/lib/bench.js +1236 -0
- package/lib/budget.js +32 -0
- package/lib/cli-args.js +17 -2
- package/lib/doctor.js +20 -2
- package/lib/gates.js +39 -0
- package/lib/guard-hooks.js +115 -0
- package/lib/guard.js +336 -0
- package/lib/hosts/commands.js +4 -3
- package/lib/hosts/doctor.js +3 -3
- package/lib/hosts/registry.js +39 -2
- package/lib/hosts/table.js +6 -4
- package/lib/impact-map.js +501 -0
- package/lib/learn.js +303 -0
- package/lib/personal-workflow.js +925 -0
- package/lib/receipt.js +362 -0
- package/lib/recipe.js +90 -13
- package/lib/run-state.js +10 -1
- package/lib/run-watcher.js +511 -0
- package/lib/runtime-files.js +2 -0
- package/lib/runtime-policy.js +42 -1
- package/lib/safe-context.js +14 -0
- package/lib/smart-context.js +21 -0
- package/lib/studio-runtime.js +8 -3
- package/lib/studio-server.js +336 -98
- package/lib/user-state.js +34 -1
- package/package.json +1 -1
- package/schemas/bench-result.schema.json +288 -0
- package/schemas/bench-scenario.schema.json +95 -0
- package/schemas/delivery.schema.json +11 -1
- package/schemas/finding-ledger.schema.json +2 -1
- package/schemas/goal-contract.schema.json +9 -0
- package/schemas/impact-map.schema.json +78 -0
- package/schemas/learnings.schema.json +47 -0
- package/schemas/receipt.schema.json +102 -0
- package/studio/app.js +599 -1621
- package/studio/index.html +43 -178
- package/studio/styles.css +17 -2
package/lib/receipt.js
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Proof-carrying PR receipt (#70). `deliver` writes RUN_DIR/receipt.json:
|
|
4
|
+
// the frozen Goal and its hash, per-AC evidence, the DA/Judge result, base and
|
|
5
|
+
// head SHA, the sha256 of every referenced artifact, and an overall digest.
|
|
6
|
+
// `verify-receipt` re-checks all of it, so a reviewer (or CI) can trust the
|
|
7
|
+
// PR body without re-reading the Run.
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { execFileSync } = require('child_process');
|
|
13
|
+
const { getCurrentRun, loadRun, runsDir } = require('./run-state.js');
|
|
14
|
+
const { canonicalJson, sha256 } = require('./recipe.js');
|
|
15
|
+
const { parseClaimedVsReality } = require('./claimed-vs-reality.js');
|
|
16
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
17
|
+
const { isBlockingFinding } = require('./orchestration.js');
|
|
18
|
+
|
|
19
|
+
const RECEIPT_FILE = 'receipt.json';
|
|
20
|
+
const RECEIPT_MARKDOWN_FILE = 'receipt.md';
|
|
21
|
+
const CLAIMED_VS_REALITY = '.ai-engineering-loop/tasks/claimed-vs-reality.md';
|
|
22
|
+
const RUN_ARTIFACTS = Object.freeze([
|
|
23
|
+
['goalContract', 'goal-contract.json', true],
|
|
24
|
+
['diff', 'diff.patch', true],
|
|
25
|
+
['verification', 'verification.json', true],
|
|
26
|
+
['findings', 'findings.json', true],
|
|
27
|
+
['verdict', 'verdict.json', true],
|
|
28
|
+
['impactMap', 'impact-map.json', false],
|
|
29
|
+
['impactDrift', 'impact-drift.json', false],
|
|
30
|
+
['guardEvents', 'guard-events.json', false]
|
|
31
|
+
]);
|
|
32
|
+
const MAX_TEXT = 500;
|
|
33
|
+
|
|
34
|
+
function receiptError(message, code = 'RECEIPT_FAILED', details = []) {
|
|
35
|
+
const error = new Error(message);
|
|
36
|
+
error.code = code;
|
|
37
|
+
error.details = details;
|
|
38
|
+
return error;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function resolveRun(rootDir, runId) {
|
|
42
|
+
if (runId) return loadRun(rootDir, runId);
|
|
43
|
+
const current = getCurrentRun(rootDir);
|
|
44
|
+
if (!current) throw receiptError('No current Run. Pass --run <id>.', 'NO_CURRENT_RUN');
|
|
45
|
+
return current;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function hashBytes(buffer) {
|
|
49
|
+
return crypto.createHash('sha256').update(buffer).digest('hex');
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function repoRelative(rootDir, filePath) {
|
|
53
|
+
return path.relative(rootDir, filePath).split(path.sep).join('/');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Receipts leave the machine: no secrets and no local absolute paths.
|
|
57
|
+
function safeText(value) {
|
|
58
|
+
return redactHomePaths(redactSecrets(String(value ?? '')).text)
|
|
59
|
+
.replace(/(?:^|\s)\/(?:tmp|var|private|Users|home)\/\S*/g, ' [REDACTED PATH]')
|
|
60
|
+
.trim()
|
|
61
|
+
.slice(0, MAX_TEXT);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function readJson(filePath, label) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
67
|
+
} catch (cause) {
|
|
68
|
+
throw receiptError(`${label} is missing or invalid: ${cause.message}`, 'RECEIPT_INPUT_MISSING');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function gitHead(rootDir) {
|
|
73
|
+
try {
|
|
74
|
+
return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
75
|
+
} catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function validateLink(url) {
|
|
81
|
+
let parsed;
|
|
82
|
+
try {
|
|
83
|
+
parsed = new URL(String(url));
|
|
84
|
+
} catch {
|
|
85
|
+
parsed = null;
|
|
86
|
+
}
|
|
87
|
+
if (!parsed || !['https:', 'http:'].includes(parsed.protocol) || parsed.username || parsed.password) {
|
|
88
|
+
throw receiptError(`Evidence links must be hosted http(s) URLs without credentials: ${safeText(url)}`, 'INVALID_EVIDENCE_LINK');
|
|
89
|
+
}
|
|
90
|
+
return parsed.toString();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// `--evidence AC-1=https://ci.example/run/1` (repeatable).
|
|
94
|
+
function parseEvidenceLinks(values = []) {
|
|
95
|
+
const links = {};
|
|
96
|
+
for (const value of values) {
|
|
97
|
+
const match = String(value).match(/^(AC-\d+)=(.+)$/);
|
|
98
|
+
if (!match) throw receiptError(`--evidence must be AC-N=<https-url>, received ${safeText(value)}`, 'INVALID_EVIDENCE_LINK');
|
|
99
|
+
(links[match[1]] ||= []).push(validateLink(match[2]));
|
|
100
|
+
}
|
|
101
|
+
return links;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function artifactEntries(rootDir, run) {
|
|
105
|
+
const directory = path.join(runsDir(rootDir), run.runId);
|
|
106
|
+
const entries = [];
|
|
107
|
+
for (const [name, file, required] of RUN_ARTIFACTS) {
|
|
108
|
+
const filePath = path.join(directory, file);
|
|
109
|
+
if (!fs.existsSync(filePath)) {
|
|
110
|
+
if (required) throw receiptError(`Receipt needs ${file}; finish the gates first`, 'RECEIPT_INPUT_MISSING');
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
entries.push({ name, path: repoRelative(rootDir, filePath), sha256: hashBytes(fs.readFileSync(filePath)) });
|
|
114
|
+
}
|
|
115
|
+
const claimed = path.join(rootDir, CLAIMED_VS_REALITY);
|
|
116
|
+
if (fs.existsSync(claimed)) {
|
|
117
|
+
entries.push({ name: 'claimedVsReality', path: CLAIMED_VS_REALITY, sha256: hashBytes(fs.readFileSync(claimed)) });
|
|
118
|
+
}
|
|
119
|
+
return entries;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function commandEvidence(command) {
|
|
123
|
+
return {
|
|
124
|
+
type: 'command',
|
|
125
|
+
command: safeText(command.command),
|
|
126
|
+
exitCode: command.exitCode,
|
|
127
|
+
timeoutStatus: command.timeoutStatus,
|
|
128
|
+
outputSha256: sha256(`${command.stdout || ''}\n${command.stderr || ''}`),
|
|
129
|
+
passed: command.exitCode === 0 && command.timeoutStatus === 'COMPLETED'
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Optional coverage numbers from verification.json (e.g. { lines: 91.2,
|
|
134
|
+
// linesDelta: 0.4 }); only finite numbers are kept, never paths or text.
|
|
135
|
+
function coverageSummary(coverage) {
|
|
136
|
+
if (!coverage || typeof coverage !== 'object' || Array.isArray(coverage)) return null;
|
|
137
|
+
const numbers = Object.entries(coverage).filter(([key, value]) => /^[A-Za-z0-9_-]{1,40}$/.test(key) && Number.isFinite(value));
|
|
138
|
+
return numbers.length ? Object.fromEntries(numbers.sort(([a], [b]) => a.localeCompare(b))) : null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function criterionEntry(criterion, { commands, claims, links }) {
|
|
142
|
+
const claim = claims.find((row) => row.ac === criterion.id);
|
|
143
|
+
const evidence = [];
|
|
144
|
+
const missing = [];
|
|
145
|
+
if (claim && String(claim.reality || '').trim()) {
|
|
146
|
+
evidence.push({ type: 'claim', claimed: safeText(claim.claimed), reality: safeText(claim.reality) });
|
|
147
|
+
} else {
|
|
148
|
+
missing.push('no claimed-vs-reality row with a Reality cell');
|
|
149
|
+
}
|
|
150
|
+
if (!commands.length) missing.push('no recorded verification command');
|
|
151
|
+
else if (commands.some((item) => !item.passed)) missing.push('a verification command did not pass');
|
|
152
|
+
evidence.push(...commands);
|
|
153
|
+
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', url });
|
|
154
|
+
return {
|
|
155
|
+
id: criterion.id,
|
|
156
|
+
statement: safeText(criterion.statement),
|
|
157
|
+
evidenceRequired: safeText(criterion.evidenceRequired),
|
|
158
|
+
status: missing.length ? 'MISSING' : 'PASS',
|
|
159
|
+
...(missing.length ? { missing } : {}),
|
|
160
|
+
evidence
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function receiptDigest(receipt) {
|
|
165
|
+
const { digest: _digest, ...body } = receipt;
|
|
166
|
+
return sha256(canonicalJson(body));
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function buildReceipt(rootDir, { runId, host, model, head, evidenceLinks = [], aelVersion, now = new Date() } = {}) {
|
|
170
|
+
const run = resolveRun(rootDir, runId);
|
|
171
|
+
const directory = path.join(runsDir(rootDir), run.runId);
|
|
172
|
+
const artifacts = artifactEntries(rootDir, run);
|
|
173
|
+
const goalFile = path.join(directory, 'goal-contract.json');
|
|
174
|
+
const goal = readJson(goalFile, 'goal-contract.json');
|
|
175
|
+
const verification = readJson(path.join(directory, 'verification.json'), 'verification.json');
|
|
176
|
+
const findings = readJson(path.join(directory, 'findings.json'), 'findings.json');
|
|
177
|
+
const verdict = readJson(path.join(directory, 'verdict.json'), 'verdict.json');
|
|
178
|
+
const claimedFile = path.join(rootDir, CLAIMED_VS_REALITY);
|
|
179
|
+
const claims = fs.existsSync(claimedFile) ? parseClaimedVsReality(fs.readFileSync(claimedFile, 'utf8')).rows : [];
|
|
180
|
+
const commands = (verification.commands || []).map(commandEvidence);
|
|
181
|
+
const links = parseEvidenceLinks(evidenceLinks);
|
|
182
|
+
const unknownLinks = Object.keys(links).filter((id) => !(goal.acceptanceCriteria || []).some((item) => item.id === id));
|
|
183
|
+
if (unknownLinks.length) throw receiptError(`--evidence names unknown AC: ${unknownLinks.join(', ')}`, 'INVALID_EVIDENCE_LINK');
|
|
184
|
+
const ledger = Array.isArray(findings.findings) ? findings.findings : [];
|
|
185
|
+
const drift = artifacts.some((item) => item.name === 'impactDrift')
|
|
186
|
+
? readJson(path.join(directory, 'impact-drift.json'), 'impact-drift.json')
|
|
187
|
+
: null;
|
|
188
|
+
const guardEvents = artifacts.some((item) => item.name === 'guardEvents')
|
|
189
|
+
? (readJson(path.join(directory, 'guard-events.json'), 'guard-events.json').events || [])
|
|
190
|
+
: [];
|
|
191
|
+
const bySeverity = {};
|
|
192
|
+
for (const finding of ledger) bySeverity[finding.severity] = (bySeverity[finding.severity] || 0) + 1;
|
|
193
|
+
const receipt = {
|
|
194
|
+
schemaVersion: 1,
|
|
195
|
+
runId: run.runId,
|
|
196
|
+
generatedAt: now.toISOString(),
|
|
197
|
+
aelVersion: aelVersion || null,
|
|
198
|
+
host: { id: safeText(host || 'unspecified'), model: model ? safeText(model) : (verdict.model ? safeText(verdict.model) : null) },
|
|
199
|
+
base: { sha: run.startedRevision || null },
|
|
200
|
+
head: { sha: head || gitHead(rootDir) },
|
|
201
|
+
goal: {
|
|
202
|
+
path: repoRelative(rootDir, goalFile),
|
|
203
|
+
sha256: hashBytes(fs.readFileSync(goalFile)),
|
|
204
|
+
contentHash: sha256(goal),
|
|
205
|
+
version: run.goal?.version || 1,
|
|
206
|
+
objective: safeText(goal.objective)
|
|
207
|
+
},
|
|
208
|
+
acceptanceCriteria: (goal.acceptanceCriteria || []).map((criterion) => criterionEntry(criterion, { commands, claims, links })),
|
|
209
|
+
verification: {
|
|
210
|
+
diffHash: verification.diffHash,
|
|
211
|
+
gitRevision: verification.gitRevision,
|
|
212
|
+
commands,
|
|
213
|
+
coverage: coverageSummary(verification.coverage)
|
|
214
|
+
},
|
|
215
|
+
review: {
|
|
216
|
+
findings: {
|
|
217
|
+
total: ledger.length,
|
|
218
|
+
valid: ledger.filter((item) => item.validity === 'VALID').length,
|
|
219
|
+
blocking: ledger.filter(isBlockingFinding).length,
|
|
220
|
+
bySeverity
|
|
221
|
+
},
|
|
222
|
+
verdict: {
|
|
223
|
+
verdict: verdict.verdict,
|
|
224
|
+
modelTier: verdict.modelTier || null,
|
|
225
|
+
decidedBy: safeText(verdict.decidedBy || 'judge'),
|
|
226
|
+
reason: safeText(verdict.reason)
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
impact: drift ? { mapStatus: drift.mapStatus, outOfMap: (drift.outOfMap || []).map(safeText) } : null,
|
|
230
|
+
guard: {
|
|
231
|
+
warn: guardEvents.filter((event) => event.decision === 'warn').length,
|
|
232
|
+
block: guardEvents.filter((event) => event.decision === 'block').length
|
|
233
|
+
},
|
|
234
|
+
artifacts
|
|
235
|
+
};
|
|
236
|
+
receipt.digest = receiptDigest(receipt);
|
|
237
|
+
return receipt;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function markdownCell(value) {
|
|
241
|
+
return String(value ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|');
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function short(sha, length = 12) {
|
|
245
|
+
return sha ? String(sha).slice(0, length) : 'unknown';
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function renderReceiptMarkdown(receipt) {
|
|
249
|
+
const lines = [
|
|
250
|
+
'## Receipt',
|
|
251
|
+
'',
|
|
252
|
+
`Run \`${receipt.runId}\` · Goal v${receipt.goal.version} \`${short(receipt.goal.contentHash)}\` · base \`${short(receipt.base.sha, 7)}\` → head \`${short(receipt.head.sha, 7)}\` · digest \`${short(receipt.digest)}\``,
|
|
253
|
+
''
|
|
254
|
+
];
|
|
255
|
+
for (const criterion of receipt.acceptanceCriteria) {
|
|
256
|
+
const parts = [];
|
|
257
|
+
for (const item of criterion.evidence) {
|
|
258
|
+
if (item.type === 'command') parts.push(`\`${markdownCell(item.command)}\` exit ${item.exitCode}`);
|
|
259
|
+
if (item.type === 'claim') parts.push(`reality: ${markdownCell(item.reality)}`);
|
|
260
|
+
if (item.type === 'link') parts.push(`[evidence](${item.url})`);
|
|
261
|
+
}
|
|
262
|
+
const missing = criterion.missing ? ` — **missing:** ${criterion.missing.join('; ')}` : '';
|
|
263
|
+
lines.push(`- ${criterion.status === 'PASS' ? '✅' : '❌'} **${criterion.id}** ${markdownCell(criterion.statement)}${parts.length ? ` — ${parts.join(', ')}` : ''}${missing}`);
|
|
264
|
+
}
|
|
265
|
+
const { findings, verdict } = receipt.review;
|
|
266
|
+
lines.push('');
|
|
267
|
+
lines.push(`Judge: **${verdict.verdict}**${verdict.modelTier ? ` (${verdict.modelTier})` : ''} · Findings: ${findings.total} (${findings.valid} valid, ${findings.blocking} blocking)` +
|
|
268
|
+
`${receipt.impact ? ` · Impact map: ${receipt.impact.mapStatus}, ${receipt.impact.outOfMap.length} file(s) outside` : ''}` +
|
|
269
|
+
` · Guard: ${receipt.guard.block} blocked, ${receipt.guard.warn} warned`);
|
|
270
|
+
lines.push('');
|
|
271
|
+
lines.push(`Check: \`npx ai-engineering-loop verify-receipt --run ${receipt.runId} --head ${short(receipt.head.sha, 40)}\``);
|
|
272
|
+
return `${lines.join('\n')}\n`;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function writeReceipt(rootDir, options = {}) {
|
|
276
|
+
const receipt = buildReceipt(rootDir, options);
|
|
277
|
+
const directory = path.join(runsDir(rootDir), receipt.runId);
|
|
278
|
+
const jsonPath = path.join(directory, RECEIPT_FILE);
|
|
279
|
+
const markdownPath = path.join(directory, RECEIPT_MARKDOWN_FILE);
|
|
280
|
+
fs.writeFileSync(jsonPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
|
|
281
|
+
fs.writeFileSync(markdownPath, renderReceiptMarkdown(receipt), { mode: 0o600 });
|
|
282
|
+
return {
|
|
283
|
+
receipt,
|
|
284
|
+
path: repoRelative(rootDir, jsonPath),
|
|
285
|
+
markdownPath: repoRelative(rootDir, markdownPath),
|
|
286
|
+
missing: receipt.acceptanceCriteria.filter((item) => item.status !== 'PASS').map((item) => item.id)
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function insideRepo(rootDir, relativePath) {
|
|
291
|
+
const absolute = path.resolve(rootDir, relativePath);
|
|
292
|
+
const relative = path.relative(rootDir, absolute);
|
|
293
|
+
if (!relativePath || path.isAbsolute(relativePath) || relative === '..' || relative.startsWith(`..${path.sep}`)) return null;
|
|
294
|
+
return absolute;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function verifyReceipt(rootDir, { runId, receiptPath, head } = {}) {
|
|
298
|
+
let file;
|
|
299
|
+
if (receiptPath) {
|
|
300
|
+
file = insideRepo(rootDir, receiptPath);
|
|
301
|
+
if (!file) throw receiptError('Receipt path must be repository-relative', 'UNSAFE_RECEIPT_PATH');
|
|
302
|
+
} else {
|
|
303
|
+
file = path.join(runsDir(rootDir), resolveRun(rootDir, runId).runId, RECEIPT_FILE);
|
|
304
|
+
}
|
|
305
|
+
const receipt = readJson(file, 'receipt.json');
|
|
306
|
+
const problems = [];
|
|
307
|
+
const fail = (code, detail) => problems.push({ code, detail });
|
|
308
|
+
if (receipt.schemaVersion !== 1 || !receipt.runId) fail('RECEIPT_INVALID', 'schemaVersion 1 and runId are required');
|
|
309
|
+
if (receiptDigest(receipt) !== receipt.digest) fail('RECEIPT_DIGEST_MISMATCH', 'receipt content does not match its digest');
|
|
310
|
+
|
|
311
|
+
for (const artifact of receipt.artifacts || []) {
|
|
312
|
+
const absolute = insideRepo(rootDir, artifact.path);
|
|
313
|
+
if (!absolute || !fs.existsSync(absolute)) {
|
|
314
|
+
fail('ARTIFACT_MISSING', `${artifact.path} is missing`);
|
|
315
|
+
} else if (hashBytes(fs.readFileSync(absolute)) !== artifact.sha256) {
|
|
316
|
+
fail('ARTIFACT_TAMPERED', `${artifact.path} changed after the receipt`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const goalFile = receipt.goal?.path && insideRepo(rootDir, receipt.goal.path);
|
|
321
|
+
let goal = null;
|
|
322
|
+
if (!goalFile || !fs.existsSync(goalFile)) {
|
|
323
|
+
fail('GOAL_CHANGED', 'the frozen Goal is missing');
|
|
324
|
+
} else {
|
|
325
|
+
goal = JSON.parse(fs.readFileSync(goalFile, 'utf8'));
|
|
326
|
+
if (hashBytes(fs.readFileSync(goalFile)) !== receipt.goal.sha256 || sha256(goal) !== receipt.goal.contentHash) {
|
|
327
|
+
fail('GOAL_CHANGED', 'the frozen Goal differs from the one in the receipt');
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const listed = new Map((receipt.acceptanceCriteria || []).map((item) => [item.id, item]));
|
|
331
|
+
for (const criterion of goal?.acceptanceCriteria || []) {
|
|
332
|
+
const entry = listed.get(criterion.id);
|
|
333
|
+
if (!entry) fail('AC_NOT_IN_RECEIPT', `${criterion.id} is not in the receipt`);
|
|
334
|
+
else if (entry.status !== 'PASS' || !(entry.evidence || []).length) {
|
|
335
|
+
fail('AC_EVIDENCE_MISSING', `${criterion.id}: ${(entry.missing || ['no evidence']).join('; ')}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
if (receipt.review?.verdict?.verdict !== 'PASS') fail('VERDICT_NOT_PASS', `Judge verdict is ${receipt.review?.verdict?.verdict || 'missing'}`);
|
|
339
|
+
if (receipt.review?.findings?.blocking) fail('BLOCKING_FINDINGS', `${receipt.review.findings.blocking} blocking finding(s)`);
|
|
340
|
+
const expectedHead = head || gitHead(rootDir);
|
|
341
|
+
if (!receipt.head?.sha || !expectedHead || receipt.head.sha !== expectedHead) {
|
|
342
|
+
fail('HEAD_MISMATCH', `receipt head ${short(receipt.head?.sha, 7)} vs ${head ? '--head' : 'HEAD'} ${short(expectedHead, 7)}`);
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
ok: problems.length === 0,
|
|
346
|
+
runId: receipt.runId,
|
|
347
|
+
digest: receipt.digest,
|
|
348
|
+
path: repoRelative(rootDir, file),
|
|
349
|
+
problems
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
module.exports = {
|
|
354
|
+
RECEIPT_FILE,
|
|
355
|
+
RECEIPT_MARKDOWN_FILE,
|
|
356
|
+
buildReceipt,
|
|
357
|
+
writeReceipt,
|
|
358
|
+
verifyReceipt,
|
|
359
|
+
renderReceiptMarkdown,
|
|
360
|
+
receiptDigest,
|
|
361
|
+
parseEvidenceLinks
|
|
362
|
+
};
|
package/lib/recipe.js
CHANGED
|
@@ -23,6 +23,40 @@ const RECIPE_ID = /^[a-z][a-z0-9-]{0,63}$/;
|
|
|
23
23
|
const NODE_ID = /^[a-z][a-z0-9-]{0,63}$/;
|
|
24
24
|
const MAX_NODES = 64;
|
|
25
25
|
const BUILTIN_RECIPE_DIR = path.join(__dirname, '..', 'recipes');
|
|
26
|
+
const MODEL_TIER_RANK = Object.freeze({
|
|
27
|
+
CHEAP_MODEL: 0,
|
|
28
|
+
STRONG_MODEL: 1,
|
|
29
|
+
HUMAN: 2
|
|
30
|
+
});
|
|
31
|
+
const BACKBONE_CODES = Object.freeze({
|
|
32
|
+
GOAL_CONTRACT: 'BACKBONE_GOAL_CONTRACT_REQUIRED',
|
|
33
|
+
MAKER: 'BACKBONE_MAKER_REQUIRED',
|
|
34
|
+
VERIFICATION: 'BACKBONE_VERIFICATION_REQUIRED',
|
|
35
|
+
DEVIL_ADVOCATE: 'BACKBONE_DEVIL_ADVOCATE_REQUIRED',
|
|
36
|
+
JUDGE: 'BACKBONE_JUDGE_REQUIRED',
|
|
37
|
+
DELIVERY: 'BACKBONE_DELIVERY_REQUIRED',
|
|
38
|
+
REPORT: 'BACKBONE_REPORT_REQUIRED',
|
|
39
|
+
CONDITIONAL_SKIP: 'BACKBONE_CONDITIONAL_SKIP',
|
|
40
|
+
ASSISTED_APPROVAL: 'BACKBONE_ASSISTED_APPROVAL_REQUIRED',
|
|
41
|
+
T2_JUDGE_MODEL: 'BACKBONE_T2_JUDGE_MODEL'
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
function backboneCodeForType(type) {
|
|
45
|
+
return {
|
|
46
|
+
'goal-contract': BACKBONE_CODES.GOAL_CONTRACT,
|
|
47
|
+
maker: BACKBONE_CODES.MAKER,
|
|
48
|
+
verification: BACKBONE_CODES.VERIFICATION,
|
|
49
|
+
'devil-advocate': BACKBONE_CODES.DEVIL_ADVOCATE,
|
|
50
|
+
judge: BACKBONE_CODES.JUDGE,
|
|
51
|
+
delivery: BACKBONE_CODES.DELIVERY,
|
|
52
|
+
report: BACKBONE_CODES.REPORT
|
|
53
|
+
}[type] || null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function pushCoded(errors, codes, message, code) {
|
|
57
|
+
errors.push(message);
|
|
58
|
+
if (code) codes.push(code);
|
|
59
|
+
}
|
|
26
60
|
|
|
27
61
|
function recipeError(message, details = [], code = 'INVALID_RECIPE') {
|
|
28
62
|
const err = new Error(message);
|
|
@@ -100,14 +134,38 @@ function forbiddenAuthorityPaths(value, location, result = []) {
|
|
|
100
134
|
return result;
|
|
101
135
|
}
|
|
102
136
|
|
|
103
|
-
function
|
|
137
|
+
function validateJudgeModelTier(node, location, errors, codes) {
|
|
138
|
+
if (node.type !== 'judge') return;
|
|
139
|
+
const tier = node.modelTier;
|
|
140
|
+
if (tier == null || tier === '') return;
|
|
141
|
+
if (!Object.hasOwn(MODEL_TIER_RANK, tier)) {
|
|
142
|
+
errors.push(`${location}.modelTier must be CHEAP_MODEL, STRONG_MODEL, or HUMAN`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (MODEL_TIER_RANK[tier] < MODEL_TIER_RANK.STRONG_MODEL) {
|
|
146
|
+
pushCoded(
|
|
147
|
+
errors,
|
|
148
|
+
codes,
|
|
149
|
+
'T2 judge cannot use a model below STRONG_MODEL',
|
|
150
|
+
BACKBONE_CODES.T2_JUDGE_MODEL
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function validateNodeConfiguration(node, index, errors, warnings, codes = []) {
|
|
104
156
|
const location = `nodes[${index}]`;
|
|
105
157
|
const definition = getNodeType(node.type);
|
|
106
158
|
if (!definition) return;
|
|
107
159
|
if (node.when != null) validateCondition(node.when, `${location}.when`, errors);
|
|
108
160
|
if (definition.builtIn && node.when != null) {
|
|
109
|
-
|
|
161
|
+
pushCoded(
|
|
162
|
+
errors,
|
|
163
|
+
codes,
|
|
164
|
+
`${location}.when cannot conditionally skip a required safety node`,
|
|
165
|
+
BACKBONE_CODES.CONDITIONAL_SKIP
|
|
166
|
+
);
|
|
110
167
|
}
|
|
168
|
+
validateJudgeModelTier(node, location, errors, codes);
|
|
111
169
|
|
|
112
170
|
if (node.type === 'condition') {
|
|
113
171
|
validateCondition(node.expression, `${location}.expression`, errors);
|
|
@@ -270,7 +328,7 @@ function firstNodeOfType(nodes, type) {
|
|
|
270
328
|
return nodes.find((node) => node.type === type);
|
|
271
329
|
}
|
|
272
330
|
|
|
273
|
-
function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
331
|
+
function validateSafetyBackbone(recipe, mode, byId, errors, codes = []) {
|
|
274
332
|
const nodes = recipe.nodes || [];
|
|
275
333
|
for (const type of ['goal-contract', 'maker', 'verification', 'devil-advocate', 'judge', 'report', 'delivery']) {
|
|
276
334
|
if (nodes.filter((node) => node.type === type).length > 1) {
|
|
@@ -278,11 +336,15 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
278
336
|
}
|
|
279
337
|
}
|
|
280
338
|
const goal = firstNodeOfType(nodes, 'goal-contract');
|
|
281
|
-
if (!goal)
|
|
339
|
+
if (!goal) {
|
|
340
|
+
pushCoded(errors, codes, 'a goal-contract node is required', BACKBONE_CODES.GOAL_CONTRACT);
|
|
341
|
+
}
|
|
282
342
|
|
|
283
343
|
if (mode === RUN_MODES.REPORT_ONLY) {
|
|
284
344
|
const report = firstNodeOfType(nodes, 'report');
|
|
285
|
-
if (!report)
|
|
345
|
+
if (!report) {
|
|
346
|
+
pushCoded(errors, codes, 'REPORT_ONLY requires a report node', BACKBONE_CODES.REPORT);
|
|
347
|
+
}
|
|
286
348
|
if (nodes.some((node) => getNodeType(node.type)?.mutation)) {
|
|
287
349
|
errors.push('REPORT_ONLY cannot contain mutation nodes');
|
|
288
350
|
}
|
|
@@ -301,7 +363,10 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
301
363
|
const chainTypes = ['goal-contract', 'maker', 'verification', 'devil-advocate', 'judge', 'delivery'];
|
|
302
364
|
const chain = chainTypes.map((type) => firstNodeOfType(nodes, type));
|
|
303
365
|
chain.forEach((node, index) => {
|
|
304
|
-
if (!node)
|
|
366
|
+
if (!node) {
|
|
367
|
+
const type = chainTypes[index];
|
|
368
|
+
pushCoded(errors, codes, `${type} node is required for ${mode}`, backboneCodeForType(type));
|
|
369
|
+
}
|
|
305
370
|
});
|
|
306
371
|
for (let index = 1; index < chain.length; index++) {
|
|
307
372
|
if (chain[index] && chain[index - 1] && !dependsTransitively(byId, chain[index].id, chain[index - 1].id)) {
|
|
@@ -318,9 +383,11 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
318
383
|
)) && judge && dependsTransitively(byId, node.id, judge.id)
|
|
319
384
|
));
|
|
320
385
|
const delivery = firstNodeOfType(nodes, 'delivery');
|
|
321
|
-
if (!approval)
|
|
386
|
+
if (!approval) {
|
|
387
|
+
pushCoded(errors, codes, 'ASSISTED requires a human approval node', BACKBONE_CODES.ASSISTED_APPROVAL);
|
|
388
|
+
}
|
|
322
389
|
if (delivery && approval && !dependsTransitively(byId, delivery.id, approval.id)) {
|
|
323
|
-
errors
|
|
390
|
+
pushCoded(errors, codes, 'delivery must depend on ASSISTED approval', BACKBONE_CODES.ASSISTED_APPROVAL);
|
|
324
391
|
}
|
|
325
392
|
}
|
|
326
393
|
}
|
|
@@ -328,8 +395,9 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
328
395
|
function validateRecipe(recipe, { mode } = {}) {
|
|
329
396
|
const errors = [];
|
|
330
397
|
const warnings = [];
|
|
398
|
+
const codes = [];
|
|
331
399
|
if (!recipe || typeof recipe !== 'object' || Array.isArray(recipe)) {
|
|
332
|
-
return { valid: false, errors: ['recipe must be a JSON object'], warnings, order: [] };
|
|
400
|
+
return { valid: false, errors: ['recipe must be a JSON object'], warnings, order: [], codes };
|
|
333
401
|
}
|
|
334
402
|
recipe = migrateLegacyRecipe(recipe);
|
|
335
403
|
if (recipe.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
@@ -371,7 +439,7 @@ function validateRecipe(recipe, { mode } = {}) {
|
|
|
371
439
|
)) {
|
|
372
440
|
errors.push(`${location}.dependsOn must be an array of node ids`);
|
|
373
441
|
}
|
|
374
|
-
validateNodeConfiguration(node, index, errors, warnings);
|
|
442
|
+
validateNodeConfiguration(node, index, errors, warnings, codes);
|
|
375
443
|
});
|
|
376
444
|
if (!Array.isArray(recipe.edges)) {
|
|
377
445
|
errors.push('edges must be an array');
|
|
@@ -426,9 +494,15 @@ function validateRecipe(recipe, { mode } = {}) {
|
|
|
426
494
|
errors.push(`recipe ${recipe.id || '(unknown)'} is not compatible with ${selectedMode}`);
|
|
427
495
|
continue;
|
|
428
496
|
}
|
|
429
|
-
validateSafetyBackbone(recipe, selectedMode, byId, errors);
|
|
497
|
+
validateSafetyBackbone(recipe, selectedMode, byId, errors, codes);
|
|
430
498
|
}
|
|
431
|
-
return {
|
|
499
|
+
return {
|
|
500
|
+
valid: errors.length === 0,
|
|
501
|
+
errors: [...new Set(errors)],
|
|
502
|
+
warnings: [...new Set(warnings)],
|
|
503
|
+
codes: [...new Set(codes)],
|
|
504
|
+
order
|
|
505
|
+
};
|
|
432
506
|
}
|
|
433
507
|
|
|
434
508
|
function compileRecipe(recipe, { mode } = {}) {
|
|
@@ -436,7 +510,9 @@ function compileRecipe(recipe, { mode } = {}) {
|
|
|
436
510
|
const selectedMode = normalizeRunMode(mode || recipe?.compatibleModes?.[0]);
|
|
437
511
|
const validation = validateRecipe(recipe, { mode: selectedMode });
|
|
438
512
|
if (!validation.valid) {
|
|
439
|
-
|
|
513
|
+
const code = (validation.codes || []).find((item) => String(item).startsWith('BACKBONE_'))
|
|
514
|
+
|| 'INVALID_RECIPE';
|
|
515
|
+
throw recipeError(`Recipe ${recipe?.id || '(unknown)'} is invalid`, validation.errors, code);
|
|
440
516
|
}
|
|
441
517
|
const byId = new Map(recipe.nodes.map((node) => [node.id, node]));
|
|
442
518
|
const compiledNodes = validation.order.map((id, index) => {
|
|
@@ -615,6 +691,7 @@ module.exports = {
|
|
|
615
691
|
MAX_NODES,
|
|
616
692
|
MAX_EDGES,
|
|
617
693
|
BUILTIN_RECIPE_DIR,
|
|
694
|
+
BACKBONE_CODES,
|
|
618
695
|
canonicalize,
|
|
619
696
|
canonicalJson,
|
|
620
697
|
sha256,
|
package/lib/run-state.js
CHANGED
|
@@ -163,7 +163,9 @@ function writeWorkflowBundle(rootDir, runId, workflow) {
|
|
|
163
163
|
recipeId: plan.recipe.id,
|
|
164
164
|
recipeVersion: plan.recipe.version,
|
|
165
165
|
sourceHash: plan.recipe.sourceHash,
|
|
166
|
-
graphHash: plan.graphHash
|
|
166
|
+
graphHash: plan.graphHash,
|
|
167
|
+
...(workflow.workflowSource ? { workflowSource: workflow.workflowSource } : {}),
|
|
168
|
+
...(workflow.workflowReason ? { workflowReason: workflow.workflowReason } : {})
|
|
167
169
|
};
|
|
168
170
|
}
|
|
169
171
|
|
|
@@ -383,6 +385,13 @@ function transitionRun(
|
|
|
383
385
|
};
|
|
384
386
|
atomicWriteJson(runStatePath(rootDir, runId), next);
|
|
385
387
|
atomicWriteJson(currentRunPath(rootDir), { schemaVersion: 1, runId });
|
|
388
|
+
if (TERMINAL_STATES.has(to)) {
|
|
389
|
+
try {
|
|
390
|
+
require('./personal-workflow.js').appendRunSummary(rootDir, next, { now });
|
|
391
|
+
} catch {
|
|
392
|
+
// The run summary is advisory history; it must never undo a terminal transition.
|
|
393
|
+
}
|
|
394
|
+
}
|
|
386
395
|
return next;
|
|
387
396
|
}
|
|
388
397
|
|