ai-engineering-loop 1.16.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.agents/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.agents/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/.claude/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.claude/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.claude/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/CHANGELOG.md +32 -0
- package/README.md +24 -22
- package/adapters/dot/gitlab.md +6 -0
- package/adapters/github/pull-request.md +5 -0
- package/adapters/gitlab/merge-request.md +5 -0
- package/adapters/quality/skills/task-impact-inquiry/SKILL.md +2 -0
- package/bin/ai-engineering-loop.js +294 -4
- package/core/blast-radius-map.md +37 -0
- package/core/learnings.md +24 -0
- package/core/pr-receipt.md +35 -0
- package/core/runtime-safety.md +24 -0
- package/lib/cli-args.js +5 -1
- package/lib/doctor.js +4 -1
- package/lib/gates.js +39 -0
- package/lib/guard-hooks.js +115 -0
- package/lib/guard.js +336 -0
- package/lib/hosts/commands.js +4 -3
- package/lib/hosts/doctor.js +3 -3
- package/lib/hosts/registry.js +21 -2
- package/lib/hosts/table.js +6 -4
- package/lib/impact-map.js +501 -0
- package/lib/learn.js +303 -0
- package/lib/receipt.js +362 -0
- package/lib/safe-context.js +14 -0
- package/lib/smart-context.js +21 -0
- package/package.json +1 -1
- package/schemas/delivery.schema.json +11 -1
- package/schemas/finding-ledger.schema.json +2 -1
- package/schemas/goal-contract.schema.json +9 -0
- package/schemas/impact-map.schema.json +78 -0
- package/schemas/learnings.schema.json +47 -0
- package/schemas/receipt.schema.json +102 -0
package/lib/learn.js
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `ael learn` (#61): mine past Runs' finding ledgers into project learnings.
|
|
4
|
+
// Deterministic (no model call): VALID findings are clustered per review axis
|
|
5
|
+
// by word overlap, deduped, and written with their runId provenance to
|
|
6
|
+
// .ai-engineering-loop/learnings.json (+ learnings.md). Context packs inject
|
|
7
|
+
// the most relevant entries within their token budget.
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { atomicWriteJson, atomicWritePrivateFile, runsDir } = require('./run-state.js');
|
|
13
|
+
const { listRunHistory } = require('./run-history.js');
|
|
14
|
+
const { estimateTokens, redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
15
|
+
|
|
16
|
+
const LEARNINGS_JSON = 'learnings.json';
|
|
17
|
+
const LEARNINGS_MD = 'learnings.md';
|
|
18
|
+
const MAX_LEDGER_BYTES = 256 * 1024;
|
|
19
|
+
const MAX_LESSON_CHARS = 300;
|
|
20
|
+
const DEFAULT_MAX_AGE_RUNS = 20;
|
|
21
|
+
const CLUSTER_THRESHOLD = 0.6;
|
|
22
|
+
const SEVERITY_RANK = Object.freeze({ BLOCKER: 4, HIGH: 3, MEDIUM: 2, LOW: 1 });
|
|
23
|
+
const MAX_CONTEXT_LEARNINGS = 5;
|
|
24
|
+
// Share of a stage's token budget that learnings may use.
|
|
25
|
+
const LEARNINGS_BUDGET_SHARE = 0.1;
|
|
26
|
+
// CLI-generated findings that describe one Run, not a reusable lesson.
|
|
27
|
+
const NON_LESSON_SOURCES = new Set(['impact-map']);
|
|
28
|
+
const REJECTED = new Set(['REJECTED', 'DISMISSED', 'WONT_FIX', 'INVALID']);
|
|
29
|
+
const STOPWORDS = new Set([
|
|
30
|
+
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'into', 'when', 'then', 'than', 'will', 'would', 'should',
|
|
31
|
+
'must', 'could', 'have', 'does', 'are', 'was', 'were', 'not', 'but', 'can', 'its', 'their', 'there', 'which',
|
|
32
|
+
'yang', 'dan', 'untuk', 'dengan', 'dari', 'pada', 'tidak', 'akan', 'bisa', 'atau', 'ini', 'itu'
|
|
33
|
+
]);
|
|
34
|
+
|
|
35
|
+
function learnError(message, code = 'LEARN_FAILED') {
|
|
36
|
+
const error = new Error(message);
|
|
37
|
+
error.code = code;
|
|
38
|
+
return error;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function learningsPath(rootDir, file = LEARNINGS_JSON) {
|
|
42
|
+
return path.join(rootDir, '.ai-engineering-loop', file);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function safeText(value, max = MAX_LESSON_CHARS) {
|
|
46
|
+
const text = redactHomePaths(redactSecrets(String(value ?? '')).text).replace(/\s+/g, ' ').trim();
|
|
47
|
+
return text.length > max ? `${text.slice(0, max - 1)}…` : text;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function signature(text) {
|
|
51
|
+
return [...new Set(String(text || '')
|
|
52
|
+
.toLowerCase()
|
|
53
|
+
.replace(/[`'"]/g, '')
|
|
54
|
+
.split(/[^a-z0-9]+/)
|
|
55
|
+
.filter((word) => word.length >= 3 && !STOPWORDS.has(word) && !/^\d+$/.test(word)))].sort();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function jaccard(left, right) {
|
|
59
|
+
if (!left.length && !right.length) return 1;
|
|
60
|
+
const set = new Set(left);
|
|
61
|
+
const shared = right.filter((word) => set.has(word)).length;
|
|
62
|
+
return shared / (left.length + right.length - shared);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function locationPath(location) {
|
|
66
|
+
const value = String(location || '').split('#')[0].replace(/:\d+(?::\d+)?$/, '').replace(/\\/g, '/').trim();
|
|
67
|
+
if (!value || path.isAbsolute(value) || value.split('/').includes('..') || /\s/.test(value)) return null;
|
|
68
|
+
return value;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readLedger(rootDir, runId) {
|
|
72
|
+
const filePath = path.join(runsDir(rootDir), runId, 'findings.json');
|
|
73
|
+
let stat;
|
|
74
|
+
try {
|
|
75
|
+
stat = fs.lstatSync(filePath);
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
if (stat.isSymbolicLink() || !stat.isFile() || stat.size > MAX_LEDGER_BYTES) return null;
|
|
80
|
+
try {
|
|
81
|
+
const ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
82
|
+
return Array.isArray(ledger?.findings) ? ledger : null;
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Valid, accepted findings only: INVALID validity or a rejected resolution
|
|
89
|
+
// never becomes a lesson.
|
|
90
|
+
function isLesson(finding) {
|
|
91
|
+
if (!finding || finding.validity !== 'VALID') return false;
|
|
92
|
+
if (NON_LESSON_SOURCES.has(finding.source)) return false;
|
|
93
|
+
for (const key of ['status', 'resolution', 'decision']) {
|
|
94
|
+
if (REJECTED.has(String(finding[key] || '').toUpperCase())) return false;
|
|
95
|
+
}
|
|
96
|
+
return Boolean(String(finding.failureScenario || '').trim());
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function readExisting(rootDir) {
|
|
100
|
+
const filePath = learningsPath(rootDir);
|
|
101
|
+
if (!fs.existsSync(filePath)) return null;
|
|
102
|
+
try {
|
|
103
|
+
const value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
104
|
+
return value?.schemaVersion === 1 ? value : null;
|
|
105
|
+
} catch {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function collectOccurrences(rootDir) {
|
|
111
|
+
// listRunHistory is newest first; mining runs oldest first keeps cluster
|
|
112
|
+
// ids stable as new Runs arrive.
|
|
113
|
+
const runs = [...listRunHistory(rootDir).runs].reverse();
|
|
114
|
+
const occurrences = [];
|
|
115
|
+
for (const run of runs) {
|
|
116
|
+
const ledger = readLedger(rootDir, run.runId);
|
|
117
|
+
if (!ledger) continue;
|
|
118
|
+
const findings = [...ledger.findings].sort((a, b) => String(a.id).localeCompare(String(b.id)));
|
|
119
|
+
for (const finding of findings.filter(isLesson)) {
|
|
120
|
+
occurrences.push({
|
|
121
|
+
runId: run.runId,
|
|
122
|
+
createdAt: run.createdAt || '',
|
|
123
|
+
axis: finding.axis || 'spec',
|
|
124
|
+
severity: SEVERITY_RANK[finding.severity] ? finding.severity : 'LOW',
|
|
125
|
+
lesson: safeText(finding.failureScenario),
|
|
126
|
+
location: locationPath(finding.location),
|
|
127
|
+
words: signature(finding.failureScenario)
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return { runs, occurrences };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function clusterOccurrences(occurrences) {
|
|
135
|
+
const clusters = [];
|
|
136
|
+
for (const item of occurrences) {
|
|
137
|
+
const cluster = clusters.find((entry) => entry.axis === item.axis && jaccard(entry.words, item.words) >= CLUSTER_THRESHOLD);
|
|
138
|
+
if (!cluster) {
|
|
139
|
+
clusters.push({
|
|
140
|
+
id: `L-${crypto.createHash('sha256').update(`${item.axis}|${item.words.join(' ')}`).digest('hex').slice(0, 10)}`,
|
|
141
|
+
axis: item.axis,
|
|
142
|
+
words: item.words,
|
|
143
|
+
items: [item]
|
|
144
|
+
});
|
|
145
|
+
} else {
|
|
146
|
+
cluster.items.push(item);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return clusters;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function entryFor(cluster) {
|
|
153
|
+
const latest = cluster.items[cluster.items.length - 1];
|
|
154
|
+
const severity = cluster.items.reduce((best, item) => (SEVERITY_RANK[item.severity] > SEVERITY_RANK[best] ? item.severity : best), 'LOW');
|
|
155
|
+
const runIds = [...new Set(cluster.items.map((item) => item.runId))].sort();
|
|
156
|
+
return {
|
|
157
|
+
id: cluster.id,
|
|
158
|
+
axis: cluster.axis,
|
|
159
|
+
severity,
|
|
160
|
+
lesson: latest.lesson,
|
|
161
|
+
locations: [...new Set(cluster.items.map((item) => item.location).filter(Boolean))].sort(),
|
|
162
|
+
occurrences: cluster.items.length,
|
|
163
|
+
runIds,
|
|
164
|
+
firstRunId: cluster.items[0].runId,
|
|
165
|
+
lastRunId: latest.runId,
|
|
166
|
+
lastSeenAt: latest.createdAt
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function byRelevance(left, right) {
|
|
171
|
+
return right.occurrences - left.occurrences ||
|
|
172
|
+
SEVERITY_RANK[right.severity] - SEVERITY_RANK[left.severity] ||
|
|
173
|
+
left.id.localeCompare(right.id);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function pruneReason(rootDir, entry, runIndex, maxAgeRuns) {
|
|
177
|
+
if (entry.locations.length && entry.locations.every((file) => !fs.existsSync(path.join(rootDir, file)))) return 'files-removed';
|
|
178
|
+
const age = runIndex.get(entry.lastRunId);
|
|
179
|
+
if (age != null && age >= maxAgeRuns) return 'stale';
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function renderMarkdown(document) {
|
|
184
|
+
const lines = [
|
|
185
|
+
'# Project learnings',
|
|
186
|
+
'',
|
|
187
|
+
`Generated by \`npx ai-engineering-loop learn\` from ${document.sourceRuns} Run(s). Do not edit entries by hand; mark one superseded with \`"supersededBy"\` in ${LEARNINGS_JSON} and run \`learn --prune\`.`,
|
|
188
|
+
''
|
|
189
|
+
];
|
|
190
|
+
if (!document.entries.length) lines.push('_No recurring lessons yet._');
|
|
191
|
+
for (const entry of document.entries) {
|
|
192
|
+
const where = entry.locations.length ? ` (${entry.locations.join(', ')})` : '';
|
|
193
|
+
lines.push(`- **${entry.id}** [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}${where} — runs: ${entry.runIds.join(', ')}`);
|
|
194
|
+
}
|
|
195
|
+
return `${lines.join('\n')}\n`;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function learn(rootDir, { dryRun = false, prune = false, maxAgeRuns = DEFAULT_MAX_AGE_RUNS } = {}) {
|
|
199
|
+
if (!Number.isSafeInteger(maxAgeRuns) || maxAgeRuns < 1) throw learnError('--max-age-runs must be a positive integer', 'INVALID_USAGE');
|
|
200
|
+
const existing = readExisting(rootDir);
|
|
201
|
+
const { runs, occurrences } = collectOccurrences(rootDir);
|
|
202
|
+
const createdAt = new Map(runs.map((run) => [run.runId, run.createdAt || '']));
|
|
203
|
+
// Newest run = age 0.
|
|
204
|
+
const runIndex = new Map([...runs].reverse().map((run, index) => [run.runId, index]));
|
|
205
|
+
const suppressed = new Map((existing?.suppressed || []).map((item) => [item.id, item]));
|
|
206
|
+
for (const entry of existing?.entries || []) {
|
|
207
|
+
if (entry.supersededBy && !suppressed.has(entry.id)) {
|
|
208
|
+
suppressed.set(entry.id, { id: entry.id, reason: 'superseded', supersededBy: String(entry.supersededBy), lastRunId: entry.lastRunId });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
const entries = [];
|
|
212
|
+
const pruned = [];
|
|
213
|
+
for (const entry of clusterOccurrences(occurrences).map(entryFor)) {
|
|
214
|
+
const hidden = suppressed.get(entry.id);
|
|
215
|
+
// A suppressed lesson returns only if it recurs in a newer Run.
|
|
216
|
+
if (hidden && String(createdAt.get(entry.lastRunId) || '') <= String(createdAt.get(hidden.lastRunId) || '')) continue;
|
|
217
|
+
if (hidden) suppressed.delete(entry.id);
|
|
218
|
+
const reason = prune ? pruneReason(rootDir, entry, runIndex, maxAgeRuns) : null;
|
|
219
|
+
if (reason) {
|
|
220
|
+
pruned.push({ id: entry.id, reason });
|
|
221
|
+
suppressed.set(entry.id, { id: entry.id, reason, lastRunId: entry.lastRunId });
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
entries.push(entry);
|
|
225
|
+
}
|
|
226
|
+
entries.sort(byRelevance);
|
|
227
|
+
const document = {
|
|
228
|
+
schemaVersion: 1,
|
|
229
|
+
generator: 'ael-learn-v1',
|
|
230
|
+
sourceRuns: runs.length,
|
|
231
|
+
latestRunId: runs.length ? runs[runs.length - 1].runId : null,
|
|
232
|
+
entries,
|
|
233
|
+
suppressed: [...suppressed.values()].sort((a, b) => a.id.localeCompare(b.id))
|
|
234
|
+
};
|
|
235
|
+
const result = {
|
|
236
|
+
document,
|
|
237
|
+
pruned,
|
|
238
|
+
path: '.ai-engineering-loop/learnings.json',
|
|
239
|
+
markdownPath: '.ai-engineering-loop/learnings.md',
|
|
240
|
+
written: false
|
|
241
|
+
};
|
|
242
|
+
if (!dryRun) {
|
|
243
|
+
atomicWriteJson(learningsPath(rootDir), document);
|
|
244
|
+
atomicWritePrivateFile(learningsPath(rootDir, LEARNINGS_MD), renderMarkdown(document));
|
|
245
|
+
result.written = true;
|
|
246
|
+
}
|
|
247
|
+
return result;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function loadLearnings(rootDir) {
|
|
251
|
+
return readExisting(rootDir)?.entries || [];
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function relevance(entry, files) {
|
|
255
|
+
let score = 0;
|
|
256
|
+
for (const file of files) {
|
|
257
|
+
if (entry.locations.includes(file)) score += 3;
|
|
258
|
+
else if (entry.locations.some((location) => path.posix.dirname(location) === path.posix.dirname(file))) score += 1;
|
|
259
|
+
}
|
|
260
|
+
return score;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Top-K learnings for a context pack, greedily kept inside `budgetTokens`.
|
|
264
|
+
// Entries relevant to the pack's files rank first; project-wide recurring
|
|
265
|
+
// lessons (2+ runs) fill the rest.
|
|
266
|
+
function selectLearnings(rootDir, { files = [], budgetTokens = 0, limit = MAX_CONTEXT_LEARNINGS } = {}) {
|
|
267
|
+
const budget = Math.max(0, Math.floor(budgetTokens));
|
|
268
|
+
const normalized = files.map((file) => String(file).replace(/\\/g, '/'));
|
|
269
|
+
const ranked = loadLearnings(rootDir)
|
|
270
|
+
.map((entry) => ({ entry, score: relevance(entry, normalized) }))
|
|
271
|
+
.filter(({ entry, score }) => score > 0 || entry.runIds.length > 1)
|
|
272
|
+
.sort((left, right) => right.score - left.score || byRelevance(left.entry, right.entry));
|
|
273
|
+
const selected = [];
|
|
274
|
+
let tokens = 0;
|
|
275
|
+
for (const { entry } of ranked) {
|
|
276
|
+
if (selected.length >= limit) break;
|
|
277
|
+
const item = {
|
|
278
|
+
id: entry.id,
|
|
279
|
+
axis: entry.axis,
|
|
280
|
+
severity: entry.severity,
|
|
281
|
+
lesson: safeText(entry.lesson),
|
|
282
|
+
locations: entry.locations.slice(0, 5),
|
|
283
|
+
occurrences: entry.occurrences
|
|
284
|
+
};
|
|
285
|
+
const cost = estimateTokens(JSON.stringify(item));
|
|
286
|
+
if (tokens + cost > budget) continue;
|
|
287
|
+
selected.push(item);
|
|
288
|
+
tokens += cost;
|
|
289
|
+
}
|
|
290
|
+
return { learnings: selected, estimatedTokens: tokens };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
module.exports = {
|
|
294
|
+
LEARNINGS_JSON,
|
|
295
|
+
LEARNINGS_MD,
|
|
296
|
+
LEARNINGS_BUDGET_SHARE,
|
|
297
|
+
DEFAULT_MAX_AGE_RUNS,
|
|
298
|
+
isLesson,
|
|
299
|
+
learn,
|
|
300
|
+
loadLearnings,
|
|
301
|
+
selectLearnings,
|
|
302
|
+
renderMarkdown
|
|
303
|
+
};
|
package/lib/receipt.js
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Proof-carrying PR receipt (#70). `deliver` writes RUN_DIR/receipt.json:
|
|
4
|
+
// the frozen Goal and its hash, per-AC evidence, the DA/Judge result, base and
|
|
5
|
+
// head SHA, the sha256 of every referenced artifact, and an overall digest.
|
|
6
|
+
// `verify-receipt` re-checks all of it, so a reviewer (or CI) can trust the
|
|
7
|
+
// PR body without re-reading the Run.
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { execFileSync } = require('child_process');
|
|
13
|
+
const { getCurrentRun, loadRun, runsDir } = require('./run-state.js');
|
|
14
|
+
const { canonicalJson, sha256 } = require('./recipe.js');
|
|
15
|
+
const { parseClaimedVsReality } = require('./claimed-vs-reality.js');
|
|
16
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
17
|
+
const { isBlockingFinding } = require('./orchestration.js');
|
|
18
|
+
|
|
19
|
+
const RECEIPT_FILE = 'receipt.json';
|
|
20
|
+
const RECEIPT_MARKDOWN_FILE = 'receipt.md';
|
|
21
|
+
const CLAIMED_VS_REALITY = '.ai-engineering-loop/tasks/claimed-vs-reality.md';
|
|
22
|
+
const RUN_ARTIFACTS = Object.freeze([
|
|
23
|
+
['goalContract', 'goal-contract.json', true],
|
|
24
|
+
['diff', 'diff.patch', true],
|
|
25
|
+
['verification', 'verification.json', true],
|
|
26
|
+
['findings', 'findings.json', true],
|
|
27
|
+
['verdict', 'verdict.json', true],
|
|
28
|
+
['impactMap', 'impact-map.json', false],
|
|
29
|
+
['impactDrift', 'impact-drift.json', false],
|
|
30
|
+
['guardEvents', 'guard-events.json', false]
|
|
31
|
+
]);
|
|
32
|
+
const MAX_TEXT = 500;
|
|
33
|
+
|
|
34
|
+
function receiptError(message, code = 'RECEIPT_FAILED', details = []) {
|
|
35
|
+
const error = new Error(message);
|
|
36
|
+
error.code = code;
|
|
37
|
+
error.details = details;
|
|
38
|
+
return error;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function resolveRun(rootDir, runId) {
|
|
42
|
+
if (runId) return loadRun(rootDir, runId);
|
|
43
|
+
const current = getCurrentRun(rootDir);
|
|
44
|
+
if (!current) throw receiptError('No current Run. Pass --run <id>.', 'NO_CURRENT_RUN');
|
|
45
|
+
return current;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function hashBytes(buffer) {
|
|
49
|
+
return crypto.createHash('sha256').update(buffer).digest('hex');
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function repoRelative(rootDir, filePath) {
|
|
53
|
+
return path.relative(rootDir, filePath).split(path.sep).join('/');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Receipts leave the machine: no secrets and no local absolute paths.
|
|
57
|
+
function safeText(value) {
|
|
58
|
+
return redactHomePaths(redactSecrets(String(value ?? '')).text)
|
|
59
|
+
.replace(/(?:^|\s)\/(?:tmp|var|private|Users|home)\/\S*/g, ' [REDACTED PATH]')
|
|
60
|
+
.trim()
|
|
61
|
+
.slice(0, MAX_TEXT);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function readJson(filePath, label) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
67
|
+
} catch (cause) {
|
|
68
|
+
throw receiptError(`${label} is missing or invalid: ${cause.message}`, 'RECEIPT_INPUT_MISSING');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function gitHead(rootDir) {
|
|
73
|
+
try {
|
|
74
|
+
return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
75
|
+
} catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function validateLink(url) {
|
|
81
|
+
let parsed;
|
|
82
|
+
try {
|
|
83
|
+
parsed = new URL(String(url));
|
|
84
|
+
} catch {
|
|
85
|
+
parsed = null;
|
|
86
|
+
}
|
|
87
|
+
if (!parsed || !['https:', 'http:'].includes(parsed.protocol) || parsed.username || parsed.password) {
|
|
88
|
+
throw receiptError(`Evidence links must be hosted http(s) URLs without credentials: ${safeText(url)}`, 'INVALID_EVIDENCE_LINK');
|
|
89
|
+
}
|
|
90
|
+
return parsed.toString();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// `--evidence AC-1=https://ci.example/run/1` (repeatable).
|
|
94
|
+
function parseEvidenceLinks(values = []) {
|
|
95
|
+
const links = {};
|
|
96
|
+
for (const value of values) {
|
|
97
|
+
const match = String(value).match(/^(AC-\d+)=(.+)$/);
|
|
98
|
+
if (!match) throw receiptError(`--evidence must be AC-N=<https-url>, received ${safeText(value)}`, 'INVALID_EVIDENCE_LINK');
|
|
99
|
+
(links[match[1]] ||= []).push(validateLink(match[2]));
|
|
100
|
+
}
|
|
101
|
+
return links;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function artifactEntries(rootDir, run) {
|
|
105
|
+
const directory = path.join(runsDir(rootDir), run.runId);
|
|
106
|
+
const entries = [];
|
|
107
|
+
for (const [name, file, required] of RUN_ARTIFACTS) {
|
|
108
|
+
const filePath = path.join(directory, file);
|
|
109
|
+
if (!fs.existsSync(filePath)) {
|
|
110
|
+
if (required) throw receiptError(`Receipt needs ${file}; finish the gates first`, 'RECEIPT_INPUT_MISSING');
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
entries.push({ name, path: repoRelative(rootDir, filePath), sha256: hashBytes(fs.readFileSync(filePath)) });
|
|
114
|
+
}
|
|
115
|
+
const claimed = path.join(rootDir, CLAIMED_VS_REALITY);
|
|
116
|
+
if (fs.existsSync(claimed)) {
|
|
117
|
+
entries.push({ name: 'claimedVsReality', path: CLAIMED_VS_REALITY, sha256: hashBytes(fs.readFileSync(claimed)) });
|
|
118
|
+
}
|
|
119
|
+
return entries;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function commandEvidence(command) {
|
|
123
|
+
return {
|
|
124
|
+
type: 'command',
|
|
125
|
+
command: safeText(command.command),
|
|
126
|
+
exitCode: command.exitCode,
|
|
127
|
+
timeoutStatus: command.timeoutStatus,
|
|
128
|
+
outputSha256: sha256(`${command.stdout || ''}\n${command.stderr || ''}`),
|
|
129
|
+
passed: command.exitCode === 0 && command.timeoutStatus === 'COMPLETED'
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Optional coverage numbers from verification.json (e.g. { lines: 91.2,
|
|
134
|
+
// linesDelta: 0.4 }); only finite numbers are kept, never paths or text.
|
|
135
|
+
function coverageSummary(coverage) {
|
|
136
|
+
if (!coverage || typeof coverage !== 'object' || Array.isArray(coverage)) return null;
|
|
137
|
+
const numbers = Object.entries(coverage).filter(([key, value]) => /^[A-Za-z0-9_-]{1,40}$/.test(key) && Number.isFinite(value));
|
|
138
|
+
return numbers.length ? Object.fromEntries(numbers.sort(([a], [b]) => a.localeCompare(b))) : null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function criterionEntry(criterion, { commands, claims, links }) {
|
|
142
|
+
const claim = claims.find((row) => row.ac === criterion.id);
|
|
143
|
+
const evidence = [];
|
|
144
|
+
const missing = [];
|
|
145
|
+
if (claim && String(claim.reality || '').trim()) {
|
|
146
|
+
evidence.push({ type: 'claim', claimed: safeText(claim.claimed), reality: safeText(claim.reality) });
|
|
147
|
+
} else {
|
|
148
|
+
missing.push('no claimed-vs-reality row with a Reality cell');
|
|
149
|
+
}
|
|
150
|
+
if (!commands.length) missing.push('no recorded verification command');
|
|
151
|
+
else if (commands.some((item) => !item.passed)) missing.push('a verification command did not pass');
|
|
152
|
+
evidence.push(...commands);
|
|
153
|
+
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', url });
|
|
154
|
+
return {
|
|
155
|
+
id: criterion.id,
|
|
156
|
+
statement: safeText(criterion.statement),
|
|
157
|
+
evidenceRequired: safeText(criterion.evidenceRequired),
|
|
158
|
+
status: missing.length ? 'MISSING' : 'PASS',
|
|
159
|
+
...(missing.length ? { missing } : {}),
|
|
160
|
+
evidence
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function receiptDigest(receipt) {
|
|
165
|
+
const { digest: _digest, ...body } = receipt;
|
|
166
|
+
return sha256(canonicalJson(body));
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function buildReceipt(rootDir, { runId, host, model, head, evidenceLinks = [], aelVersion, now = new Date() } = {}) {
|
|
170
|
+
const run = resolveRun(rootDir, runId);
|
|
171
|
+
const directory = path.join(runsDir(rootDir), run.runId);
|
|
172
|
+
const artifacts = artifactEntries(rootDir, run);
|
|
173
|
+
const goalFile = path.join(directory, 'goal-contract.json');
|
|
174
|
+
const goal = readJson(goalFile, 'goal-contract.json');
|
|
175
|
+
const verification = readJson(path.join(directory, 'verification.json'), 'verification.json');
|
|
176
|
+
const findings = readJson(path.join(directory, 'findings.json'), 'findings.json');
|
|
177
|
+
const verdict = readJson(path.join(directory, 'verdict.json'), 'verdict.json');
|
|
178
|
+
const claimedFile = path.join(rootDir, CLAIMED_VS_REALITY);
|
|
179
|
+
const claims = fs.existsSync(claimedFile) ? parseClaimedVsReality(fs.readFileSync(claimedFile, 'utf8')).rows : [];
|
|
180
|
+
const commands = (verification.commands || []).map(commandEvidence);
|
|
181
|
+
const links = parseEvidenceLinks(evidenceLinks);
|
|
182
|
+
const unknownLinks = Object.keys(links).filter((id) => !(goal.acceptanceCriteria || []).some((item) => item.id === id));
|
|
183
|
+
if (unknownLinks.length) throw receiptError(`--evidence names unknown AC: ${unknownLinks.join(', ')}`, 'INVALID_EVIDENCE_LINK');
|
|
184
|
+
const ledger = Array.isArray(findings.findings) ? findings.findings : [];
|
|
185
|
+
const drift = artifacts.some((item) => item.name === 'impactDrift')
|
|
186
|
+
? readJson(path.join(directory, 'impact-drift.json'), 'impact-drift.json')
|
|
187
|
+
: null;
|
|
188
|
+
const guardEvents = artifacts.some((item) => item.name === 'guardEvents')
|
|
189
|
+
? (readJson(path.join(directory, 'guard-events.json'), 'guard-events.json').events || [])
|
|
190
|
+
: [];
|
|
191
|
+
const bySeverity = {};
|
|
192
|
+
for (const finding of ledger) bySeverity[finding.severity] = (bySeverity[finding.severity] || 0) + 1;
|
|
193
|
+
const receipt = {
|
|
194
|
+
schemaVersion: 1,
|
|
195
|
+
runId: run.runId,
|
|
196
|
+
generatedAt: now.toISOString(),
|
|
197
|
+
aelVersion: aelVersion || null,
|
|
198
|
+
host: { id: safeText(host || 'unspecified'), model: model ? safeText(model) : (verdict.model ? safeText(verdict.model) : null) },
|
|
199
|
+
base: { sha: run.startedRevision || null },
|
|
200
|
+
head: { sha: head || gitHead(rootDir) },
|
|
201
|
+
goal: {
|
|
202
|
+
path: repoRelative(rootDir, goalFile),
|
|
203
|
+
sha256: hashBytes(fs.readFileSync(goalFile)),
|
|
204
|
+
contentHash: sha256(goal),
|
|
205
|
+
version: run.goal?.version || 1,
|
|
206
|
+
objective: safeText(goal.objective)
|
|
207
|
+
},
|
|
208
|
+
acceptanceCriteria: (goal.acceptanceCriteria || []).map((criterion) => criterionEntry(criterion, { commands, claims, links })),
|
|
209
|
+
verification: {
|
|
210
|
+
diffHash: verification.diffHash,
|
|
211
|
+
gitRevision: verification.gitRevision,
|
|
212
|
+
commands,
|
|
213
|
+
coverage: coverageSummary(verification.coverage)
|
|
214
|
+
},
|
|
215
|
+
review: {
|
|
216
|
+
findings: {
|
|
217
|
+
total: ledger.length,
|
|
218
|
+
valid: ledger.filter((item) => item.validity === 'VALID').length,
|
|
219
|
+
blocking: ledger.filter(isBlockingFinding).length,
|
|
220
|
+
bySeverity
|
|
221
|
+
},
|
|
222
|
+
verdict: {
|
|
223
|
+
verdict: verdict.verdict,
|
|
224
|
+
modelTier: verdict.modelTier || null,
|
|
225
|
+
decidedBy: safeText(verdict.decidedBy || 'judge'),
|
|
226
|
+
reason: safeText(verdict.reason)
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
impact: drift ? { mapStatus: drift.mapStatus, outOfMap: (drift.outOfMap || []).map(safeText) } : null,
|
|
230
|
+
guard: {
|
|
231
|
+
warn: guardEvents.filter((event) => event.decision === 'warn').length,
|
|
232
|
+
block: guardEvents.filter((event) => event.decision === 'block').length
|
|
233
|
+
},
|
|
234
|
+
artifacts
|
|
235
|
+
};
|
|
236
|
+
receipt.digest = receiptDigest(receipt);
|
|
237
|
+
return receipt;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function markdownCell(value) {
|
|
241
|
+
return String(value ?? '').replace(/\r?\n/g, ' ').replace(/\|/g, '\\|');
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function short(sha, length = 12) {
|
|
245
|
+
return sha ? String(sha).slice(0, length) : 'unknown';
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function renderReceiptMarkdown(receipt) {
|
|
249
|
+
const lines = [
|
|
250
|
+
'## Receipt',
|
|
251
|
+
'',
|
|
252
|
+
`Run \`${receipt.runId}\` · Goal v${receipt.goal.version} \`${short(receipt.goal.contentHash)}\` · base \`${short(receipt.base.sha, 7)}\` → head \`${short(receipt.head.sha, 7)}\` · digest \`${short(receipt.digest)}\``,
|
|
253
|
+
''
|
|
254
|
+
];
|
|
255
|
+
for (const criterion of receipt.acceptanceCriteria) {
|
|
256
|
+
const parts = [];
|
|
257
|
+
for (const item of criterion.evidence) {
|
|
258
|
+
if (item.type === 'command') parts.push(`\`${markdownCell(item.command)}\` exit ${item.exitCode}`);
|
|
259
|
+
if (item.type === 'claim') parts.push(`reality: ${markdownCell(item.reality)}`);
|
|
260
|
+
if (item.type === 'link') parts.push(`[evidence](${item.url})`);
|
|
261
|
+
}
|
|
262
|
+
const missing = criterion.missing ? ` — **missing:** ${criterion.missing.join('; ')}` : '';
|
|
263
|
+
lines.push(`- ${criterion.status === 'PASS' ? '✅' : '❌'} **${criterion.id}** ${markdownCell(criterion.statement)}${parts.length ? ` — ${parts.join(', ')}` : ''}${missing}`);
|
|
264
|
+
}
|
|
265
|
+
const { findings, verdict } = receipt.review;
|
|
266
|
+
lines.push('');
|
|
267
|
+
lines.push(`Judge: **${verdict.verdict}**${verdict.modelTier ? ` (${verdict.modelTier})` : ''} · Findings: ${findings.total} (${findings.valid} valid, ${findings.blocking} blocking)` +
|
|
268
|
+
`${receipt.impact ? ` · Impact map: ${receipt.impact.mapStatus}, ${receipt.impact.outOfMap.length} file(s) outside` : ''}` +
|
|
269
|
+
` · Guard: ${receipt.guard.block} blocked, ${receipt.guard.warn} warned`);
|
|
270
|
+
lines.push('');
|
|
271
|
+
lines.push(`Check: \`npx ai-engineering-loop verify-receipt --run ${receipt.runId} --head ${short(receipt.head.sha, 40)}\``);
|
|
272
|
+
return `${lines.join('\n')}\n`;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function writeReceipt(rootDir, options = {}) {
|
|
276
|
+
const receipt = buildReceipt(rootDir, options);
|
|
277
|
+
const directory = path.join(runsDir(rootDir), receipt.runId);
|
|
278
|
+
const jsonPath = path.join(directory, RECEIPT_FILE);
|
|
279
|
+
const markdownPath = path.join(directory, RECEIPT_MARKDOWN_FILE);
|
|
280
|
+
fs.writeFileSync(jsonPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
|
|
281
|
+
fs.writeFileSync(markdownPath, renderReceiptMarkdown(receipt), { mode: 0o600 });
|
|
282
|
+
return {
|
|
283
|
+
receipt,
|
|
284
|
+
path: repoRelative(rootDir, jsonPath),
|
|
285
|
+
markdownPath: repoRelative(rootDir, markdownPath),
|
|
286
|
+
missing: receipt.acceptanceCriteria.filter((item) => item.status !== 'PASS').map((item) => item.id)
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function insideRepo(rootDir, relativePath) {
|
|
291
|
+
const absolute = path.resolve(rootDir, relativePath);
|
|
292
|
+
const relative = path.relative(rootDir, absolute);
|
|
293
|
+
if (!relativePath || path.isAbsolute(relativePath) || relative === '..' || relative.startsWith(`..${path.sep}`)) return null;
|
|
294
|
+
return absolute;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function verifyReceipt(rootDir, { runId, receiptPath, head } = {}) {
|
|
298
|
+
let file;
|
|
299
|
+
if (receiptPath) {
|
|
300
|
+
file = insideRepo(rootDir, receiptPath);
|
|
301
|
+
if (!file) throw receiptError('Receipt path must be repository-relative', 'UNSAFE_RECEIPT_PATH');
|
|
302
|
+
} else {
|
|
303
|
+
file = path.join(runsDir(rootDir), resolveRun(rootDir, runId).runId, RECEIPT_FILE);
|
|
304
|
+
}
|
|
305
|
+
const receipt = readJson(file, 'receipt.json');
|
|
306
|
+
const problems = [];
|
|
307
|
+
const fail = (code, detail) => problems.push({ code, detail });
|
|
308
|
+
if (receipt.schemaVersion !== 1 || !receipt.runId) fail('RECEIPT_INVALID', 'schemaVersion 1 and runId are required');
|
|
309
|
+
if (receiptDigest(receipt) !== receipt.digest) fail('RECEIPT_DIGEST_MISMATCH', 'receipt content does not match its digest');
|
|
310
|
+
|
|
311
|
+
for (const artifact of receipt.artifacts || []) {
|
|
312
|
+
const absolute = insideRepo(rootDir, artifact.path);
|
|
313
|
+
if (!absolute || !fs.existsSync(absolute)) {
|
|
314
|
+
fail('ARTIFACT_MISSING', `${artifact.path} is missing`);
|
|
315
|
+
} else if (hashBytes(fs.readFileSync(absolute)) !== artifact.sha256) {
|
|
316
|
+
fail('ARTIFACT_TAMPERED', `${artifact.path} changed after the receipt`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const goalFile = receipt.goal?.path && insideRepo(rootDir, receipt.goal.path);
|
|
321
|
+
let goal = null;
|
|
322
|
+
if (!goalFile || !fs.existsSync(goalFile)) {
|
|
323
|
+
fail('GOAL_CHANGED', 'the frozen Goal is missing');
|
|
324
|
+
} else {
|
|
325
|
+
goal = JSON.parse(fs.readFileSync(goalFile, 'utf8'));
|
|
326
|
+
if (hashBytes(fs.readFileSync(goalFile)) !== receipt.goal.sha256 || sha256(goal) !== receipt.goal.contentHash) {
|
|
327
|
+
fail('GOAL_CHANGED', 'the frozen Goal differs from the one in the receipt');
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const listed = new Map((receipt.acceptanceCriteria || []).map((item) => [item.id, item]));
|
|
331
|
+
for (const criterion of goal?.acceptanceCriteria || []) {
|
|
332
|
+
const entry = listed.get(criterion.id);
|
|
333
|
+
if (!entry) fail('AC_NOT_IN_RECEIPT', `${criterion.id} is not in the receipt`);
|
|
334
|
+
else if (entry.status !== 'PASS' || !(entry.evidence || []).length) {
|
|
335
|
+
fail('AC_EVIDENCE_MISSING', `${criterion.id}: ${(entry.missing || ['no evidence']).join('; ')}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
if (receipt.review?.verdict?.verdict !== 'PASS') fail('VERDICT_NOT_PASS', `Judge verdict is ${receipt.review?.verdict?.verdict || 'missing'}`);
|
|
339
|
+
if (receipt.review?.findings?.blocking) fail('BLOCKING_FINDINGS', `${receipt.review.findings.blocking} blocking finding(s)`);
|
|
340
|
+
const expectedHead = head || gitHead(rootDir);
|
|
341
|
+
if (!receipt.head?.sha || !expectedHead || receipt.head.sha !== expectedHead) {
|
|
342
|
+
fail('HEAD_MISMATCH', `receipt head ${short(receipt.head?.sha, 7)} vs ${head ? '--head' : 'HEAD'} ${short(expectedHead, 7)}`);
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
ok: problems.length === 0,
|
|
346
|
+
runId: receipt.runId,
|
|
347
|
+
digest: receipt.digest,
|
|
348
|
+
path: repoRelative(rootDir, file),
|
|
349
|
+
problems
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
module.exports = {
|
|
354
|
+
RECEIPT_FILE,
|
|
355
|
+
RECEIPT_MARKDOWN_FILE,
|
|
356
|
+
buildReceipt,
|
|
357
|
+
writeReceipt,
|
|
358
|
+
verifyReceipt,
|
|
359
|
+
renderReceiptMarkdown,
|
|
360
|
+
receiptDigest,
|
|
361
|
+
parseEvidenceLinks
|
|
362
|
+
};
|