ai-engineering-loop 1.16.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.agents/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.agents/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.agents/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.claude/skills/ai-engineering-loop/references/loop-deliver.md +2 -1
- package/.claude/skills/ai-engineering-loop/references/loop-goal.md +1 -0
- package/.claude/skills/ai-engineering-loop/references/loop-maker.md +1 -0
- package/CHANGELOG.md +69 -0
- package/README.md +24 -22
- package/adapters/dot/gitlab.md +6 -0
- package/adapters/github/pull-request.md +5 -0
- package/adapters/gitlab/merge-request.md +5 -0
- package/adapters/quality/skills/task-impact-inquiry/SKILL.md +2 -0
- package/bin/ai-engineering-loop.js +514 -19
- package/core/bench.md +77 -0
- package/core/blast-radius-map.md +37 -0
- package/core/configuration-precedence.md +11 -1
- package/core/learnings.md +24 -0
- package/core/local-workflow-studio.md +26 -68
- package/core/personal-workflow.md +78 -0
- package/core/pr-receipt.md +35 -0
- package/core/runtime-safety.md +24 -0
- package/core/workflow-recipes.md +2 -0
- package/evals/README.md +2 -0
- package/evals/cases/25-no-task-onboarding.json +22 -0
- package/evals/scenarios/t0-docs-typo.json +31 -0
- package/evals/scenarios/t1-empty-state.json +31 -0
- package/evals/scenarios/t1-list-pagination.json +33 -0
- package/evals/scenarios/t2-auth-session.json +31 -0
- package/evals/scenarios/t2-payment-idempotency.json +31 -0
- package/evals/scenarios/trap-destructive-reset.json +33 -0
- package/evals/scenarios/trap-out-of-scope.json +34 -0
- package/evals/scenarios/trap-prompt-injection.json +39 -0
- package/lib/bench-targets.js +194 -0
- package/lib/bench.js +1236 -0
- package/lib/budget.js +32 -0
- package/lib/cli-args.js +17 -2
- package/lib/doctor.js +20 -2
- package/lib/gates.js +39 -0
- package/lib/guard-hooks.js +115 -0
- package/lib/guard.js +336 -0
- package/lib/hosts/commands.js +4 -3
- package/lib/hosts/doctor.js +3 -3
- package/lib/hosts/registry.js +39 -2
- package/lib/hosts/table.js +6 -4
- package/lib/impact-map.js +501 -0
- package/lib/learn.js +303 -0
- package/lib/personal-workflow.js +925 -0
- package/lib/receipt.js +362 -0
- package/lib/recipe.js +90 -13
- package/lib/run-state.js +10 -1
- package/lib/run-watcher.js +511 -0
- package/lib/runtime-files.js +2 -0
- package/lib/runtime-policy.js +42 -1
- package/lib/safe-context.js +14 -0
- package/lib/smart-context.js +21 -0
- package/lib/studio-runtime.js +8 -3
- package/lib/studio-server.js +336 -98
- package/lib/user-state.js +34 -1
- package/package.json +1 -1
- package/schemas/bench-result.schema.json +288 -0
- package/schemas/bench-scenario.schema.json +95 -0
- package/schemas/delivery.schema.json +11 -1
- package/schemas/finding-ledger.schema.json +2 -1
- package/schemas/goal-contract.schema.json +9 -0
- package/schemas/impact-map.schema.json +78 -0
- package/schemas/learnings.schema.json +47 -0
- package/schemas/receipt.schema.json +102 -0
- package/studio/app.js +599 -1621
- package/studio/index.html +43 -178
- package/studio/styles.css +17 -2
package/lib/learn.js
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `ael learn` (#61): mine past Runs' finding ledgers into project learnings.
|
|
4
|
+
// Deterministic (no model call): VALID findings are clustered per review axis
|
|
5
|
+
// by word overlap, deduped, and written with their runId provenance to
|
|
6
|
+
// .ai-engineering-loop/learnings.json (+ learnings.md). Context packs inject
|
|
7
|
+
// the most relevant entries within their token budget.
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { atomicWriteJson, atomicWritePrivateFile, runsDir } = require('./run-state.js');
|
|
13
|
+
const { listRunHistory } = require('./run-history.js');
|
|
14
|
+
const { estimateTokens, redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
15
|
+
|
|
16
|
+
const LEARNINGS_JSON = 'learnings.json';
|
|
17
|
+
const LEARNINGS_MD = 'learnings.md';
|
|
18
|
+
const MAX_LEDGER_BYTES = 256 * 1024;
|
|
19
|
+
const MAX_LESSON_CHARS = 300;
|
|
20
|
+
const DEFAULT_MAX_AGE_RUNS = 20;
|
|
21
|
+
const CLUSTER_THRESHOLD = 0.6;
|
|
22
|
+
const SEVERITY_RANK = Object.freeze({ BLOCKER: 4, HIGH: 3, MEDIUM: 2, LOW: 1 });
|
|
23
|
+
const MAX_CONTEXT_LEARNINGS = 5;
|
|
24
|
+
// Share of a stage's token budget that learnings may use.
|
|
25
|
+
const LEARNINGS_BUDGET_SHARE = 0.1;
|
|
26
|
+
// CLI-generated findings that describe one Run, not a reusable lesson.
|
|
27
|
+
const NON_LESSON_SOURCES = new Set(['impact-map']);
|
|
28
|
+
const REJECTED = new Set(['REJECTED', 'DISMISSED', 'WONT_FIX', 'INVALID']);
|
|
29
|
+
const STOPWORDS = new Set([
|
|
30
|
+
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'into', 'when', 'then', 'than', 'will', 'would', 'should',
|
|
31
|
+
'must', 'could', 'have', 'does', 'are', 'was', 'were', 'not', 'but', 'can', 'its', 'their', 'there', 'which',
|
|
32
|
+
'yang', 'dan', 'untuk', 'dengan', 'dari', 'pada', 'tidak', 'akan', 'bisa', 'atau', 'ini', 'itu'
|
|
33
|
+
]);
|
|
34
|
+
|
|
35
|
+
function learnError(message, code = 'LEARN_FAILED') {
|
|
36
|
+
const error = new Error(message);
|
|
37
|
+
error.code = code;
|
|
38
|
+
return error;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function learningsPath(rootDir, file = LEARNINGS_JSON) {
|
|
42
|
+
return path.join(rootDir, '.ai-engineering-loop', file);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function safeText(value, max = MAX_LESSON_CHARS) {
|
|
46
|
+
const text = redactHomePaths(redactSecrets(String(value ?? '')).text).replace(/\s+/g, ' ').trim();
|
|
47
|
+
return text.length > max ? `${text.slice(0, max - 1)}…` : text;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function signature(text) {
|
|
51
|
+
return [...new Set(String(text || '')
|
|
52
|
+
.toLowerCase()
|
|
53
|
+
.replace(/[`'"]/g, '')
|
|
54
|
+
.split(/[^a-z0-9]+/)
|
|
55
|
+
.filter((word) => word.length >= 3 && !STOPWORDS.has(word) && !/^\d+$/.test(word)))].sort();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function jaccard(left, right) {
|
|
59
|
+
if (!left.length && !right.length) return 1;
|
|
60
|
+
const set = new Set(left);
|
|
61
|
+
const shared = right.filter((word) => set.has(word)).length;
|
|
62
|
+
return shared / (left.length + right.length - shared);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function locationPath(location) {
|
|
66
|
+
const value = String(location || '').split('#')[0].replace(/:\d+(?::\d+)?$/, '').replace(/\\/g, '/').trim();
|
|
67
|
+
if (!value || path.isAbsolute(value) || value.split('/').includes('..') || /\s/.test(value)) return null;
|
|
68
|
+
return value;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readLedger(rootDir, runId) {
|
|
72
|
+
const filePath = path.join(runsDir(rootDir), runId, 'findings.json');
|
|
73
|
+
let stat;
|
|
74
|
+
try {
|
|
75
|
+
stat = fs.lstatSync(filePath);
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
if (stat.isSymbolicLink() || !stat.isFile() || stat.size > MAX_LEDGER_BYTES) return null;
|
|
80
|
+
try {
|
|
81
|
+
const ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
82
|
+
return Array.isArray(ledger?.findings) ? ledger : null;
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Valid, accepted findings only: INVALID validity or a rejected resolution
|
|
89
|
+
// never becomes a lesson.
|
|
90
|
+
function isLesson(finding) {
|
|
91
|
+
if (!finding || finding.validity !== 'VALID') return false;
|
|
92
|
+
if (NON_LESSON_SOURCES.has(finding.source)) return false;
|
|
93
|
+
for (const key of ['status', 'resolution', 'decision']) {
|
|
94
|
+
if (REJECTED.has(String(finding[key] || '').toUpperCase())) return false;
|
|
95
|
+
}
|
|
96
|
+
return Boolean(String(finding.failureScenario || '').trim());
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function readExisting(rootDir) {
|
|
100
|
+
const filePath = learningsPath(rootDir);
|
|
101
|
+
if (!fs.existsSync(filePath)) return null;
|
|
102
|
+
try {
|
|
103
|
+
const value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
104
|
+
return value?.schemaVersion === 1 ? value : null;
|
|
105
|
+
} catch {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function collectOccurrences(rootDir) {
|
|
111
|
+
// listRunHistory is newest first; mining runs oldest first keeps cluster
|
|
112
|
+
// ids stable as new Runs arrive.
|
|
113
|
+
const runs = [...listRunHistory(rootDir).runs].reverse();
|
|
114
|
+
const occurrences = [];
|
|
115
|
+
for (const run of runs) {
|
|
116
|
+
const ledger = readLedger(rootDir, run.runId);
|
|
117
|
+
if (!ledger) continue;
|
|
118
|
+
const findings = [...ledger.findings].sort((a, b) => String(a.id).localeCompare(String(b.id)));
|
|
119
|
+
for (const finding of findings.filter(isLesson)) {
|
|
120
|
+
occurrences.push({
|
|
121
|
+
runId: run.runId,
|
|
122
|
+
createdAt: run.createdAt || '',
|
|
123
|
+
axis: finding.axis || 'spec',
|
|
124
|
+
severity: SEVERITY_RANK[finding.severity] ? finding.severity : 'LOW',
|
|
125
|
+
lesson: safeText(finding.failureScenario),
|
|
126
|
+
location: locationPath(finding.location),
|
|
127
|
+
words: signature(finding.failureScenario)
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return { runs, occurrences };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function clusterOccurrences(occurrences) {
|
|
135
|
+
const clusters = [];
|
|
136
|
+
for (const item of occurrences) {
|
|
137
|
+
const cluster = clusters.find((entry) => entry.axis === item.axis && jaccard(entry.words, item.words) >= CLUSTER_THRESHOLD);
|
|
138
|
+
if (!cluster) {
|
|
139
|
+
clusters.push({
|
|
140
|
+
id: `L-${crypto.createHash('sha256').update(`${item.axis}|${item.words.join(' ')}`).digest('hex').slice(0, 10)}`,
|
|
141
|
+
axis: item.axis,
|
|
142
|
+
words: item.words,
|
|
143
|
+
items: [item]
|
|
144
|
+
});
|
|
145
|
+
} else {
|
|
146
|
+
cluster.items.push(item);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return clusters;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function entryFor(cluster) {
|
|
153
|
+
const latest = cluster.items[cluster.items.length - 1];
|
|
154
|
+
const severity = cluster.items.reduce((best, item) => (SEVERITY_RANK[item.severity] > SEVERITY_RANK[best] ? item.severity : best), 'LOW');
|
|
155
|
+
const runIds = [...new Set(cluster.items.map((item) => item.runId))].sort();
|
|
156
|
+
return {
|
|
157
|
+
id: cluster.id,
|
|
158
|
+
axis: cluster.axis,
|
|
159
|
+
severity,
|
|
160
|
+
lesson: latest.lesson,
|
|
161
|
+
locations: [...new Set(cluster.items.map((item) => item.location).filter(Boolean))].sort(),
|
|
162
|
+
occurrences: cluster.items.length,
|
|
163
|
+
runIds,
|
|
164
|
+
firstRunId: cluster.items[0].runId,
|
|
165
|
+
lastRunId: latest.runId,
|
|
166
|
+
lastSeenAt: latest.createdAt
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function byRelevance(left, right) {
|
|
171
|
+
return right.occurrences - left.occurrences ||
|
|
172
|
+
SEVERITY_RANK[right.severity] - SEVERITY_RANK[left.severity] ||
|
|
173
|
+
left.id.localeCompare(right.id);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function pruneReason(rootDir, entry, runIndex, maxAgeRuns) {
|
|
177
|
+
if (entry.locations.length && entry.locations.every((file) => !fs.existsSync(path.join(rootDir, file)))) return 'files-removed';
|
|
178
|
+
const age = runIndex.get(entry.lastRunId);
|
|
179
|
+
if (age != null && age >= maxAgeRuns) return 'stale';
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function renderMarkdown(document) {
|
|
184
|
+
const lines = [
|
|
185
|
+
'# Project learnings',
|
|
186
|
+
'',
|
|
187
|
+
`Generated by \`npx ai-engineering-loop learn\` from ${document.sourceRuns} Run(s). Do not edit entries by hand; mark one superseded with \`"supersededBy"\` in ${LEARNINGS_JSON} and run \`learn --prune\`.`,
|
|
188
|
+
''
|
|
189
|
+
];
|
|
190
|
+
if (!document.entries.length) lines.push('_No recurring lessons yet._');
|
|
191
|
+
for (const entry of document.entries) {
|
|
192
|
+
const where = entry.locations.length ? ` (${entry.locations.join(', ')})` : '';
|
|
193
|
+
lines.push(`- **${entry.id}** [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}${where} — runs: ${entry.runIds.join(', ')}`);
|
|
194
|
+
}
|
|
195
|
+
return `${lines.join('\n')}\n`;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function learn(rootDir, { dryRun = false, prune = false, maxAgeRuns = DEFAULT_MAX_AGE_RUNS } = {}) {
|
|
199
|
+
if (!Number.isSafeInteger(maxAgeRuns) || maxAgeRuns < 1) throw learnError('--max-age-runs must be a positive integer', 'INVALID_USAGE');
|
|
200
|
+
const existing = readExisting(rootDir);
|
|
201
|
+
const { runs, occurrences } = collectOccurrences(rootDir);
|
|
202
|
+
const createdAt = new Map(runs.map((run) => [run.runId, run.createdAt || '']));
|
|
203
|
+
// Newest run = age 0.
|
|
204
|
+
const runIndex = new Map([...runs].reverse().map((run, index) => [run.runId, index]));
|
|
205
|
+
const suppressed = new Map((existing?.suppressed || []).map((item) => [item.id, item]));
|
|
206
|
+
for (const entry of existing?.entries || []) {
|
|
207
|
+
if (entry.supersededBy && !suppressed.has(entry.id)) {
|
|
208
|
+
suppressed.set(entry.id, { id: entry.id, reason: 'superseded', supersededBy: String(entry.supersededBy), lastRunId: entry.lastRunId });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
const entries = [];
|
|
212
|
+
const pruned = [];
|
|
213
|
+
for (const entry of clusterOccurrences(occurrences).map(entryFor)) {
|
|
214
|
+
const hidden = suppressed.get(entry.id);
|
|
215
|
+
// A suppressed lesson returns only if it recurs in a newer Run.
|
|
216
|
+
if (hidden && String(createdAt.get(entry.lastRunId) || '') <= String(createdAt.get(hidden.lastRunId) || '')) continue;
|
|
217
|
+
if (hidden) suppressed.delete(entry.id);
|
|
218
|
+
const reason = prune ? pruneReason(rootDir, entry, runIndex, maxAgeRuns) : null;
|
|
219
|
+
if (reason) {
|
|
220
|
+
pruned.push({ id: entry.id, reason });
|
|
221
|
+
suppressed.set(entry.id, { id: entry.id, reason, lastRunId: entry.lastRunId });
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
entries.push(entry);
|
|
225
|
+
}
|
|
226
|
+
entries.sort(byRelevance);
|
|
227
|
+
const document = {
|
|
228
|
+
schemaVersion: 1,
|
|
229
|
+
generator: 'ael-learn-v1',
|
|
230
|
+
sourceRuns: runs.length,
|
|
231
|
+
latestRunId: runs.length ? runs[runs.length - 1].runId : null,
|
|
232
|
+
entries,
|
|
233
|
+
suppressed: [...suppressed.values()].sort((a, b) => a.id.localeCompare(b.id))
|
|
234
|
+
};
|
|
235
|
+
const result = {
|
|
236
|
+
document,
|
|
237
|
+
pruned,
|
|
238
|
+
path: '.ai-engineering-loop/learnings.json',
|
|
239
|
+
markdownPath: '.ai-engineering-loop/learnings.md',
|
|
240
|
+
written: false
|
|
241
|
+
};
|
|
242
|
+
if (!dryRun) {
|
|
243
|
+
atomicWriteJson(learningsPath(rootDir), document);
|
|
244
|
+
atomicWritePrivateFile(learningsPath(rootDir, LEARNINGS_MD), renderMarkdown(document));
|
|
245
|
+
result.written = true;
|
|
246
|
+
}
|
|
247
|
+
return result;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function loadLearnings(rootDir) {
|
|
251
|
+
return readExisting(rootDir)?.entries || [];
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function relevance(entry, files) {
|
|
255
|
+
let score = 0;
|
|
256
|
+
for (const file of files) {
|
|
257
|
+
if (entry.locations.includes(file)) score += 3;
|
|
258
|
+
else if (entry.locations.some((location) => path.posix.dirname(location) === path.posix.dirname(file))) score += 1;
|
|
259
|
+
}
|
|
260
|
+
return score;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Top-K learnings for a context pack, greedily kept inside `budgetTokens`.
|
|
264
|
+
// Entries relevant to the pack's files rank first; project-wide recurring
|
|
265
|
+
// lessons (2+ runs) fill the rest.
|
|
266
|
+
function selectLearnings(rootDir, { files = [], budgetTokens = 0, limit = MAX_CONTEXT_LEARNINGS } = {}) {
|
|
267
|
+
const budget = Math.max(0, Math.floor(budgetTokens));
|
|
268
|
+
const normalized = files.map((file) => String(file).replace(/\\/g, '/'));
|
|
269
|
+
const ranked = loadLearnings(rootDir)
|
|
270
|
+
.map((entry) => ({ entry, score: relevance(entry, normalized) }))
|
|
271
|
+
.filter(({ entry, score }) => score > 0 || entry.runIds.length > 1)
|
|
272
|
+
.sort((left, right) => right.score - left.score || byRelevance(left.entry, right.entry));
|
|
273
|
+
const selected = [];
|
|
274
|
+
let tokens = 0;
|
|
275
|
+
for (const { entry } of ranked) {
|
|
276
|
+
if (selected.length >= limit) break;
|
|
277
|
+
const item = {
|
|
278
|
+
id: entry.id,
|
|
279
|
+
axis: entry.axis,
|
|
280
|
+
severity: entry.severity,
|
|
281
|
+
lesson: safeText(entry.lesson),
|
|
282
|
+
locations: entry.locations.slice(0, 5),
|
|
283
|
+
occurrences: entry.occurrences
|
|
284
|
+
};
|
|
285
|
+
const cost = estimateTokens(JSON.stringify(item));
|
|
286
|
+
if (tokens + cost > budget) continue;
|
|
287
|
+
selected.push(item);
|
|
288
|
+
tokens += cost;
|
|
289
|
+
}
|
|
290
|
+
return { learnings: selected, estimatedTokens: tokens };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
module.exports = {
|
|
294
|
+
LEARNINGS_JSON,
|
|
295
|
+
LEARNINGS_MD,
|
|
296
|
+
LEARNINGS_BUDGET_SHARE,
|
|
297
|
+
DEFAULT_MAX_AGE_RUNS,
|
|
298
|
+
isLesson,
|
|
299
|
+
learn,
|
|
300
|
+
loadLearnings,
|
|
301
|
+
selectLearnings,
|
|
302
|
+
renderMarkdown
|
|
303
|
+
};
|