ai-engineering-loop 1.11.1 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/animate/RECIPES.md +324 -0
- package/.agents/skills/animate/SKILL.md +207 -0
- package/.agents/skills/animate-expo/RECIPES.md +385 -0
- package/.agents/skills/animate-expo/SKILL.md +263 -0
- package/.agents/skills/animation-vocabulary/SKILL.md +181 -0
- package/.agents/skills/apple-design/SKILL.md +290 -0
- package/.agents/skills/ask-sonner/API.md +64 -0
- package/.agents/skills/ask-sonner/SKILL.md +88 -0
- package/.agents/skills/emil-design-eng/SKILL.md +674 -0
- package/.agents/skills/find-animation-opportunities/SKILL.md +140 -0
- package/.agents/skills/improve-animations/AUDIT.md +115 -0
- package/.agents/skills/improve-animations/PLAN-TEMPLATE.md +73 -0
- package/.agents/skills/improve-animations/SKILL.md +109 -0
- package/.agents/skills/mobile-native/SKILL.md +311 -0
- package/.agents/skills/pick-ui-library/SKILL.md +85 -0
- package/.agents/skills/prototype/PICKER.md +197 -0
- package/.agents/skills/prototype/SKILL.md +98 -0
- package/.agents/skills/review-animations/SKILL.md +120 -0
- package/.agents/skills/review-animations/STANDARDS.md +187 -0
- package/.agents/skills/write-swift/SKILL.md +396 -0
- package/.agents/workflows/ai-engineering-loop.md +2 -2
- package/.claude/skills/ai-engineering-loop/SKILL.md +3 -3
- package/.claude/skills/animate/RECIPES.md +324 -0
- package/.claude/skills/animate/SKILL.md +207 -0
- package/.claude/skills/animate-expo/RECIPES.md +385 -0
- package/.claude/skills/animate-expo/SKILL.md +263 -0
- package/.claude/skills/animation-vocabulary/SKILL.md +181 -0
- package/.claude/skills/apple-design/SKILL.md +290 -0
- package/.claude/skills/ask-sonner/API.md +64 -0
- package/.claude/skills/ask-sonner/SKILL.md +88 -0
- package/.claude/skills/emil-design-eng/SKILL.md +674 -0
- package/.claude/skills/find-animation-opportunities/SKILL.md +140 -0
- package/.claude/skills/improve-animations/AUDIT.md +115 -0
- package/.claude/skills/improve-animations/PLAN-TEMPLATE.md +73 -0
- package/.claude/skills/improve-animations/SKILL.md +109 -0
- package/.claude/skills/mobile-native/SKILL.md +311 -0
- package/.claude/skills/pick-ui-library/SKILL.md +85 -0
- package/.claude/skills/prototype/PICKER.md +197 -0
- package/.claude/skills/prototype/SKILL.md +98 -0
- package/.claude/skills/review-animations/SKILL.md +120 -0
- package/.claude/skills/review-animations/STANDARDS.md +187 -0
- package/.claude/skills/write-swift/SKILL.md +396 -0
- package/.gemini/skills/ai-engineering-loop/SKILL.md +3 -3
- package/.gemini/skills/animate/RECIPES.md +324 -0
- package/.gemini/skills/animate/SKILL.md +207 -0
- package/.gemini/skills/animate-expo/RECIPES.md +385 -0
- package/.gemini/skills/animate-expo/SKILL.md +263 -0
- package/.gemini/skills/animation-vocabulary/SKILL.md +181 -0
- package/.gemini/skills/apple-design/SKILL.md +290 -0
- package/.gemini/skills/ask-sonner/API.md +64 -0
- package/.gemini/skills/ask-sonner/SKILL.md +88 -0
- package/.gemini/skills/emil-design-eng/SKILL.md +674 -0
- package/.gemini/skills/find-animation-opportunities/SKILL.md +140 -0
- package/.gemini/skills/improve-animations/AUDIT.md +115 -0
- package/.gemini/skills/improve-animations/PLAN-TEMPLATE.md +73 -0
- package/.gemini/skills/improve-animations/SKILL.md +109 -0
- package/.gemini/skills/mobile-native/SKILL.md +311 -0
- package/.gemini/skills/pick-ui-library/SKILL.md +85 -0
- package/.gemini/skills/prototype/PICKER.md +197 -0
- package/.gemini/skills/prototype/SKILL.md +98 -0
- package/.gemini/skills/review-animations/SKILL.md +120 -0
- package/.gemini/skills/review-animations/STANDARDS.md +187 -0
- package/.gemini/skills/write-swift/SKILL.md +396 -0
- package/.grok/skills/ai-engineering-loop/SKILL.md +3 -3
- package/.grok/skills/animate/RECIPES.md +324 -0
- package/.grok/skills/animate/SKILL.md +207 -0
- package/.grok/skills/animate-expo/RECIPES.md +385 -0
- package/.grok/skills/animate-expo/SKILL.md +263 -0
- package/.grok/skills/animation-vocabulary/SKILL.md +181 -0
- package/.grok/skills/apple-design/SKILL.md +290 -0
- package/.grok/skills/ask-sonner/API.md +64 -0
- package/.grok/skills/ask-sonner/SKILL.md +88 -0
- package/.grok/skills/emil-design-eng/SKILL.md +674 -0
- package/.grok/skills/find-animation-opportunities/SKILL.md +140 -0
- package/.grok/skills/improve-animations/AUDIT.md +115 -0
- package/.grok/skills/improve-animations/PLAN-TEMPLATE.md +73 -0
- package/.grok/skills/improve-animations/SKILL.md +109 -0
- package/.grok/skills/mobile-native/SKILL.md +311 -0
- package/.grok/skills/pick-ui-library/SKILL.md +85 -0
- package/.grok/skills/prototype/PICKER.md +197 -0
- package/.grok/skills/prototype/SKILL.md +98 -0
- package/.grok/skills/review-animations/SKILL.md +120 -0
- package/.grok/skills/review-animations/STANDARDS.md +187 -0
- package/.grok/skills/write-swift/SKILL.md +396 -0
- package/README.md +13 -1
- package/README.npm.md +2 -1
- package/THIRD_PARTY_NOTICES.md +10 -0
- package/agents/third-party/emilkowalski-skills/LICENSE +21 -0
- package/bin/ai-engineering-loop.js +158 -9
- package/core/artifact-gates.md +32 -1
- package/core/runtime-safety.md +2 -1
- package/core/verification-loop.md +6 -0
- package/lib/agent-artifacts.js +250 -0
- package/lib/claimed-vs-reality.js +134 -0
- package/lib/doctor.js +12 -0
- package/lib/safe-context.js +62 -4
- package/lib/sync-hosts.js +97 -3
- package/lib/verification-recorder.js +224 -0
- package/lib/workflow-runtime.js +26 -1
- package/package.json +2 -1
- package/policies/review-budget.md +1 -1
- package/schemas/verification.schema.json +2 -2
|
@@ -39,7 +39,8 @@ const {
|
|
|
39
39
|
createOrResumeRun,
|
|
40
40
|
getCurrentRun,
|
|
41
41
|
loadRun,
|
|
42
|
-
newRunId
|
|
42
|
+
newRunId,
|
|
43
|
+
runsDir
|
|
43
44
|
} = require('../lib/run-state.js');
|
|
44
45
|
const {
|
|
45
46
|
DEFAULT_POLICY,
|
|
@@ -80,6 +81,16 @@ const {
|
|
|
80
81
|
const {
|
|
81
82
|
writeVerificationSummary
|
|
82
83
|
} = require('../lib/verification-summary.js');
|
|
84
|
+
const {
|
|
85
|
+
recordVerificationCommand
|
|
86
|
+
} = require('../lib/verification-recorder.js');
|
|
87
|
+
const {
|
|
88
|
+
scaffoldArtifact,
|
|
89
|
+
validateArtifactFile
|
|
90
|
+
} = require('../lib/agent-artifacts.js');
|
|
91
|
+
const {
|
|
92
|
+
createClaimedVsRealityTemplate
|
|
93
|
+
} = require('../lib/claimed-vs-reality.js');
|
|
83
94
|
const {
|
|
84
95
|
runDoctor
|
|
85
96
|
} = require('../lib/doctor.js');
|
|
@@ -1163,12 +1174,72 @@ function handleSandbox() {
|
|
|
1163
1174
|
}
|
|
1164
1175
|
}
|
|
1165
1176
|
|
|
1166
|
-
function handleVerification() {
|
|
1177
|
+
async function handleVerification() {
|
|
1167
1178
|
const action = process.argv[3] || 'summarize';
|
|
1168
|
-
const
|
|
1169
|
-
const
|
|
1179
|
+
const separatorIndex = process.argv.indexOf('--');
|
|
1180
|
+
const optionArgs = action === 'record' && separatorIndex !== -1
|
|
1181
|
+
? process.argv.slice(0, separatorIndex)
|
|
1182
|
+
: process.argv;
|
|
1183
|
+
const readOption = (name) => {
|
|
1184
|
+
const index = optionArgs.indexOf(name);
|
|
1185
|
+
return index === -1 ? null : optionArgs[index + 1] || null;
|
|
1186
|
+
};
|
|
1187
|
+
const json = optionArgs.includes('--json');
|
|
1188
|
+
const runId = readOption('--run');
|
|
1170
1189
|
try {
|
|
1171
|
-
if (action
|
|
1190
|
+
if (action === 'record') {
|
|
1191
|
+
if (separatorIndex === -1 || !process.argv[separatorIndex + 1]) {
|
|
1192
|
+
throw Object.assign(
|
|
1193
|
+
new Error('Provide an executable after `--`, for example `ai-engineering-loop verification record --run <id> -- npm test`.'),
|
|
1194
|
+
{ code: 'INVALID_COMMAND' }
|
|
1195
|
+
);
|
|
1196
|
+
}
|
|
1197
|
+
const command = process.argv[separatorIndex + 1];
|
|
1198
|
+
const commandArgs = process.argv.slice(separatorIndex + 2);
|
|
1199
|
+
const timeoutValue = readOption('--timeout-ms');
|
|
1200
|
+
const result = await recordVerificationCommand(CWD, {
|
|
1201
|
+
runId,
|
|
1202
|
+
command,
|
|
1203
|
+
args: commandArgs,
|
|
1204
|
+
shell: optionArgs.includes('--shell'),
|
|
1205
|
+
timeoutMs: timeoutValue == null
|
|
1206
|
+
? undefined
|
|
1207
|
+
: parsePositiveIntegerArg('--timeout-ms', timeoutValue)
|
|
1208
|
+
});
|
|
1209
|
+
if (json) {
|
|
1210
|
+
console.log(JSON.stringify({
|
|
1211
|
+
ok: result.command.exitCode === 0 && result.command.timeoutStatus === 'COMPLETED',
|
|
1212
|
+
runId: result.runId,
|
|
1213
|
+
path: result.path,
|
|
1214
|
+
summaryPath: path.relative(CWD, path.join(runsDir(CWD), result.runId, 'verification-summary.json'))
|
|
1215
|
+
.split(path.sep).join('/'),
|
|
1216
|
+
command: result.command.command,
|
|
1217
|
+
exitCode: result.command.exitCode,
|
|
1218
|
+
timeoutStatus: result.command.timeoutStatus,
|
|
1219
|
+
testCounts: result.summary.testCounts
|
|
1220
|
+
}));
|
|
1221
|
+
} else {
|
|
1222
|
+
const passed = result.command.exitCode === 0 && result.command.timeoutStatus === 'COMPLETED';
|
|
1223
|
+
if (passed) log.success('✓ Verification command recorded');
|
|
1224
|
+
else log.error('Verification command recorded with a failing result');
|
|
1225
|
+
console.log(`- Run: ${result.runId}`);
|
|
1226
|
+
console.log(`- Command: ${result.command.command}`);
|
|
1227
|
+
console.log(`- Exit code: ${result.command.exitCode}; timeout: ${result.command.timeoutStatus}`);
|
|
1228
|
+
console.log(`- Evidence: ${result.path}`);
|
|
1229
|
+
console.log(`- Summary: ${JSON.stringify(result.summary.testCounts)}`);
|
|
1230
|
+
if (!passed) {
|
|
1231
|
+
const excerpt = [result.command.stderr, result.command.stdout]
|
|
1232
|
+
.map((value) => String(value || '').trim())
|
|
1233
|
+
.find(Boolean);
|
|
1234
|
+
if (excerpt) console.error(`- Output: ${excerpt.split('\n').slice(-3).join('\n').slice(-800)}`);
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
if (result.command.exitCode !== 0 || result.command.timeoutStatus !== 'COMPLETED') {
|
|
1238
|
+
process.exitCode = result.command.exitCode > 0 ? result.command.exitCode : 1;
|
|
1239
|
+
}
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
if (action !== 'summarize') throw new Error('Unknown verification action. Use summarize|record');
|
|
1172
1243
|
const result = writeVerificationSummary(CWD, { runId });
|
|
1173
1244
|
if (json) {
|
|
1174
1245
|
console.log(JSON.stringify({
|
|
@@ -1192,13 +1263,51 @@ function handleVerification() {
|
|
|
1192
1263
|
}
|
|
1193
1264
|
}
|
|
1194
1265
|
|
|
1266
|
+
function handleArtifactTool(action) {
|
|
1267
|
+
const json = process.argv.includes('--json');
|
|
1268
|
+
try {
|
|
1269
|
+
if (action === 'scaffold') {
|
|
1270
|
+
const type = process.argv[3];
|
|
1271
|
+
const result = scaffoldArtifact(CWD, { type, runId: argValue('--run') });
|
|
1272
|
+
if (json) console.log(JSON.stringify({ ok: true, ...result, templateOnly: true }));
|
|
1273
|
+
else {
|
|
1274
|
+
log.success(`✓ ${result.type} template created`);
|
|
1275
|
+
console.log(`- Run: ${result.runId}`);
|
|
1276
|
+
console.log(`- Path: ${result.path}`);
|
|
1277
|
+
console.log('- Template is deliberately incomplete; it cannot pass its gate until replaced with observed evidence.');
|
|
1278
|
+
}
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
if (action === 'validate') {
|
|
1282
|
+
const file = process.argv[3];
|
|
1283
|
+
if (!file || file.startsWith('-')) throw new Error('validate requires a repository-relative JSON file');
|
|
1284
|
+
const result = validateArtifactFile(CWD, { file, type: argValue('--type') });
|
|
1285
|
+
if (json) console.log(JSON.stringify({ ok: result.valid, ...result }));
|
|
1286
|
+
else {
|
|
1287
|
+
console.log(`Artifact ${result.path}: ${result.valid ? 'VALID' : 'INVALID'}`);
|
|
1288
|
+
for (const error of result.errors) console.error(`- ${error}`);
|
|
1289
|
+
if (result.note) console.log(`- ${result.note}`);
|
|
1290
|
+
}
|
|
1291
|
+
if (!result.valid) process.exitCode = 1;
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
throw new Error('Use scaffold <verification|findings|delivery> or validate <file>');
|
|
1295
|
+
} catch (err) {
|
|
1296
|
+
if (json) console.log(JSON.stringify({ ok: false, code: err.code || 'ARTIFACT_TOOL_FAILED', error: err.message }));
|
|
1297
|
+
else log.error(err.message);
|
|
1298
|
+
process.exit(1);
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1195
1302
|
function handleGate() {
|
|
1196
1303
|
const gate = process.argv[3];
|
|
1197
1304
|
const runId = argValue('--run');
|
|
1198
1305
|
const json = process.argv.includes('--json');
|
|
1306
|
+
let targetRunId = runId;
|
|
1199
1307
|
try {
|
|
1200
1308
|
const target = runId ? loadRun(CWD, runId) : getCurrentRun(CWD);
|
|
1201
1309
|
if (!target) throw new Error('No current run. Start one with `ai-engineering-loop run`.');
|
|
1310
|
+
targetRunId = target.runId;
|
|
1202
1311
|
listRunLifecycle(CWD, target.runId);
|
|
1203
1312
|
let lifecycleNode = null;
|
|
1204
1313
|
if (target.workflow) {
|
|
@@ -1243,16 +1352,20 @@ function handleGate() {
|
|
|
1243
1352
|
console.log(`- State: ${run.state}, iteration ${run.iteration}`);
|
|
1244
1353
|
}
|
|
1245
1354
|
} catch (err) {
|
|
1355
|
+
const retryHint = targetRunId && gate && err.code === 'GATE_FAILED'
|
|
1356
|
+
? `After correcting the reported issue, retry: npx ai-engineering-loop gate ${gate} --run ${targetRunId}`
|
|
1357
|
+
: null;
|
|
1358
|
+
const details = [...(err.details || []), ...(retryHint ? [retryHint] : [])];
|
|
1246
1359
|
if (json) {
|
|
1247
1360
|
console.log(JSON.stringify({
|
|
1248
1361
|
ok: false,
|
|
1249
1362
|
code: err.code || 'GATE_FAILED',
|
|
1250
1363
|
error: err.message,
|
|
1251
|
-
details
|
|
1364
|
+
details
|
|
1252
1365
|
}));
|
|
1253
1366
|
} else {
|
|
1254
1367
|
log.error(err.message);
|
|
1255
|
-
for (const detail of
|
|
1368
|
+
for (const detail of details) console.error(`- ${detail}`);
|
|
1256
1369
|
}
|
|
1257
1370
|
process.exit(1);
|
|
1258
1371
|
}
|
|
@@ -1385,7 +1498,7 @@ function handleContext() {
|
|
|
1385
1498
|
const limit = Number(argValue('--limit') || 10);
|
|
1386
1499
|
const json = process.argv.includes('--json');
|
|
1387
1500
|
const includeIgnored = process.argv.includes('--include-ignored');
|
|
1388
|
-
const files = commandFiles(4, ['--run', '--profile', '--base', '--query', '--limit']);
|
|
1501
|
+
const files = commandFiles(4, ['--run', '--profile', '--base', '--query', '--limit', '--output']);
|
|
1389
1502
|
try {
|
|
1390
1503
|
if (stage === 'index') {
|
|
1391
1504
|
const result = buildContextIndex(CWD, { files, includeIgnored });
|
|
@@ -1445,6 +1558,20 @@ function handleContext() {
|
|
|
1445
1558
|
}
|
|
1446
1559
|
return;
|
|
1447
1560
|
}
|
|
1561
|
+
if (stage === 'claimed-vs-reality') {
|
|
1562
|
+
const result = createClaimedVsRealityTemplate(CWD, {
|
|
1563
|
+
runId,
|
|
1564
|
+
outputPath: argValue('--output') || undefined
|
|
1565
|
+
});
|
|
1566
|
+
if (json) console.log(JSON.stringify({ ok: true, ...result }));
|
|
1567
|
+
else {
|
|
1568
|
+
log.success('✓ Claimed-vs-reality template created from the frozen Goal');
|
|
1569
|
+
console.log(`- Run: ${result.runId}`);
|
|
1570
|
+
console.log(`- Path: ${result.path}`);
|
|
1571
|
+
console.log('- Claimed and Reality remain blank; populate Reality only from actual command output.');
|
|
1572
|
+
}
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1448
1575
|
if (stage === 'diff-hunks' || stage === 'diff-pack') {
|
|
1449
1576
|
const result = createDiffHunkPack(CWD, { runId, base, profile, includeIgnored });
|
|
1450
1577
|
const summary = smartContextSummary(result);
|
|
@@ -1473,9 +1600,16 @@ function handleContext() {
|
|
|
1473
1600
|
}
|
|
1474
1601
|
} catch (err) {
|
|
1475
1602
|
if (json) {
|
|
1476
|
-
console.log(JSON.stringify({
|
|
1603
|
+
console.log(JSON.stringify({
|
|
1604
|
+
ok: false,
|
|
1605
|
+
code: err.code || 'UNSAFE_CONTEXT',
|
|
1606
|
+
error: err.message,
|
|
1607
|
+
details: err.details || [],
|
|
1608
|
+
...(err.budget ? { budget: err.budget } : {})
|
|
1609
|
+
}));
|
|
1477
1610
|
} else {
|
|
1478
1611
|
log.error(err.message);
|
|
1612
|
+
for (const detail of err.details || []) console.error(`- ${detail}`);
|
|
1479
1613
|
}
|
|
1480
1614
|
process.exit(1);
|
|
1481
1615
|
}
|
|
@@ -2089,6 +2223,13 @@ Commands:
|
|
|
2089
2223
|
freeze --revision <n> --hash <sha256> [--run <id>] [--by <name>]
|
|
2090
2224
|
verification Summarize verification.json for compact Judge context
|
|
2091
2225
|
summarize [--run <id>] [--json]
|
|
2226
|
+
record [--run <id>] [--timeout-ms <n>] [--shell] [--json] -- <executable> [args...]
|
|
2227
|
+
Captures bounded, redacted output; updates verification.json and its summary
|
|
2228
|
+
--shell opts into a shell; pass one quoted command string, e.g. -- "npm test"
|
|
2229
|
+
scaffold Create a private, deliberately incomplete stage-artifact template
|
|
2230
|
+
verification | findings | delivery [--run <id>] [--json]
|
|
2231
|
+
validate Check a verification, findings, or delivery JSON file without advancing a gate
|
|
2232
|
+
<repository-relative-file> [--type verification|findings|delivery] [--json]
|
|
2092
2233
|
activity Publish safe Run lifecycle summaries for Studio
|
|
2093
2234
|
goal-drafting | goal-ready | heartbeat | waiting
|
|
2094
2235
|
node-started | node-activity | node-completed | loop-iterated | run-completed
|
|
@@ -2125,6 +2266,8 @@ Commands:
|
|
|
2125
2266
|
diff-hunks build a hunk-only review pack with unresolved findings
|
|
2126
2267
|
fast-path assess whether a small task can stay on lean review
|
|
2127
2268
|
compact write run-summary.md/json and open-findings.json
|
|
2269
|
+
claimed-vs-reality scaffold AC rows from the frozen Goal; evidence stays blank
|
|
2270
|
+
[--run <id>] [--output <repository-relative-file>]
|
|
2128
2271
|
--run <id> target a non-current run
|
|
2129
2272
|
--profile lean|standard|thorough override runtime policy for one pack
|
|
2130
2273
|
--base <git-ref> base ref for diff-hunks; default HEAD
|
|
@@ -2222,6 +2365,12 @@ switch (command) {
|
|
|
2222
2365
|
case 'verification':
|
|
2223
2366
|
handleVerification();
|
|
2224
2367
|
break;
|
|
2368
|
+
case 'scaffold':
|
|
2369
|
+
handleArtifactTool('scaffold');
|
|
2370
|
+
break;
|
|
2371
|
+
case 'validate':
|
|
2372
|
+
handleArtifactTool('validate');
|
|
2373
|
+
break;
|
|
2225
2374
|
case 'sandbox':
|
|
2226
2375
|
handleSandbox();
|
|
2227
2376
|
break;
|
package/core/artifact-gates.md
CHANGED
|
@@ -28,6 +28,37 @@ JSON sidecars follow the schemas shipped in `schemas/`. Keep the Markdown Goal C
|
|
|
28
28
|
|
|
29
29
|
Gate failures are evidence. Do not edit state to bypass one. Repair the artifact or return to Maker.
|
|
30
30
|
|
|
31
|
+
## Agent-friendly artifact diagnostics
|
|
32
|
+
|
|
33
|
+
Use the recorder for verification commands instead of hand-writing process metadata:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
npx ai-engineering-loop verification record --run <run-id> -- npm test
|
|
37
|
+
npx ai-engineering-loop verification summarize --run <run-id>
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The recorder runs commands without a shell by default, limits and redacts captured output, binds it to the gated Maker diff, and preserves failed results as failed evidence. Use `--shell -- "command"` only when shell syntax is required. A scaffold is only a starting point, never gate evidence:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
npx ai-engineering-loop scaffold delivery --run <run-id>
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Copy or rename the generated `.template.json` to the gate's real artifact filename (for example, `delivery.json`), fill it with observed evidence, and remove `_templateOnly`. Then validate the real file:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
npx ai-engineering-loop validate .ai-engineering-loop/runs/<run-id>/delivery.json
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`validate` reports artifact-content problems without advancing the workflow; its gate remains responsible for freshness, readiness, and human approval.
|
|
53
|
+
|
|
54
|
+
For a frozen Goal, generate the claimed-vs-reality table with blank evidence fields and fill them only from actual changes and verification output:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
npx ai-engineering-loop context claimed-vs-reality --run <run-id>
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
In a recipe-bound run, `gate delivery` may report that a prerequisite node is ready but not approved. Inspect `node status --run <run-id>`, then explicitly approve that named approval node; never skip the human checkpoint.
|
|
61
|
+
|
|
31
62
|
## Context boundary
|
|
32
63
|
|
|
33
64
|
Create model-visible context through:
|
|
@@ -38,7 +69,7 @@ npx ai-engineering-loop context devil-advocate <files...>
|
|
|
38
69
|
npx ai-engineering-loop context judge <files...>
|
|
39
70
|
```
|
|
40
71
|
|
|
41
|
-
The command rejects sensitive paths and repository escapes, redacts recognized credentials, caps files and estimated tokens, and marks truncation. Never replace a bounded pack with the parent transcript.
|
|
72
|
+
The command rejects sensitive paths and repository escapes, redacts recognized credentials, caps files and estimated tokens, and marks truncation. Budget errors report the limit, estimated requested size, and per-file contribution (paths and counts only). Never replace a bounded pack with the parent transcript.
|
|
42
73
|
|
|
43
74
|
Before Judge, run:
|
|
44
75
|
|
package/core/runtime-safety.md
CHANGED
|
@@ -69,11 +69,12 @@ npx ai-engineering-loop context related lib/studio-server.js
|
|
|
69
69
|
npx ai-engineering-loop context fast-path --json
|
|
70
70
|
npx ai-engineering-loop context diff-hunks --run <run-id>
|
|
71
71
|
npx ai-engineering-loop context diff-hunks --run <run-id> --include-ignored
|
|
72
|
+
npx ai-engineering-loop verification record --run <run-id> -- npm test
|
|
72
73
|
npx ai-engineering-loop verification summarize --run <run-id>
|
|
73
74
|
npx ai-engineering-loop context compact --run <run-id>
|
|
74
75
|
```
|
|
75
76
|
|
|
76
|
-
The index stores file summaries and hashes, never file bodies, and reuses per-hash summaries when files are unchanged. `query` and `related` return metadata only so agents can choose the right files before reading source. `fast-path` keeps low-risk small diffs on lean review and names the reason when a task must use the standard path. The diff-hunk pack stores bounded changed hunks, file summaries, token estimates, unresolved blocking findings from the previous ledger, and a count of paths skipped by context policy. Smart context skips CSS, generated bundles, lockfiles, build output, and `.aelcontextignore` matches by default; pass `--include-ignored` only when those files are the task.
|
|
77
|
+
The index stores file summaries and hashes, never file bodies, and reuses per-hash summaries when files are unchanged. `query` and `related` return metadata only so agents can choose the right files before reading source. `fast-path` keeps low-risk small diffs on lean review and names the reason when a task must use the standard path. The diff-hunk pack stores bounded changed hunks, file summaries, token estimates, unresolved blocking findings from the previous ledger, and a count of paths skipped by context policy. Smart context skips CSS, generated bundles, lockfiles, build output, and `.aelcontextignore` matches by default; pass `--include-ignored` only when those files are the task. `verification record` executes the command without a shell by default, stores bounded output after secret and local-path redaction, binds it to the gated Maker diff, and records failures instead of hiding them. `verification summarize` extracts available test counts and short failure excerpts without forwarding raw logs. Context budget errors include per-file estimates (paths and counts only). `context compact` writes `run-summary.md`, `run-summary.json`, and `open-findings.json` so iteration 2+ reviewers and Judge start from compact artifacts and request full source or logs only when the hunk or summary cannot prove or disprove a finding.
|
|
77
78
|
|
|
78
79
|
## Package update workflow
|
|
79
80
|
|
|
@@ -70,6 +70,12 @@ The Verification Gate immediately halts and returns to the Maker if:
|
|
|
70
70
|
|
|
71
71
|
Write `.ai-engineering-loop/tasks/claimed-vs-reality.md` after verification logs exist:
|
|
72
72
|
|
|
73
|
+
```bash
|
|
74
|
+
npx ai-engineering-loop context claimed-vs-reality --run <run-id>
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
This scaffolds one row per frozen acceptance criterion but leaves Claimed and Reality empty. Fill them from the implemented change and actual command output; the scaffold is not verification evidence.
|
|
78
|
+
|
|
73
79
|
```markdown
|
|
74
80
|
# Claimed vs Reality
|
|
75
81
|
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { validateFindingLedger } = require('./orchestration.js');
|
|
6
|
+
const { validateVerificationBundle } = require('./gates.js');
|
|
7
|
+
const {
|
|
8
|
+
getCurrentRun,
|
|
9
|
+
getGitRevision,
|
|
10
|
+
loadRun,
|
|
11
|
+
runsDir
|
|
12
|
+
} = require('./run-state.js');
|
|
13
|
+
|
|
14
|
+
const ARTIFACT_TYPES = Object.freeze({
|
|
15
|
+
verification: 'verification',
|
|
16
|
+
findings: 'findings',
|
|
17
|
+
delivery: 'delivery'
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
function artifactError(message, code = 'ARTIFACT_TOOL_FAILED') {
|
|
21
|
+
const error = new Error(message);
|
|
22
|
+
error.code = code;
|
|
23
|
+
return error;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function resolveRun(rootDir, runId) {
|
|
27
|
+
if (runId) return loadRun(rootDir, runId);
|
|
28
|
+
const current = getCurrentRun(rootDir);
|
|
29
|
+
if (!current) throw artifactError('No current Run. Pass --run <id>.', 'NO_CURRENT_RUN');
|
|
30
|
+
return current;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function assertNoSymlinkPath(rootDir, targetPath) {
|
|
34
|
+
const root = path.resolve(rootDir);
|
|
35
|
+
const target = path.resolve(targetPath);
|
|
36
|
+
const relative = path.relative(root, target);
|
|
37
|
+
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
38
|
+
throw artifactError('Artifact path must stay inside the repository.', 'UNSAFE_ARTIFACT_PATH');
|
|
39
|
+
}
|
|
40
|
+
let current = root;
|
|
41
|
+
for (const segment of relative.split(path.sep).filter(Boolean)) {
|
|
42
|
+
current = path.join(current, segment);
|
|
43
|
+
try {
|
|
44
|
+
if (fs.lstatSync(current).isSymbolicLink()) {
|
|
45
|
+
throw artifactError(`Artifact path crosses a symlink: ${path.relative(root, current)}`, 'UNSAFE_ARTIFACT_PATH');
|
|
46
|
+
}
|
|
47
|
+
} catch (cause) {
|
|
48
|
+
if (cause.code === 'ENOENT') return;
|
|
49
|
+
throw cause;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function templateFor(type, run, rootDir) {
|
|
55
|
+
const common = {
|
|
56
|
+
schemaVersion: 1,
|
|
57
|
+
runId: run.runId,
|
|
58
|
+
_templateOnly: true,
|
|
59
|
+
_instructions: 'Template only. Replace placeholders with observed evidence before validation or gating.'
|
|
60
|
+
};
|
|
61
|
+
const diffHash = run.artifacts?.diff?.sha256 || '__FILL_AFTER_GATE_MAKER__';
|
|
62
|
+
|
|
63
|
+
if (type === 'verification') {
|
|
64
|
+
return {
|
|
65
|
+
...common,
|
|
66
|
+
gitRevision: getGitRevision(rootDir),
|
|
67
|
+
diffHash,
|
|
68
|
+
commands: [{
|
|
69
|
+
command: '__FILL_WITH_COMMAND__',
|
|
70
|
+
executionIdentity: '__FILL_WITH_PROCESS_OR_AGENT_ID__',
|
|
71
|
+
startTime: '__FILL_WITH_ISO_TIMESTAMP__',
|
|
72
|
+
endTime: '__FILL_WITH_ISO_TIMESTAMP__',
|
|
73
|
+
exitCode: null,
|
|
74
|
+
stdout: '',
|
|
75
|
+
timeoutStatus: 'NOT_RUN'
|
|
76
|
+
}]
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (type === 'findings') {
|
|
80
|
+
return {
|
|
81
|
+
...common,
|
|
82
|
+
diffHash,
|
|
83
|
+
findings: [{
|
|
84
|
+
id: '',
|
|
85
|
+
axis: 'spec',
|
|
86
|
+
location: '',
|
|
87
|
+
failureScenario: '',
|
|
88
|
+
evidence: '',
|
|
89
|
+
severity: '',
|
|
90
|
+
validity: '',
|
|
91
|
+
disposition: ''
|
|
92
|
+
}]
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
if (type === 'delivery') {
|
|
96
|
+
return {
|
|
97
|
+
...common,
|
|
98
|
+
destination: '',
|
|
99
|
+
summary: '',
|
|
100
|
+
humanApproved: false
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
throw artifactError(`Unknown artifact type '${type}'. Use ${Object.keys(ARTIFACT_TYPES).join('|')}.`, 'UNKNOWN_ARTIFACT_TYPE');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function scaffoldArtifact(rootDir, { type, runId } = {}) {
|
|
107
|
+
if (!Object.hasOwn(ARTIFACT_TYPES, type)) {
|
|
108
|
+
throw artifactError(`Unknown artifact type '${type || ''}'. Use ${Object.keys(ARTIFACT_TYPES).join('|')}.`, 'UNKNOWN_ARTIFACT_TYPE');
|
|
109
|
+
}
|
|
110
|
+
const run = resolveRun(rootDir, runId);
|
|
111
|
+
const filePath = path.join(runsDir(rootDir), run.runId, `${type}.template.json`);
|
|
112
|
+
const directory = path.dirname(filePath);
|
|
113
|
+
const repositoryPath = path.resolve(rootDir);
|
|
114
|
+
const realRoot = fs.realpathSync(rootDir);
|
|
115
|
+
assertNoSymlinkPath(repositoryPath, directory);
|
|
116
|
+
const realDirectory = fs.realpathSync(directory);
|
|
117
|
+
if (!realDirectory.startsWith(`${realRoot}${path.sep}`)) {
|
|
118
|
+
throw artifactError('Artifact template path escapes the repository.', 'UNSAFE_ARTIFACT_PATH');
|
|
119
|
+
}
|
|
120
|
+
let descriptor;
|
|
121
|
+
try {
|
|
122
|
+
descriptor = fs.openSync(filePath, 'wx', 0o600);
|
|
123
|
+
} catch (cause) {
|
|
124
|
+
if (cause.code === 'EEXIST') {
|
|
125
|
+
throw artifactError(`Template already exists: ${path.relative(rootDir, filePath)}. It was left unchanged.`, 'ARTIFACT_EXISTS');
|
|
126
|
+
}
|
|
127
|
+
throw cause;
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
fs.writeFileSync(descriptor, `${JSON.stringify(templateFor(type, run, rootDir), null, 2)}\n`);
|
|
131
|
+
fs.fsyncSync(descriptor);
|
|
132
|
+
} finally {
|
|
133
|
+
fs.closeSync(descriptor);
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
type,
|
|
137
|
+
runId: run.runId,
|
|
138
|
+
path: path.relative(rootDir, filePath).split(path.sep).join('/')
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function resolveArtifactFile(rootDir, requestedPath) {
|
|
143
|
+
if (!requestedPath || path.isAbsolute(requestedPath)) {
|
|
144
|
+
throw artifactError('Artifact path must be repository-relative.', 'UNSAFE_ARTIFACT_PATH');
|
|
145
|
+
}
|
|
146
|
+
const root = fs.realpathSync(rootDir);
|
|
147
|
+
const absolute = path.resolve(root, requestedPath);
|
|
148
|
+
const relative = path.relative(root, absolute);
|
|
149
|
+
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
150
|
+
throw artifactError('Artifact path must stay inside the repository.', 'UNSAFE_ARTIFACT_PATH');
|
|
151
|
+
}
|
|
152
|
+
assertNoSymlinkPath(root, absolute);
|
|
153
|
+
const stat = fs.lstatSync(absolute);
|
|
154
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
155
|
+
throw artifactError('Artifact must be a regular file, not a symlink.', 'UNSAFE_ARTIFACT_PATH');
|
|
156
|
+
}
|
|
157
|
+
const realFile = fs.realpathSync(absolute);
|
|
158
|
+
if (!realFile.startsWith(`${root}${path.sep}`)) {
|
|
159
|
+
throw artifactError('Artifact path escapes the repository through a symlink.', 'UNSAFE_ARTIFACT_PATH');
|
|
160
|
+
}
|
|
161
|
+
return absolute;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function inferArtifactType(file, explicitType) {
|
|
165
|
+
if (explicitType && Object.hasOwn(ARTIFACT_TYPES, explicitType)) return explicitType;
|
|
166
|
+
const base = path.basename(file).replace(/\.template(?=\.json$)/, '');
|
|
167
|
+
const inferred = base.endsWith('.json') ? base.slice(0, -'.json'.length) : base;
|
|
168
|
+
if (Object.hasOwn(ARTIFACT_TYPES, inferred)) return inferred;
|
|
169
|
+
throw artifactError(
|
|
170
|
+
`Unable to infer artifact type from '${path.basename(file)}'. Pass --type ${Object.keys(ARTIFACT_TYPES).join('|')}.`,
|
|
171
|
+
'UNKNOWN_ARTIFACT_TYPE'
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function metadataErrors(document, run) {
|
|
176
|
+
const errors = [];
|
|
177
|
+
if (document.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
178
|
+
if (document.runId !== run.runId) errors.push(`runId must be ${run.runId}`);
|
|
179
|
+
return errors;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function validateDelivery(document, run) {
|
|
183
|
+
const errors = metadataErrors(document, run);
|
|
184
|
+
if (!String(document.destination || '').trim()) errors.push('destination is required');
|
|
185
|
+
if (!String(document.summary || '').trim()) errors.push('summary is required');
|
|
186
|
+
if (run.mode === 'ASSISTED' && document.humanApproved !== true) {
|
|
187
|
+
errors.push('humanApproved must be true for ASSISTED delivery');
|
|
188
|
+
}
|
|
189
|
+
return errors;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function validateArtifactDocument(type, document, run, rootDir) {
|
|
193
|
+
if (document?._templateOnly === true) {
|
|
194
|
+
return ['This is a scaffold, not evidence. Fill it with observed facts and remove _templateOnly before validation.'];
|
|
195
|
+
}
|
|
196
|
+
if (!document || typeof document !== 'object' || Array.isArray(document)) {
|
|
197
|
+
return ['Artifact must be a JSON object.'];
|
|
198
|
+
}
|
|
199
|
+
if (type === 'verification') {
|
|
200
|
+
const diffHash = run.artifacts?.diff?.sha256;
|
|
201
|
+
if (!diffHash) return ['Maker diff has not passed its gate yet.'];
|
|
202
|
+
try {
|
|
203
|
+
const result = validateVerificationBundle(document, run, diffHash, getGitRevision(rootDir));
|
|
204
|
+
return result.errors;
|
|
205
|
+
} catch (cause) {
|
|
206
|
+
return [cause.message];
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (type === 'findings') {
|
|
210
|
+
const errors = metadataErrors(document, run);
|
|
211
|
+
if (!run.artifacts?.diff?.sha256) errors.push('Maker diff has not passed its gate yet.');
|
|
212
|
+
if (document.diffHash !== run.artifacts?.diff?.sha256) errors.push('diffHash does not match the gated Maker diff.');
|
|
213
|
+
const result = validateFindingLedger(document);
|
|
214
|
+
if (!result.valid) errors.push(result.reason);
|
|
215
|
+
return errors;
|
|
216
|
+
}
|
|
217
|
+
return validateDelivery(document, run);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function validateArtifactFile(rootDir, { file, type } = {}) {
|
|
221
|
+
const filePath = resolveArtifactFile(rootDir, file);
|
|
222
|
+
const inferredType = inferArtifactType(file, type);
|
|
223
|
+
let document;
|
|
224
|
+
try {
|
|
225
|
+
document = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
226
|
+
} catch (cause) {
|
|
227
|
+
return {
|
|
228
|
+
valid: false,
|
|
229
|
+
type: inferredType,
|
|
230
|
+
path: path.relative(rootDir, filePath).split(path.sep).join('/'),
|
|
231
|
+
errors: [`Invalid JSON: ${cause.message}`]
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const run = resolveRun(rootDir, document?.runId);
|
|
235
|
+
const errors = validateArtifactDocument(inferredType, document, run, rootDir);
|
|
236
|
+
return {
|
|
237
|
+
valid: errors.length === 0,
|
|
238
|
+
type: inferredType,
|
|
239
|
+
runId: run.runId,
|
|
240
|
+
path: path.relative(rootDir, filePath).split(path.sep).join('/'),
|
|
241
|
+
errors,
|
|
242
|
+
note: 'Validation checks artifact content only; the owning gate still enforces stage readiness, freshness, and explicit approvals.'
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
module.exports = {
|
|
247
|
+
ARTIFACT_TYPES,
|
|
248
|
+
scaffoldArtifact,
|
|
249
|
+
validateArtifactFile
|
|
250
|
+
};
|