ai-engineering-loop 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +2 -2
- package/.claude/skills/ai-engineering-loop/SKILL.md +3 -3
- package/.gemini/skills/ai-engineering-loop/SKILL.md +3 -3
- package/.grok/skills/ai-engineering-loop/SKILL.md +3 -3
- package/README.md +12 -0
- package/bin/ai-engineering-loop.js +158 -9
- package/core/artifact-gates.md +32 -1
- package/core/runtime-safety.md +2 -1
- package/core/verification-loop.md +6 -0
- package/lib/agent-artifacts.js +250 -0
- package/lib/claimed-vs-reality.js +134 -0
- package/lib/safe-context.js +62 -4
- package/lib/sync-hosts.js +48 -3
- package/lib/verification-recorder.js +224 -0
- package/lib/workflow-runtime.js +26 -1
- package/package.json +1 -1
- package/policies/review-budget.md +1 -1
- package/schemas/verification.schema.json +2 -2
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { spawn } = require('child_process');
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const {
|
|
7
|
+
atomicWriteJson,
|
|
8
|
+
getCurrentRun,
|
|
9
|
+
getGitRevision,
|
|
10
|
+
loadRun,
|
|
11
|
+
runsDir
|
|
12
|
+
} = require('./run-state.js');
|
|
13
|
+
const { hashFile } = require('./gates.js');
|
|
14
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
15
|
+
const { writeVerificationSummary } = require('./verification-summary.js');
|
|
16
|
+
|
|
17
|
+
const MAX_CAPTURE_BYTES = 256 * 1024;
|
|
18
|
+
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
19
|
+
const MAX_TIMEOUT_MS = 60 * 60 * 1000;
|
|
20
|
+
const KILL_GRACE_MS = 2_000;
|
|
21
|
+
|
|
22
|
+
function recorderError(message, code = 'VERIFICATION_RECORD_FAILED') {
|
|
23
|
+
const error = new Error(message);
|
|
24
|
+
error.code = code;
|
|
25
|
+
return error;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function resolveRun(rootDir, runId) {
|
|
29
|
+
if (runId) return loadRun(rootDir, runId);
|
|
30
|
+
const current = getCurrentRun(rootDir);
|
|
31
|
+
if (!current) throw recorderError('No current Run. Pass --run <id> or start a Run first.', 'NO_CURRENT_RUN');
|
|
32
|
+
return current;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function validateRecordingRequest({ command, args, timeoutMs, shell }) {
|
|
36
|
+
if (typeof command !== 'string' || !command.trim() || command.includes('\0')) {
|
|
37
|
+
throw recorderError('Provide an executable after `--`, for example `verification record -- npm test`.', 'INVALID_COMMAND');
|
|
38
|
+
}
|
|
39
|
+
if (!Array.isArray(args) || args.some((argument) => typeof argument !== 'string' || argument.includes('\0'))) {
|
|
40
|
+
throw recorderError('Command arguments must be strings without NUL characters.', 'INVALID_COMMAND');
|
|
41
|
+
}
|
|
42
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > MAX_TIMEOUT_MS) {
|
|
43
|
+
throw recorderError(`Timeout must be between 1 and ${MAX_TIMEOUT_MS} milliseconds.`, 'INVALID_TIMEOUT');
|
|
44
|
+
}
|
|
45
|
+
if (shell && args.length > 0) {
|
|
46
|
+
throw recorderError('With --shell, pass one quoted command string after `--`; shell argument arrays are intentionally unsupported.', 'INVALID_COMMAND');
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function resolveDiff(rootDir, run) {
|
|
51
|
+
const artifact = run.artifacts?.diff;
|
|
52
|
+
if (!artifact?.path || !artifact.sha256) {
|
|
53
|
+
throw recorderError('Maker evidence is not gated yet. Run `gate maker` before recording verification.', 'MAKER_NOT_GATED');
|
|
54
|
+
}
|
|
55
|
+
const root = fs.realpathSync(rootDir);
|
|
56
|
+
const absolute = path.resolve(root, artifact.path);
|
|
57
|
+
const relative = path.relative(root, absolute);
|
|
58
|
+
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
59
|
+
throw recorderError('The gated Maker diff path escapes the repository.', 'UNSAFE_DIFF_PATH');
|
|
60
|
+
}
|
|
61
|
+
const stat = fs.lstatSync(absolute);
|
|
62
|
+
if (!stat.isFile() || stat.isSymbolicLink() || hashFile(absolute) !== artifact.sha256) {
|
|
63
|
+
throw recorderError('The gated Maker diff is missing, unsafe, or changed. Re-run the Maker gate.', 'STALE_DIFF');
|
|
64
|
+
}
|
|
65
|
+
return artifact.sha256;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function readExistingBundle(filePath, { runId, gitRevision, diffHash }) {
|
|
69
|
+
if (!fs.existsSync(filePath)) {
|
|
70
|
+
return { schemaVersion: 1, runId, gitRevision, diffHash, commands: [] };
|
|
71
|
+
}
|
|
72
|
+
let bundle;
|
|
73
|
+
try {
|
|
74
|
+
bundle = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
75
|
+
} catch (cause) {
|
|
76
|
+
throw recorderError(`Existing verification.json is invalid: ${cause.message}`, 'INVALID_VERIFICATION');
|
|
77
|
+
}
|
|
78
|
+
if (
|
|
79
|
+
bundle.schemaVersion !== 1
|
|
80
|
+
|| bundle.runId !== runId
|
|
81
|
+
|| bundle.gitRevision !== gitRevision
|
|
82
|
+
|| bundle.diffHash !== diffHash
|
|
83
|
+
|| !Array.isArray(bundle.commands)
|
|
84
|
+
) {
|
|
85
|
+
throw recorderError(
|
|
86
|
+
'Existing verification.json belongs to another Run, revision, or Maker diff. Reconcile it before recording more commands.',
|
|
87
|
+
'STALE_VERIFICATION'
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return bundle;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function captureStream(stream) {
|
|
94
|
+
const chunks = [];
|
|
95
|
+
let bytes = 0;
|
|
96
|
+
let truncated = false;
|
|
97
|
+
stream.on('data', (chunk) => {
|
|
98
|
+
const remaining = MAX_CAPTURE_BYTES - bytes;
|
|
99
|
+
if (remaining <= 0) {
|
|
100
|
+
truncated = true;
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
const captured = chunk.length > remaining ? chunk.subarray(0, remaining) : chunk;
|
|
104
|
+
chunks.push(captured);
|
|
105
|
+
bytes += captured.length;
|
|
106
|
+
if (captured.length < chunk.length) truncated = true;
|
|
107
|
+
});
|
|
108
|
+
return {
|
|
109
|
+
value: () => Buffer.concat(chunks).toString('utf8'),
|
|
110
|
+
truncated: () => truncated
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function displayCommand(command, args) {
|
|
115
|
+
const sensitiveArgument = /^--?(?:api[-_]?key|access[-_]?token|auth[-_]?token|token|password|secret|authorization)$/i;
|
|
116
|
+
let redactNext = false;
|
|
117
|
+
const safeParts = [command, ...args].map((part) => {
|
|
118
|
+
if (redactNext) {
|
|
119
|
+
redactNext = false;
|
|
120
|
+
return '[REDACTED ARGUMENT]';
|
|
121
|
+
}
|
|
122
|
+
if (sensitiveArgument.test(part)) {
|
|
123
|
+
redactNext = true;
|
|
124
|
+
return part;
|
|
125
|
+
}
|
|
126
|
+
return redactHomePaths(redactSecrets(part).text);
|
|
127
|
+
});
|
|
128
|
+
const rendered = safeParts
|
|
129
|
+
.map((part) => (/\s|["']/u.test(part) ? JSON.stringify(part) : part))
|
|
130
|
+
.join(' ');
|
|
131
|
+
return redactHomePaths(redactSecrets(rendered).text);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function runCommand(command, args, { cwd, timeoutMs, shell }) {
|
|
135
|
+
return new Promise((resolve) => {
|
|
136
|
+
const started = new Date();
|
|
137
|
+
let timedOut = false;
|
|
138
|
+
let launchError = null;
|
|
139
|
+
let forceKillTimer = null;
|
|
140
|
+
const child = spawn(command, shell ? [] : args, {
|
|
141
|
+
cwd,
|
|
142
|
+
env: process.env,
|
|
143
|
+
shell: Boolean(shell),
|
|
144
|
+
stdio: ['ignore', 'pipe', 'pipe']
|
|
145
|
+
});
|
|
146
|
+
const stdout = captureStream(child.stdout);
|
|
147
|
+
const stderr = captureStream(child.stderr);
|
|
148
|
+
|
|
149
|
+
const timeout = setTimeout(() => {
|
|
150
|
+
timedOut = true;
|
|
151
|
+
child.kill('SIGTERM');
|
|
152
|
+
forceKillTimer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS);
|
|
153
|
+
forceKillTimer.unref?.();
|
|
154
|
+
}, timeoutMs);
|
|
155
|
+
timeout.unref?.();
|
|
156
|
+
|
|
157
|
+
child.once('error', (error) => {
|
|
158
|
+
launchError = error;
|
|
159
|
+
});
|
|
160
|
+
child.once('close', (code, signal) => {
|
|
161
|
+
clearTimeout(timeout);
|
|
162
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
163
|
+
const finished = new Date();
|
|
164
|
+
const rawStdout = stdout.value();
|
|
165
|
+
const rawStderr = launchError ? launchError.message : stderr.value();
|
|
166
|
+
const stdoutValue = redactHomePaths(redactSecrets(rawStdout).text);
|
|
167
|
+
const stderrValue = redactHomePaths(redactSecrets(rawStderr).text);
|
|
168
|
+
resolve({
|
|
169
|
+
command: displayCommand(command, args),
|
|
170
|
+
executionIdentity: `pid:${child.pid || process.pid}`,
|
|
171
|
+
startTime: started.toISOString(),
|
|
172
|
+
endTime: finished.toISOString(),
|
|
173
|
+
durationMs: Math.max(0, finished.getTime() - started.getTime()),
|
|
174
|
+
exitCode: launchError ? 127 : (timedOut ? (code ?? 124) : (code ?? 1)),
|
|
175
|
+
signal: signal || null,
|
|
176
|
+
timeoutStatus: timedOut ? 'TIMED_OUT' : 'COMPLETED',
|
|
177
|
+
stdout: stdoutValue || '(no stdout)',
|
|
178
|
+
stdoutWasEmpty: stdoutValue.length === 0,
|
|
179
|
+
stderr: stderrValue,
|
|
180
|
+
stdoutTruncated: stdout.truncated(),
|
|
181
|
+
stderrTruncated: stderr.truncated(),
|
|
182
|
+
...(launchError ? { launchError: true } : {}),
|
|
183
|
+
...(timedOut ? { summary: `Command timed out after ${timeoutMs} ms.` } : {})
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async function recordVerificationCommand(rootDir, {
|
|
190
|
+
runId,
|
|
191
|
+
command,
|
|
192
|
+
args = [],
|
|
193
|
+
shell = false,
|
|
194
|
+
timeoutMs = DEFAULT_TIMEOUT_MS
|
|
195
|
+
} = {}) {
|
|
196
|
+
validateRecordingRequest({ command, args, timeoutMs, shell });
|
|
197
|
+
const run = resolveRun(rootDir, runId);
|
|
198
|
+
if (run.artifacts?.verification) {
|
|
199
|
+
throw recorderError('Verification has already passed its gate. Do not mutate gated evidence; start a new verification iteration instead.', 'VERIFICATION_ALREADY_GATED');
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const diffHash = resolveDiff(rootDir, run);
|
|
203
|
+
const gitRevision = getGitRevision(rootDir);
|
|
204
|
+
const filePath = path.join(runsDir(rootDir), run.runId, 'verification.json');
|
|
205
|
+
const bundle = readExistingBundle(filePath, { runId: run.runId, gitRevision, diffHash });
|
|
206
|
+
const result = await runCommand(command, args, { cwd: rootDir, timeoutMs, shell });
|
|
207
|
+
bundle.commands.push(result);
|
|
208
|
+
atomicWriteJson(filePath, bundle);
|
|
209
|
+
|
|
210
|
+
const summary = writeVerificationSummary(rootDir, { runId: run.runId }).summary;
|
|
211
|
+
return {
|
|
212
|
+
runId: run.runId,
|
|
213
|
+
path: path.relative(rootDir, filePath).split(path.sep).join('/'),
|
|
214
|
+
command: result,
|
|
215
|
+
summary
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
module.exports = {
|
|
220
|
+
DEFAULT_TIMEOUT_MS,
|
|
221
|
+
MAX_CAPTURE_BYTES,
|
|
222
|
+
MAX_TIMEOUT_MS,
|
|
223
|
+
recordVerificationCommand
|
|
224
|
+
};
|
package/lib/workflow-runtime.js
CHANGED
|
@@ -441,7 +441,32 @@ function assertReady(workflow, nodeId) {
|
|
|
441
441
|
if (!node) throw runtimeError(`Unknown workflow node: ${nodeId}`, 'UNKNOWN_WORKFLOW_NODE');
|
|
442
442
|
const runtimeNode = workflow.state.nodes[nodeId];
|
|
443
443
|
if (runtimeNode.status !== NODE_STATES.READY) {
|
|
444
|
-
|
|
444
|
+
const dependencies = (node.dependsOn || [])
|
|
445
|
+
.map((dependencyId) => ({
|
|
446
|
+
node: workflow.plan.nodes.find((candidate) => candidate.id === dependencyId),
|
|
447
|
+
state: workflow.state.nodes[dependencyId]
|
|
448
|
+
}))
|
|
449
|
+
.filter(({ state }) => state && !SUCCESS_STATES.has(state.status));
|
|
450
|
+
const details = dependencies.map(({ node: dependency, state }) =>
|
|
451
|
+
`Prerequisite node '${dependency?.id || 'unknown'}' is ${state.status}${dependency?.displayName ? ` (${dependency.displayName})` : ''}.`
|
|
452
|
+
);
|
|
453
|
+
const readyApproval = dependencies.length === 1
|
|
454
|
+
&& dependencies[0].node?.type === 'approval'
|
|
455
|
+
&& dependencies[0].state.status === NODE_STATES.READY
|
|
456
|
+
? dependencies[0].node
|
|
457
|
+
: null;
|
|
458
|
+
if (readyApproval) {
|
|
459
|
+
details.push(
|
|
460
|
+
`Approve it explicitly with: ai-engineering-loop node approve ${readyApproval.id} --yes --by <name> --run ${workflow.run.runId}`
|
|
461
|
+
);
|
|
462
|
+
} else if (runtimeNode.reason) {
|
|
463
|
+
details.push(`Reason: ${runtimeNode.reason}.`);
|
|
464
|
+
}
|
|
465
|
+
throw runtimeError(
|
|
466
|
+
`Node ${nodeId} is ${runtimeNode.status}, not READY`,
|
|
467
|
+
'NODE_NOT_READY',
|
|
468
|
+
details
|
|
469
|
+
);
|
|
445
470
|
}
|
|
446
471
|
return node;
|
|
447
472
|
}
|
package/package.json
CHANGED
|
@@ -15,7 +15,7 @@ Host files (`.claude/agents/`, `.grok/agents/`, `.agents/`) may differ only in Y
|
|
|
15
15
|
- Pass artifact **paths**, not Maker chat.
|
|
16
16
|
- Use the runtime policy `reviewProfile` to size context packs. Default `lean` protects Codex/GPT Plus-style sessions from repeated full-file prompts while keeping all gates intact.
|
|
17
17
|
- Prefer `npx ai-engineering-loop context diff-hunks --run <run-id>` for review. It writes a private hunk-only pack with file summaries and unresolved findings, so reviewers do not need full source bodies unless a cited hunk is insufficient.
|
|
18
|
-
-
|
|
18
|
+
- Prefer `npx ai-engineering-loop verification record --run <run-id> -- <executable> [args...]` to capture bounded, redacted command evidence without invoking a shell implicitly; it writes `verification.json` and refreshes its summary. Run `verification summarize --run <run-id>` when evidence was authored manually, and pass the compact summary to Judge before any raw logs.
|
|
19
19
|
- Spawn DA and Judge as siblings. Do not nest.
|
|
20
20
|
- Prefer named types `devil-advocate` and `judge`. Use `general-purpose` only if the named type is rejected.
|
|
21
21
|
- Skip `*.css`, `*report-css*`, generated/vendor. Do not run `git log`.
|
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
"executionIdentity": { "type": "string", "minLength": 1 },
|
|
21
21
|
"startTime": { "type": "string", "minLength": 1 },
|
|
22
22
|
"endTime": { "type": "string", "minLength": 1 },
|
|
23
|
-
"exitCode": { "
|
|
23
|
+
"exitCode": { "type": ["integer", "null"] },
|
|
24
24
|
"stdout": { "type": "string", "minLength": 1 },
|
|
25
|
-
"timeoutStatus": { "
|
|
25
|
+
"timeoutStatus": { "enum": ["COMPLETED", "TIMED_OUT", "NOT_RUN"] }
|
|
26
26
|
},
|
|
27
27
|
"additionalProperties": true
|
|
28
28
|
}
|