ai-engineering-loop 1.0.21 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +8 -4
- package/.claude/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.gemini/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.grok/skills/ai-engineering-loop/SKILL.md +15 -7
- package/README.md +43 -3
- package/README.npm.md +9 -1
- package/SECURITY.md +29 -0
- package/SUPPORT.md +28 -0
- package/agents/judge.md +5 -5
- package/agents/maker.md +6 -6
- package/bin/ai-engineering-loop.js +472 -10
- package/core/artifact-gates.md +49 -0
- package/core/configuration-precedence.md +4 -4
- package/core/context-impact-assessment.md +1 -1
- package/core/definition-of-done.md +3 -3
- package/core/escalation-policy.md +2 -2
- package/core/goal-contract.md +2 -2
- package/core/iteration-policy.md +4 -4
- package/core/project-initialization.md +2 -2
- package/core/runtime-safety.md +75 -0
- package/core/verification-loop.md +1 -1
- package/evals/README.md +35 -0
- package/evals/cases/01-stale-verification.json +17 -0
- package/evals/cases/02-vague-verification.json +16 -0
- package/evals/cases/03-missing-ac-evidence.json +17 -0
- package/evals/cases/04-hallucinated-citation.json +17 -0
- package/evals/cases/05-prompt-injection.json +17 -0
- package/evals/cases/06-source-secret.json +16 -0
- package/evals/cases/07-log-secret.json +16 -0
- package/evals/cases/08-symlink-escape.json +16 -0
- package/evals/cases/09-authorization-bypass.json +16 -0
- package/evals/cases/10-payment-idempotency.json +16 -0
- package/evals/cases/11-concurrent-update.json +16 -0
- package/evals/cases/12-swallowed-error.json +16 -0
- package/evals/cases/13-happy-path-only.json +17 -0
- package/evals/cases/14-invalid-finding-axis.json +16 -0
- package/evals/cases/15-blocker-without-alternative.json +16 -0
- package/evals/cases/16-unsupported-pass.json +17 -0
- package/evals/cases/17-soft-standard.json +16 -0
- package/evals/cases/18-iteration-ceiling.json +17 -0
- package/evals/cases/19-safe-documentation-change.json +17 -0
- package/evals/cases/20-valid-boundary-feature.json +18 -0
- package/lib/budget.js +220 -0
- package/lib/doctor.js +111 -0
- package/lib/escalation.js +192 -0
- package/lib/evaluation.js +137 -0
- package/lib/gates.js +369 -0
- package/lib/orchestration.js +5 -1
- package/lib/package-audit.js +78 -0
- package/lib/run-state.js +260 -0
- package/lib/runtime-files.js +53 -0
- package/lib/runtime-policy.js +118 -0
- package/lib/safe-context.js +194 -0
- package/lib/sandbox.js +305 -0
- package/lib/sync-hosts.js +6 -5
- package/package.json +16 -10
- package/policies/finding-policy.md +2 -2
- package/policies/model-escalation.md +41 -0
- package/policies/no-progress-policy.md +2 -2
- package/profiles/README.md +5 -5
- package/profiles/backend-api.md +1 -1
- package/profiles/library.md +1 -1
- package/profiles/mobile-app.md +1 -1
- package/profiles/monorepo.md +1 -1
- package/profiles/web-app.md +1 -1
- package/schemas/delivery.schema.json +15 -0
- package/schemas/finding-ledger.schema.json +27 -0
- package/schemas/goal-contract.schema.json +32 -0
- package/schemas/report.schema.json +17 -0
- package/schemas/verdict.schema.json +17 -0
- package/schemas/verification.schema.json +32 -0
- package/docs/antigravity-feasibility.md +0 -92
- package/docs/claude-code-feasibility.md +0 -80
- package/docs/grok-cli-feasibility.md +0 -99
- package/docs/migration-plan.md +0 -70
- package/examples/backend-api/payment-idempotency/README.md +0 -33
- package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
- package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
- package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
- package/examples/dot/status-display/README.md +0 -23
- package/examples/dot/status-display/delivery-report.md +0 -41
- package/examples/dot/status-display/goal-contract.md +0 -46
- package/examples/dot/status-display/judge-verdict.md +0 -44
- package/examples/dot/status-display/review-findings.md +0 -63
- package/examples/initialization/README.md +0 -19
- package/examples/initialization/discovery-trace.md +0 -63
- package/examples/initialization/generated-context.md +0 -110
- package/examples/mobile-app/offline-sync-queue/README.md +0 -33
- package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
- package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
- package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
- package/scripts/init.sh +0 -106
- package/templates/repo-config/adapter.md +0 -11
- package/templates/repo-config/adr-readme.md +0 -33
- package/templates/repo-config/architecture.md +0 -15
- package/templates/repo-config/claude-permissions.json +0 -10
- package/templates/repo-config/config.md +0 -11
- package/templates/repo-config/conventions.md +0 -16
- package/templates/repo-config/glossary.md +0 -18
- package/templates/repo-config/lessons.md +0 -8
- package/templates/repo-config/verification.md +0 -13
- package/templates/repo-config/workflow.md +0 -25
- package/tests/capability-selection.test.js +0 -207
- package/tests/claimed-vs-reality.test.js +0 -27
- package/tests/dot-branch-policy.test.js +0 -65
- package/tests/generate-adapter.test.js +0 -80
- package/tests/generate-workflow.test.js +0 -180
- package/tests/grok-runtime.test.js +0 -201
- package/tests/intern-catalog.test.js +0 -72
- package/tests/living-context.test.js +0 -61
- package/tests/no-company-leak.test.js +0 -82
- package/tests/orchestration.test.js +0 -368
- package/tests/qa-delivery-template.test.js +0 -90
- package/tests/skill-host-compat.test.js +0 -406
- package/tests/sync-hosts.test.js +0 -168
package/lib/sandbox.js
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const os = require('os');
|
|
6
|
+
const path = require('path');
|
|
7
|
+
const { execFileSync } = require('child_process');
|
|
8
|
+
const {
|
|
9
|
+
RUN_MODES,
|
|
10
|
+
RUN_STATES,
|
|
11
|
+
assertRunId,
|
|
12
|
+
atomicWritePrivateFile,
|
|
13
|
+
atomicWriteJson,
|
|
14
|
+
getCurrentRun,
|
|
15
|
+
loadRun,
|
|
16
|
+
runsDir
|
|
17
|
+
} = require('./run-state.js');
|
|
18
|
+
const { assertBudgetAvailable } = require('./budget.js');
|
|
19
|
+
const { assertModeAllowed } = require('./runtime-policy.js');
|
|
20
|
+
const { ensurePrivateRuntimeIgnores } = require('./runtime-files.js');
|
|
21
|
+
|
|
22
|
+
const LOCK_MAX_AGE_MS = 2 * 60 * 60 * 1000;
|
|
23
|
+
|
|
24
|
+
function sandboxError(message, code = 'SANDBOX_ERROR') {
|
|
25
|
+
const err = new Error(message);
|
|
26
|
+
err.code = code;
|
|
27
|
+
return err;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function sandboxRoot(rootDir) {
|
|
31
|
+
return path.join(path.resolve(rootDir), '.ai-engineering-loop', 'worktrees');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function sandboxPath(rootDir, runId) {
|
|
35
|
+
return path.join(sandboxRoot(rootDir), assertRunId(runId));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function lockPath(rootDir) {
|
|
39
|
+
return path.join(rootDir, '.ai-engineering-loop', 'locks', 'maker.json');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function metadataPath(rootDir, runId) {
|
|
43
|
+
return path.join(runsDir(rootDir), assertRunId(runId), 'sandbox.json');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function evidencePath(rootDir, runId) {
|
|
47
|
+
return path.join(runsDir(rootDir), assertRunId(runId), 'sandbox-evidence.json');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function pathEntryExists(filePath) {
|
|
51
|
+
try {
|
|
52
|
+
fs.lstatSync(filePath);
|
|
53
|
+
return true;
|
|
54
|
+
} catch (cause) {
|
|
55
|
+
if (cause.code === 'ENOENT') return false;
|
|
56
|
+
throw cause;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function resolveRun(rootDir, runId) {
|
|
61
|
+
if (runId) return loadRun(rootDir, assertRunId(runId));
|
|
62
|
+
const current = getCurrentRun(rootDir);
|
|
63
|
+
if (!current) throw sandboxError('No current run. Start one first.', 'NO_CURRENT_RUN');
|
|
64
|
+
return current;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function git(rootDir, args, options = {}) {
|
|
68
|
+
const { trim = true, ...execOptions } = options;
|
|
69
|
+
try {
|
|
70
|
+
const output = execFileSync('git', args, {
|
|
71
|
+
cwd: rootDir,
|
|
72
|
+
encoding: 'utf8',
|
|
73
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
74
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
75
|
+
...execOptions
|
|
76
|
+
});
|
|
77
|
+
return trim ? output.trim() : output;
|
|
78
|
+
} catch (cause) {
|
|
79
|
+
const detail = String(cause.stderr || cause.message || '').trim();
|
|
80
|
+
throw sandboxError(`Git worktree operation failed${detail ? `: ${detail}` : ''}`, 'GIT_WORKTREE_FAILED');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function assertGitRoot(rootDir) {
|
|
85
|
+
const prefix = git(rootDir, ['rev-parse', '--show-prefix']);
|
|
86
|
+
if (prefix !== '') {
|
|
87
|
+
throw sandboxError('Sandbox commands must run from the Git repository root', 'NOT_REPOSITORY_ROOT');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function processIsAlive(pid) {
|
|
92
|
+
try {
|
|
93
|
+
process.kill(pid, 0);
|
|
94
|
+
return true;
|
|
95
|
+
} catch (cause) {
|
|
96
|
+
return cause.code === 'EPERM';
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function readLock(rootDir) {
|
|
101
|
+
const filePath = lockPath(rootDir);
|
|
102
|
+
if (!fs.existsSync(filePath)) return null;
|
|
103
|
+
if (fs.lstatSync(filePath).isSymbolicLink()) {
|
|
104
|
+
throw sandboxError('Maker lock must not be a symbolic link', 'UNSAFE_RUNTIME_PATH');
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
108
|
+
} catch (cause) {
|
|
109
|
+
throw sandboxError(`Maker lock is unreadable: ${cause.message}`, 'SANDBOX_LOCKED');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function staleLocalLock(lock, now) {
|
|
114
|
+
const created = Date.parse(lock.createdAt);
|
|
115
|
+
return (
|
|
116
|
+
lock.hostname === os.hostname() &&
|
|
117
|
+
Number.isSafeInteger(lock.pid) &&
|
|
118
|
+
Number.isFinite(created) &&
|
|
119
|
+
now.getTime() - created > LOCK_MAX_AGE_MS &&
|
|
120
|
+
!processIsAlive(lock.pid)
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function acquireMakerLock(rootDir, runId, now = new Date()) {
|
|
125
|
+
ensurePrivateRuntimeIgnores(rootDir);
|
|
126
|
+
const filePath = lockPath(rootDir);
|
|
127
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
128
|
+
const lock = {
|
|
129
|
+
schemaVersion: 1,
|
|
130
|
+
runId: assertRunId(runId),
|
|
131
|
+
nonce: crypto.randomBytes(16).toString('hex'),
|
|
132
|
+
pid: process.pid,
|
|
133
|
+
hostname: os.hostname(),
|
|
134
|
+
createdAt: now.toISOString()
|
|
135
|
+
};
|
|
136
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
137
|
+
let descriptor;
|
|
138
|
+
try {
|
|
139
|
+
descriptor = fs.openSync(filePath, 'wx', 0o600);
|
|
140
|
+
fs.writeFileSync(descriptor, `${JSON.stringify(lock, null, 2)}\n`);
|
|
141
|
+
fs.fchmodSync(descriptor, 0o600);
|
|
142
|
+
return lock;
|
|
143
|
+
} catch (cause) {
|
|
144
|
+
if (cause.code !== 'EEXIST') throw cause;
|
|
145
|
+
const existing = readLock(rootDir);
|
|
146
|
+
if (attempt === 0 && staleLocalLock(existing, now)) {
|
|
147
|
+
fs.unlinkSync(filePath);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
throw sandboxError(
|
|
151
|
+
`Maker is locked by run ${existing?.runId || 'unknown'} on ${existing?.hostname || 'unknown host'}`,
|
|
152
|
+
'SANDBOX_LOCKED'
|
|
153
|
+
);
|
|
154
|
+
} finally {
|
|
155
|
+
if (descriptor != null) fs.closeSync(descriptor);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
throw sandboxError('Unable to acquire Maker lock', 'SANDBOX_LOCKED');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function releaseMakerLock(rootDir, ownedLock) {
|
|
162
|
+
const existing = readLock(rootDir);
|
|
163
|
+
if (!existing) return false;
|
|
164
|
+
if (existing.runId !== ownedLock.runId || existing.nonce !== ownedLock.nonce) {
|
|
165
|
+
throw sandboxError('Refusing to release a Maker lock owned by another process', 'LOCK_OWNERSHIP_MISMATCH');
|
|
166
|
+
}
|
|
167
|
+
fs.unlinkSync(lockPath(rootDir));
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function createSandbox(rootDir, { runId, now = new Date() } = {}) {
|
|
172
|
+
const run = resolveRun(rootDir, runId);
|
|
173
|
+
const mode = run.mode || RUN_MODES.ASSISTED;
|
|
174
|
+
assertModeAllowed(rootDir, mode);
|
|
175
|
+
assertBudgetAvailable(rootDir, { runId: run.runId });
|
|
176
|
+
if (mode === RUN_MODES.REPORT_ONLY) {
|
|
177
|
+
throw sandboxError('REPORT_ONLY mode cannot create a Maker sandbox', 'MODE_FORBIDS_MAKER');
|
|
178
|
+
}
|
|
179
|
+
if (run.state !== RUN_STATES.GOAL_FROZEN) {
|
|
180
|
+
throw sandboxError(`Maker sandbox requires GOAL_FROZEN; run is ${run.state}`, 'INVALID_SANDBOX_STATE');
|
|
181
|
+
}
|
|
182
|
+
assertGitRoot(rootDir);
|
|
183
|
+
ensurePrivateRuntimeIgnores(rootDir);
|
|
184
|
+
const worktree = sandboxPath(rootDir, run.runId);
|
|
185
|
+
if (pathEntryExists(worktree)) {
|
|
186
|
+
throw sandboxError(`Sandbox already exists for run ${run.runId}`, 'SANDBOX_EXISTS');
|
|
187
|
+
}
|
|
188
|
+
const ownedLock = acquireMakerLock(rootDir, run.runId, now);
|
|
189
|
+
try {
|
|
190
|
+
const baseRevision = run.startedRevision;
|
|
191
|
+
if (!/^[a-f0-9]{40,64}$/.test(String(baseRevision || ''))) {
|
|
192
|
+
throw sandboxError('Run did not start from a reproducible Git revision', 'INVALID_BASE_REVISION');
|
|
193
|
+
}
|
|
194
|
+
fs.mkdirSync(path.dirname(worktree), { recursive: true, mode: 0o700 });
|
|
195
|
+
git(rootDir, ['worktree', 'add', '--detach', worktree, baseRevision]);
|
|
196
|
+
const metadata = {
|
|
197
|
+
schemaVersion: 1,
|
|
198
|
+
runId: run.runId,
|
|
199
|
+
baseRevision,
|
|
200
|
+
worktreePath: worktree,
|
|
201
|
+
lockNonce: ownedLock.nonce,
|
|
202
|
+
createdAt: now.toISOString()
|
|
203
|
+
};
|
|
204
|
+
atomicWriteJson(metadataPath(rootDir, run.runId), metadata);
|
|
205
|
+
return metadata;
|
|
206
|
+
} catch (cause) {
|
|
207
|
+
try {
|
|
208
|
+
if (pathEntryExists(worktree)) git(rootDir, ['worktree', 'remove', '--force', worktree]);
|
|
209
|
+
} catch {
|
|
210
|
+
// Preserve the original failure; `git worktree prune` can recover a partial registration.
|
|
211
|
+
}
|
|
212
|
+
releaseMakerLock(rootDir, ownedLock);
|
|
213
|
+
throw cause;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function loadOwnedSandbox(rootDir, runId) {
|
|
218
|
+
const run = resolveRun(rootDir, runId);
|
|
219
|
+
const expectedPath = sandboxPath(rootDir, run.runId);
|
|
220
|
+
let metadata;
|
|
221
|
+
try {
|
|
222
|
+
metadata = JSON.parse(fs.readFileSync(metadataPath(rootDir, run.runId), 'utf8'));
|
|
223
|
+
} catch (cause) {
|
|
224
|
+
throw sandboxError(`Sandbox metadata is unavailable: ${cause.message}`, 'SANDBOX_MISSING');
|
|
225
|
+
}
|
|
226
|
+
if (
|
|
227
|
+
metadata.runId !== run.runId ||
|
|
228
|
+
metadata.worktreePath !== expectedPath ||
|
|
229
|
+
!/^[a-f0-9]{40,64}$/.test(String(metadata.baseRevision || ''))
|
|
230
|
+
) {
|
|
231
|
+
throw sandboxError('Sandbox metadata is invalid', 'INVALID_SANDBOX_METADATA');
|
|
232
|
+
}
|
|
233
|
+
const lock = readLock(rootDir);
|
|
234
|
+
if (!lock || lock.runId !== run.runId || lock.nonce !== metadata.lockNonce) {
|
|
235
|
+
throw sandboxError('Maker lock does not match sandbox metadata', 'LOCK_OWNERSHIP_MISMATCH');
|
|
236
|
+
}
|
|
237
|
+
if (!fs.existsSync(expectedPath)) throw sandboxError('Sandbox worktree is missing', 'SANDBOX_MISSING');
|
|
238
|
+
return { run, metadata, lock, worktree: expectedPath };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function captureSandbox(rootDir, { runId, now = new Date() } = {}) {
|
|
242
|
+
const owned = loadOwnedSandbox(rootDir, runId);
|
|
243
|
+
try {
|
|
244
|
+
fs.unlinkSync(evidencePath(rootDir, owned.run.runId));
|
|
245
|
+
} catch (cause) {
|
|
246
|
+
if (cause.code !== 'ENOENT') throw cause;
|
|
247
|
+
}
|
|
248
|
+
git(owned.worktree, ['add', '--intent-to-add', '--all']);
|
|
249
|
+
const diff = git(
|
|
250
|
+
owned.worktree,
|
|
251
|
+
['diff', '--binary', '--no-ext-diff', 'HEAD', '--'],
|
|
252
|
+
{ trim: false }
|
|
253
|
+
);
|
|
254
|
+
if (!diff.trim()) throw sandboxError('Maker sandbox has no changes to capture', 'EMPTY_SANDBOX_DIFF');
|
|
255
|
+
const diffFile = path.join(runsDir(rootDir), owned.run.runId, 'diff.patch');
|
|
256
|
+
atomicWritePrivateFile(diffFile, diff);
|
|
257
|
+
const evidence = {
|
|
258
|
+
schemaVersion: 1,
|
|
259
|
+
runId: owned.run.runId,
|
|
260
|
+
sandboxed: true,
|
|
261
|
+
baseRevision: owned.metadata.baseRevision,
|
|
262
|
+
diffSha256: crypto.createHash('sha256').update(diff).digest('hex'),
|
|
263
|
+
capturedAt: now.toISOString()
|
|
264
|
+
};
|
|
265
|
+
git(rootDir, ['worktree', 'remove', '--force', owned.worktree]);
|
|
266
|
+
releaseMakerLock(rootDir, owned.lock);
|
|
267
|
+
atomicWriteJson(evidencePath(rootDir, owned.run.runId), evidence);
|
|
268
|
+
return { diffPath: diffFile, evidence };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function abortSandbox(rootDir, { runId } = {}) {
|
|
272
|
+
const owned = loadOwnedSandbox(rootDir, runId);
|
|
273
|
+
git(rootDir, ['worktree', 'remove', '--force', owned.worktree]);
|
|
274
|
+
releaseMakerLock(rootDir, owned.lock);
|
|
275
|
+
return { runId: owned.run.runId, aborted: true };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function sandboxStatus(rootDir, { runId } = {}) {
|
|
279
|
+
const run = resolveRun(rootDir, runId);
|
|
280
|
+
const filePath = metadataPath(rootDir, run.runId);
|
|
281
|
+
if (!fs.existsSync(filePath)) return { runId: run.runId, exists: false, locked: false };
|
|
282
|
+
const metadata = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
283
|
+
const lock = readLock(rootDir);
|
|
284
|
+
return {
|
|
285
|
+
runId: run.runId,
|
|
286
|
+
exists: fs.existsSync(sandboxPath(rootDir, run.runId)),
|
|
287
|
+
locked: lock?.runId === run.runId && lock?.nonce === metadata.lockNonce,
|
|
288
|
+
baseRevision: metadata.baseRevision
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
module.exports = {
|
|
293
|
+
LOCK_MAX_AGE_MS,
|
|
294
|
+
sandboxRoot,
|
|
295
|
+
sandboxPath,
|
|
296
|
+
lockPath,
|
|
297
|
+
metadataPath,
|
|
298
|
+
evidencePath,
|
|
299
|
+
acquireMakerLock,
|
|
300
|
+
releaseMakerLock,
|
|
301
|
+
createSandbox,
|
|
302
|
+
captureSandbox,
|
|
303
|
+
abortSandbox,
|
|
304
|
+
sandboxStatus
|
|
305
|
+
};
|
package/lib/sync-hosts.js
CHANGED
|
@@ -14,12 +14,13 @@ function homeDir(env = process.env) {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
function applyGrokSkillOverlay(content) {
|
|
17
|
-
const
|
|
18
|
-
|
|
17
|
+
const normalized = String(content).replace(/\r\n?/g, '\n');
|
|
18
|
+
const match = normalized.match(/^---\n([\s\S]*?)\n---\n/);
|
|
19
|
+
if (!match) return normalized;
|
|
19
20
|
const yaml = match[1];
|
|
20
|
-
if (/^user-invocable:\s*true\s*$/m.test(yaml)) return
|
|
21
|
+
if (/^user-invocable:\s*true\s*$/m.test(yaml)) return normalized;
|
|
21
22
|
const nextYaml = `${yaml.replace(/\s*$/, '')}\nuser-invocable: true`;
|
|
22
|
-
return `---\n${nextYaml}\n---\n${
|
|
23
|
+
return `---\n${nextYaml}\n---\n${normalized.slice(match[0].length)}`;
|
|
23
24
|
}
|
|
24
25
|
|
|
25
26
|
function hostFileMap(home) {
|
|
@@ -133,7 +134,7 @@ function planHostSync({ packageRoot: root = packageRoot(), home = homeDir() } =
|
|
|
133
134
|
results.push({ ...item, action: 'skip', reason: 'not-installed' });
|
|
134
135
|
continue;
|
|
135
136
|
}
|
|
136
|
-
let content = fs.readFileSync(srcAbs, 'utf8');
|
|
137
|
+
let content = fs.readFileSync(srcAbs, 'utf8').replace(/\r\n?/g, '\n');
|
|
137
138
|
if (file.grokSkillOverlay) content = applyGrokSkillOverlay(content);
|
|
138
139
|
if (destExists(file.dest) && fs.readFileSync(file.dest, 'utf8') === content) {
|
|
139
140
|
results.push({ ...item, action: 'current' });
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-engineering-loop",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "A reusable, framework-agnostic AI Engineering Operating System for
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"description": "A reusable, framework-agnostic AI Engineering Operating System for safe coding agents.",
|
|
5
5
|
"main": "bin/ai-engineering-loop.js",
|
|
6
6
|
"bin": {
|
|
7
7
|
"ai-engineering-loop": "bin/ai-engineering-loop.js"
|
|
@@ -14,11 +14,8 @@
|
|
|
14
14
|
"policies/",
|
|
15
15
|
"profiles/",
|
|
16
16
|
"adapters/",
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"docs/",
|
|
20
|
-
"tests/",
|
|
21
|
-
"scripts/",
|
|
17
|
+
"schemas/",
|
|
18
|
+
"evals/",
|
|
22
19
|
".agents/",
|
|
23
20
|
".grok/",
|
|
24
21
|
".gemini/skills/",
|
|
@@ -26,12 +23,21 @@
|
|
|
26
23
|
".claude/commands/",
|
|
27
24
|
".claude/skills/",
|
|
28
25
|
"LICENSE",
|
|
29
|
-
"
|
|
30
|
-
"
|
|
26
|
+
"SECURITY.md",
|
|
27
|
+
"SUPPORT.md",
|
|
28
|
+
"README.md"
|
|
31
29
|
],
|
|
32
30
|
"scripts": {
|
|
33
31
|
"init": "node bin/ai-engineering-loop.js",
|
|
34
|
-
"test": "node --test
|
|
32
|
+
"test": "node --test",
|
|
33
|
+
"doctor": "node bin/ai-engineering-loop.js doctor",
|
|
34
|
+
"eval": "node bin/ai-engineering-loop.js eval",
|
|
35
|
+
"audit:package": "node scripts/audit-package.js",
|
|
36
|
+
"verify:release": "node scripts/verify-release.js",
|
|
37
|
+
"prepublishOnly": "npm test && npm run doctor && npm run eval && npm run audit:package"
|
|
38
|
+
},
|
|
39
|
+
"engines": {
|
|
40
|
+
"node": ">=18"
|
|
35
41
|
},
|
|
36
42
|
"repository": {
|
|
37
43
|
"type": "git",
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
## 1. Purpose & Core Philosophy
|
|
4
4
|
|
|
5
|
-
The **Finding Policy** defines the standardized schema, severity definitions, validity rules, and lifecycle states for all issues identified by review agents (such as the [Devil's Advocate](
|
|
5
|
+
The **Finding Policy** defines the standardized schema, severity definitions, validity rules, and lifecycle states for all issues identified by review agents (such as the [Devil's Advocate](../agents/devil-advocate.md) or external review bots).
|
|
6
6
|
|
|
7
7
|
A standardized finding schema ensures that:
|
|
8
8
|
1. Every criticism is actionable, localized, and backed by factual code evidence.
|
|
9
9
|
2. Review findings can be parsed, hashed, and tracked across autonomous iterations.
|
|
10
|
-
3. The [Judge Agent](
|
|
10
|
+
3. The [Judge Agent](../agents/judge.md) can evaluate findings deterministically based on **Validity + Severity** without subjective bias.
|
|
11
11
|
|
|
12
12
|
---
|
|
13
13
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Model Escalation Policy
|
|
2
|
+
|
|
3
|
+
Cheap models are the default Maker, Devil's Advocate, and Judge. Required workflow stages never disappear when the model tier changes.
|
|
4
|
+
|
|
5
|
+
The objective is not to prevent every model error. Unsupported output must fail a deterministic gate, and repeated uncertainty must move to a stronger reviewer.
|
|
6
|
+
|
|
7
|
+
## Deterministic routing
|
|
8
|
+
|
|
9
|
+
Use `CHEAP_MODEL` when all evidence is current, context is complete, and no high-risk path changed.
|
|
10
|
+
|
|
11
|
+
Require `STRONG_MODEL` when any of these signals exist:
|
|
12
|
+
|
|
13
|
+
- Authentication, authorization, payment, billing, database migration, infrastructure, cryptography, or secret-handling paths changed.
|
|
14
|
+
- The model emitted two invalid structured artifacts.
|
|
15
|
+
- A finding cites a path that does not exist.
|
|
16
|
+
- Verification is flaky.
|
|
17
|
+
- A stage context pack was truncated.
|
|
18
|
+
- A critical acceptance criterion remains `UNKNOWN`.
|
|
19
|
+
- Devil's Advocate and Judge disagree about a blocking finding.
|
|
20
|
+
|
|
21
|
+
Require `HUMAN` when:
|
|
22
|
+
|
|
23
|
+
- The iteration ceiling is reached.
|
|
24
|
+
- A credential or evaluation canary reached model-visible output.
|
|
25
|
+
|
|
26
|
+
Diff size alone is not a risk classification. A one-line authorization change can require stronger review.
|
|
27
|
+
|
|
28
|
+
## Verdict metadata
|
|
29
|
+
|
|
30
|
+
Judge writes `modelTier` in `verdict.json`:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{
|
|
34
|
+
"modelTier": "CHEAP_MODEL",
|
|
35
|
+
"humanApproved": false
|
|
36
|
+
}
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`ai-engineering-loop gate judge` computes the required tier. A lower tier may safely issue `ESCALATE`, but it cannot issue `PASS` when stronger review is required.
|
|
40
|
+
|
|
41
|
+
Run `ai-engineering-loop escalation --json` before spawning Judge to select the minimum acceptable tier. Do not invent a provider or model slug; resolve the tier through the active host's model catalog.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
A major risk in autonomous AI agent loops is **thrashing (stagnation)**: the agent modifies code repeatedly across iterations without actually resolving the underlying defect, oscillating between two broken states, or generating the same unresolved findings repeatedly.
|
|
6
6
|
|
|
7
|
-
The **No-Progress Policy** defines an algorithmic mechanism to detect stagnation early and halt the loop with a [Human Escalation](
|
|
7
|
+
The **No-Progress Policy** defines an algorithmic mechanism to detect stagnation early and halt the loop with a [Human Escalation](../core/escalation-policy.md) rather than wasting tokens and computational cycles.
|
|
8
8
|
|
|
9
9
|
```mermaid
|
|
10
10
|
flowchart LR
|
|
@@ -89,4 +89,4 @@ def check_stagnation(
|
|
|
89
89
|
When `check_stagnation` returns `True`:
|
|
90
90
|
1. The Judge Agent immediately sets the verdict to **`ESCALATE`**.
|
|
91
91
|
2. Autonomous iteration stops.
|
|
92
|
-
3. The agent generates the standardized [Human Escalation Report](
|
|
92
|
+
3. The agent generates the standardized [Human Escalation Report](../core/escalation-policy.md) highlighting the recurring signature and exact blocking contradiction.
|
package/profiles/README.md
CHANGED
|
@@ -23,11 +23,11 @@ flowchart TD
|
|
|
23
23
|
|
|
24
24
|
| Profile | Target Archetype | Primary Verification Focus | Active Review Domains |
|
|
25
25
|
|---|---|---|---|
|
|
26
|
-
| **[`web-app`](
|
|
27
|
-
| **[`backend-api`](
|
|
28
|
-
| **[`mobile-app`](
|
|
29
|
-
| **[`library`](
|
|
30
|
-
| **[`monorepo`](
|
|
26
|
+
| **[`web-app`](./web-app.md)** | Single-Page Apps, SSR/SSG websites, Admin Portals | DOM rendering, browser responsiveness, bundle size, CSS/tokens | Responsive layout, accessibility (a11y), client state, Core Web Vitals |
|
|
27
|
+
| **[`backend-api`](./backend-api.md)** | REST / GraphQL / gRPC Microservices, Background Workers | Unit tests, database migrations, schema contracts | Auth/Authz, SQL injection, concurrency, transactions, API contracts |
|
|
28
|
+
| **[`mobile-app`](./mobile-app.md)** | iOS, Android, React Native, Flutter | Platform builds, simulator smoke tests, widget tests | Offline sync, device permissions, lifecycle transitions, network resilience |
|
|
29
|
+
| **[`library`](./library.md)** | Reusable SDKs, packages, shared utilities | Broad runtime compatibility, zero external dependencies | Public API stability, semver breaking changes, bundle tree-shaking |
|
|
30
|
+
| **[`monorepo`](./monorepo.md)** | Turborepo, Nx, Lerna, Yarn/pnpm Workspaces | Workspace scope tests, affected package compilation | Cross-package boundaries, circular dependencies, dependency isolation |
|
|
31
31
|
|
|
32
32
|
---
|
|
33
33
|
|
package/profiles/backend-api.md
CHANGED
|
@@ -30,7 +30,7 @@ When working in a `backend-api` repository, deterministic verification should pr
|
|
|
30
30
|
|
|
31
31
|
## 4. Active Review Domains for Devil's Advocate
|
|
32
32
|
|
|
33
|
-
When reviewing diffs in a `backend-api` project, the [Devil's Advocate](
|
|
33
|
+
When reviewing diffs in a `backend-api` project, the [Devil's Advocate](../agents/devil-advocate.md) activates these targeted checks:
|
|
34
34
|
|
|
35
35
|
```mermaid
|
|
36
36
|
flowchart LR
|
package/profiles/library.md
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
|
|
28
28
|
## 4. Active Review Domains for Devil's Advocate
|
|
29
29
|
|
|
30
|
-
When reviewing diffs in a `library` project, the [Devil's Advocate](
|
|
30
|
+
When reviewing diffs in a `library` project, the [Devil's Advocate](../agents/devil-advocate.md) activates these targeted checks:
|
|
31
31
|
|
|
32
32
|
```mermaid
|
|
33
33
|
flowchart LR
|
package/profiles/mobile-app.md
CHANGED
|
@@ -30,7 +30,7 @@ When working in a `mobile-app` repository, deterministic verification should pri
|
|
|
30
30
|
|
|
31
31
|
## 4. Active Review Domains for Devil's Advocate
|
|
32
32
|
|
|
33
|
-
When reviewing diffs in a `mobile-app` project, the [Devil's Advocate](
|
|
33
|
+
When reviewing diffs in a `mobile-app` project, the [Devil's Advocate](../agents/devil-advocate.md) activates these targeted checks:
|
|
34
34
|
|
|
35
35
|
```mermaid
|
|
36
36
|
flowchart LR
|
package/profiles/monorepo.md
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
|
|
27
27
|
## 4. Active Review Domains for Devil's Advocate
|
|
28
28
|
|
|
29
|
-
When reviewing diffs in a `monorepo` project, the [Devil's Advocate](
|
|
29
|
+
When reviewing diffs in a `monorepo` project, the [Devil's Advocate](../agents/devil-advocate.md) activates these targeted checks:
|
|
30
30
|
|
|
31
31
|
```mermaid
|
|
32
32
|
flowchart LR
|
package/profiles/web-app.md
CHANGED
|
@@ -30,7 +30,7 @@ When working in a `web-app` repository, deterministic verification should priori
|
|
|
30
30
|
|
|
31
31
|
## 4. Active Review Domains for Devil's Advocate
|
|
32
32
|
|
|
33
|
-
When reviewing diffs in a `web-app` project, the [Devil's Advocate](
|
|
33
|
+
When reviewing diffs in a `web-app` project, the [Devil's Advocate](../agents/devil-advocate.md) activates these targeted checks:
|
|
34
34
|
|
|
35
35
|
```mermaid
|
|
36
36
|
flowchart LR
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/delivery.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Delivery Record",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "destination", "summary"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"destination": { "type": "string", "minLength": 1 },
|
|
11
|
+
"summary": { "type": "string", "minLength": 1 },
|
|
12
|
+
"humanApproved": { "type": "boolean" }
|
|
13
|
+
},
|
|
14
|
+
"additionalProperties": true
|
|
15
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/finding-ledger.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Finding Ledger",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "diffHash", "findings"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"diffHash": { "type": "string", "minLength": 1 },
|
|
11
|
+
"findings": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"items": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"required": ["id", "location", "failureScenario", "evidence", "severity", "validity", "disposition"],
|
|
16
|
+
"properties": {
|
|
17
|
+
"axis": { "enum": ["spec", "standards"] },
|
|
18
|
+
"severity": { "enum": ["BLOCKER", "HIGH", "MEDIUM", "LOW"] },
|
|
19
|
+
"validity": { "enum": ["VALID", "INVALID"] },
|
|
20
|
+
"disposition": { "enum": ["STRONG", "ACCEPTABLE", "WEAK"] }
|
|
21
|
+
},
|
|
22
|
+
"additionalProperties": true
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"additionalProperties": true
|
|
27
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/goal-contract.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Goal Contract",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "objective", "acceptanceCriteria"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"objective": { "type": "string", "minLength": 1 },
|
|
11
|
+
"acceptanceCriteria": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"minItems": 1,
|
|
14
|
+
"items": {
|
|
15
|
+
"type": "object",
|
|
16
|
+
"required": ["id", "statement", "evidenceRequired", "failureCases"],
|
|
17
|
+
"properties": {
|
|
18
|
+
"id": { "type": "string", "pattern": "^AC-[0-9]+$" },
|
|
19
|
+
"statement": { "type": "string", "minLength": 1 },
|
|
20
|
+
"evidenceRequired": { "type": "string", "minLength": 1 },
|
|
21
|
+
"failureCases": {
|
|
22
|
+
"type": "array",
|
|
23
|
+
"minItems": 1,
|
|
24
|
+
"items": { "type": "string", "minLength": 1 }
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"additionalProperties": true
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"additionalProperties": true
|
|
32
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/report.schema.json",
|
|
4
|
+
"title": "Report-only completion",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "summary", "recommendations"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"summary": { "type": "string", "minLength": 1 },
|
|
11
|
+
"recommendations": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"items": { "type": "string", "minLength": 1 }
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"additionalProperties": true
|
|
17
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/verdict.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Judge Verdict",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "verdict", "reason", "action"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"verdict": { "enum": ["PASS", "ITERATE", "ESCALATE"] },
|
|
11
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
12
|
+
"action": { "type": "string", "minLength": 1 },
|
|
13
|
+
"modelTier": { "enum": ["CHEAP_MODEL", "STRONG_MODEL", "HUMAN"] },
|
|
14
|
+
"humanApproved": { "type": "boolean" }
|
|
15
|
+
},
|
|
16
|
+
"additionalProperties": true
|
|
17
|
+
}
|