ai-engineering-loop 1.0.21 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +7 -3
- package/.claude/skills/ai-engineering-loop/SKILL.md +13 -7
- package/.gemini/skills/ai-engineering-loop/SKILL.md +13 -7
- package/.grok/skills/ai-engineering-loop/SKILL.md +13 -7
- package/README.md +26 -3
- package/SECURITY.md +23 -0
- package/SUPPORT.md +28 -0
- package/bin/ai-engineering-loop.js +255 -3
- package/core/artifact-gates.md +49 -0
- package/evals/README.md +35 -0
- package/evals/cases/01-stale-verification.json +17 -0
- package/evals/cases/02-vague-verification.json +16 -0
- package/evals/cases/03-missing-ac-evidence.json +17 -0
- package/evals/cases/04-hallucinated-citation.json +17 -0
- package/evals/cases/05-prompt-injection.json +17 -0
- package/evals/cases/06-source-secret.json +16 -0
- package/evals/cases/07-log-secret.json +16 -0
- package/evals/cases/08-symlink-escape.json +16 -0
- package/evals/cases/09-authorization-bypass.json +16 -0
- package/evals/cases/10-payment-idempotency.json +16 -0
- package/evals/cases/11-concurrent-update.json +16 -0
- package/evals/cases/12-swallowed-error.json +16 -0
- package/evals/cases/13-happy-path-only.json +17 -0
- package/evals/cases/14-invalid-finding-axis.json +16 -0
- package/evals/cases/15-blocker-without-alternative.json +16 -0
- package/evals/cases/16-unsupported-pass.json +17 -0
- package/evals/cases/17-soft-standard.json +16 -0
- package/evals/cases/18-iteration-ceiling.json +17 -0
- package/evals/cases/19-safe-documentation-change.json +17 -0
- package/evals/cases/20-valid-boundary-feature.json +18 -0
- package/lib/doctor.js +98 -0
- package/lib/escalation.js +192 -0
- package/lib/evaluation.js +137 -0
- package/lib/gates.js +315 -0
- package/lib/orchestration.js +5 -1
- package/lib/run-state.js +221 -0
- package/lib/safe-context.js +191 -0
- package/lib/sync-hosts.js +6 -5
- package/package.json +12 -9
- package/policies/model-escalation.md +41 -0
- package/schemas/delivery.schema.json +14 -0
- package/schemas/finding-ledger.schema.json +27 -0
- package/schemas/goal-contract.schema.json +32 -0
- package/schemas/verdict.schema.json +17 -0
- package/schemas/verification.schema.json +32 -0
- package/docs/antigravity-feasibility.md +0 -92
- package/docs/claude-code-feasibility.md +0 -80
- package/docs/grok-cli-feasibility.md +0 -99
- package/docs/migration-plan.md +0 -70
- package/examples/backend-api/payment-idempotency/README.md +0 -33
- package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
- package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
- package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
- package/examples/dot/status-display/README.md +0 -23
- package/examples/dot/status-display/delivery-report.md +0 -41
- package/examples/dot/status-display/goal-contract.md +0 -46
- package/examples/dot/status-display/judge-verdict.md +0 -44
- package/examples/dot/status-display/review-findings.md +0 -63
- package/examples/initialization/README.md +0 -19
- package/examples/initialization/discovery-trace.md +0 -63
- package/examples/initialization/generated-context.md +0 -110
- package/examples/mobile-app/offline-sync-queue/README.md +0 -33
- package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
- package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
- package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
- package/scripts/init.sh +0 -106
- package/templates/repo-config/adapter.md +0 -11
- package/templates/repo-config/adr-readme.md +0 -33
- package/templates/repo-config/architecture.md +0 -15
- package/templates/repo-config/claude-permissions.json +0 -10
- package/templates/repo-config/config.md +0 -11
- package/templates/repo-config/conventions.md +0 -16
- package/templates/repo-config/glossary.md +0 -18
- package/templates/repo-config/lessons.md +0 -8
- package/templates/repo-config/verification.md +0 -13
- package/templates/repo-config/workflow.md +0 -25
- package/tests/capability-selection.test.js +0 -207
- package/tests/claimed-vs-reality.test.js +0 -27
- package/tests/dot-branch-policy.test.js +0 -65
- package/tests/generate-adapter.test.js +0 -80
- package/tests/generate-workflow.test.js +0 -180
- package/tests/grok-runtime.test.js +0 -201
- package/tests/intern-catalog.test.js +0 -72
- package/tests/living-context.test.js +0 -61
- package/tests/no-company-leak.test.js +0 -82
- package/tests/orchestration.test.js +0 -368
- package/tests/qa-delivery-template.test.js +0 -90
- package/tests/skill-host-compat.test.js +0 -406
- package/tests/sync-hosts.test.js +0 -168
package/lib/run-state.js
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { execFileSync } = require('child_process');
|
|
7
|
+
|
|
8
|
+
const RUN_STATES = Object.freeze({
|
|
9
|
+
STARTED: 'STARTED',
|
|
10
|
+
GOAL_FROZEN: 'GOAL_FROZEN',
|
|
11
|
+
MAKER_COMPLETE: 'MAKER_COMPLETE',
|
|
12
|
+
VERIFIED: 'VERIFIED',
|
|
13
|
+
REVIEWED: 'REVIEWED',
|
|
14
|
+
JUDGED_PASS: 'JUDGED_PASS',
|
|
15
|
+
DELIVERED: 'DELIVERED',
|
|
16
|
+
ESCALATED: 'ESCALATED'
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const ALLOWED_TRANSITIONS = Object.freeze({
|
|
20
|
+
STARTED: ['GOAL_FROZEN'],
|
|
21
|
+
GOAL_FROZEN: ['MAKER_COMPLETE'],
|
|
22
|
+
MAKER_COMPLETE: ['VERIFIED'],
|
|
23
|
+
VERIFIED: ['REVIEWED'],
|
|
24
|
+
REVIEWED: ['GOAL_FROZEN', 'JUDGED_PASS', 'ESCALATED'],
|
|
25
|
+
JUDGED_PASS: ['DELIVERED'],
|
|
26
|
+
DELIVERED: [],
|
|
27
|
+
ESCALATED: []
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const TERMINAL_STATES = new Set([RUN_STATES.DELIVERED, RUN_STATES.ESCALATED]);
|
|
31
|
+
|
|
32
|
+
function runsDir(rootDir) {
|
|
33
|
+
return path.join(rootDir, '.ai-engineering-loop', 'runs');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function assertRunId(runId) {
|
|
37
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(String(runId || ''))) {
|
|
38
|
+
const err = new Error(`Invalid run id: ${runId || '(empty)'}`);
|
|
39
|
+
err.code = 'INVALID_RUN_ID';
|
|
40
|
+
throw err;
|
|
41
|
+
}
|
|
42
|
+
return runId;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function atomicWriteJson(filePath, value) {
|
|
46
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
47
|
+
const temporary = `${filePath}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`;
|
|
48
|
+
try {
|
|
49
|
+
fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
50
|
+
fs.renameSync(temporary, filePath);
|
|
51
|
+
fs.chmodSync(filePath, 0o600);
|
|
52
|
+
} finally {
|
|
53
|
+
try {
|
|
54
|
+
fs.unlinkSync(temporary);
|
|
55
|
+
} catch {
|
|
56
|
+
// Rename succeeded or no temporary file was written.
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function readJson(filePath, label) {
|
|
62
|
+
try {
|
|
63
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
64
|
+
} catch (cause) {
|
|
65
|
+
const err = new Error(`Unable to read ${label}: ${cause.message}`);
|
|
66
|
+
err.code = 'INVALID_RUN_STATE';
|
|
67
|
+
err.cause = cause;
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function getGitRevision(rootDir) {
|
|
73
|
+
try {
|
|
74
|
+
return execFileSync('git', ['rev-parse', 'HEAD'], {
|
|
75
|
+
cwd: rootDir,
|
|
76
|
+
encoding: 'utf8',
|
|
77
|
+
stdio: ['ignore', 'pipe', 'ignore']
|
|
78
|
+
}).trim();
|
|
79
|
+
} catch {
|
|
80
|
+
return 'untracked';
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function newRunId(now = new Date()) {
|
|
85
|
+
const timestamp = now.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z');
|
|
86
|
+
return `${timestamp}-${crypto.randomUUID().slice(0, 8)}`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function runStatePath(rootDir, runId) {
|
|
90
|
+
return path.join(runsDir(rootDir), assertRunId(runId), 'state.json');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function currentRunPath(rootDir) {
|
|
94
|
+
return path.join(runsDir(rootDir), 'current.json');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function loadRun(rootDir, runId) {
|
|
98
|
+
const state = readJson(runStatePath(rootDir, runId), `run ${runId}`);
|
|
99
|
+
if (state.runId !== runId || !RUN_STATES[state.state]) {
|
|
100
|
+
const err = new Error(`Run ${runId} has an invalid state document`);
|
|
101
|
+
err.code = 'INVALID_RUN_STATE';
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
return state;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function getCurrentRun(rootDir) {
|
|
108
|
+
const pointer = currentRunPath(rootDir);
|
|
109
|
+
if (!fs.existsSync(pointer)) return null;
|
|
110
|
+
const { runId } = readJson(pointer, 'current run pointer');
|
|
111
|
+
return loadRun(rootDir, assertRunId(runId));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function createRun(rootDir, { task = '', runId = newRunId(), now = new Date() } = {}) {
|
|
115
|
+
assertRunId(runId);
|
|
116
|
+
const statePath = runStatePath(rootDir, runId);
|
|
117
|
+
if (fs.existsSync(statePath)) {
|
|
118
|
+
const err = new Error(`Run already exists: ${runId}`);
|
|
119
|
+
err.code = 'RUN_EXISTS';
|
|
120
|
+
throw err;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const timestamp = now.toISOString();
|
|
124
|
+
const state = {
|
|
125
|
+
schemaVersion: 1,
|
|
126
|
+
runId,
|
|
127
|
+
state: RUN_STATES.STARTED,
|
|
128
|
+
iteration: 1,
|
|
129
|
+
task: String(task || '').trim(),
|
|
130
|
+
startedRevision: getGitRevision(rootDir),
|
|
131
|
+
createdAt: timestamp,
|
|
132
|
+
updatedAt: timestamp,
|
|
133
|
+
artifacts: {},
|
|
134
|
+
history: [
|
|
135
|
+
{
|
|
136
|
+
from: null,
|
|
137
|
+
to: RUN_STATES.STARTED,
|
|
138
|
+
gate: 'start',
|
|
139
|
+
at: timestamp
|
|
140
|
+
}
|
|
141
|
+
]
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
atomicWriteJson(statePath, state);
|
|
145
|
+
atomicWriteJson(currentRunPath(rootDir), { schemaVersion: 1, runId });
|
|
146
|
+
return state;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function createOrResumeRun(rootDir, { task = '', forceNew = false, now = new Date() } = {}) {
|
|
150
|
+
const current = getCurrentRun(rootDir);
|
|
151
|
+
const requestedTask = String(task || '').trim();
|
|
152
|
+
if (current && !TERMINAL_STATES.has(current.state) && !forceNew) {
|
|
153
|
+
if (requestedTask && current.task && requestedTask !== current.task) {
|
|
154
|
+
const err = new Error(
|
|
155
|
+
`Run ${current.runId} is still ${current.state}. Resume it or pass --new to start another task.`
|
|
156
|
+
);
|
|
157
|
+
err.code = 'ACTIVE_RUN';
|
|
158
|
+
throw err;
|
|
159
|
+
}
|
|
160
|
+
return { run: current, resumed: true };
|
|
161
|
+
}
|
|
162
|
+
return { run: createRun(rootDir, { task: requestedTask, now }), resumed: false };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function transitionRun(
|
|
166
|
+
rootDir,
|
|
167
|
+
runId,
|
|
168
|
+
to,
|
|
169
|
+
{ gate, artifacts = {}, reason = null, incrementIteration = false, now = new Date() } = {}
|
|
170
|
+
) {
|
|
171
|
+
const run = loadRun(rootDir, runId);
|
|
172
|
+
if (!RUN_STATES[to]) {
|
|
173
|
+
const err = new Error(`Unknown run state: ${to}`);
|
|
174
|
+
err.code = 'UNKNOWN_RUN_STATE';
|
|
175
|
+
throw err;
|
|
176
|
+
}
|
|
177
|
+
if (!ALLOWED_TRANSITIONS[run.state].includes(to)) {
|
|
178
|
+
const err = new Error(`Invalid transition: ${run.state} -> ${to}`);
|
|
179
|
+
err.code = 'INVALID_TRANSITION';
|
|
180
|
+
throw err;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const timestamp = now.toISOString();
|
|
184
|
+
const next = {
|
|
185
|
+
...run,
|
|
186
|
+
state: to,
|
|
187
|
+
iteration: incrementIteration ? run.iteration + 1 : run.iteration,
|
|
188
|
+
updatedAt: timestamp,
|
|
189
|
+
artifacts: { ...run.artifacts, ...artifacts },
|
|
190
|
+
history: [
|
|
191
|
+
...run.history,
|
|
192
|
+
{
|
|
193
|
+
from: run.state,
|
|
194
|
+
to,
|
|
195
|
+
gate: gate || 'unspecified',
|
|
196
|
+
at: timestamp,
|
|
197
|
+
...(reason ? { reason } : {})
|
|
198
|
+
}
|
|
199
|
+
]
|
|
200
|
+
};
|
|
201
|
+
atomicWriteJson(runStatePath(rootDir, runId), next);
|
|
202
|
+
atomicWriteJson(currentRunPath(rootDir), { schemaVersion: 1, runId });
|
|
203
|
+
return next;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
module.exports = {
|
|
207
|
+
RUN_STATES,
|
|
208
|
+
ALLOWED_TRANSITIONS,
|
|
209
|
+
TERMINAL_STATES,
|
|
210
|
+
runsDir,
|
|
211
|
+
runStatePath,
|
|
212
|
+
currentRunPath,
|
|
213
|
+
atomicWriteJson,
|
|
214
|
+
getGitRevision,
|
|
215
|
+
newRunId,
|
|
216
|
+
loadRun,
|
|
217
|
+
getCurrentRun,
|
|
218
|
+
createRun,
|
|
219
|
+
createOrResumeRun,
|
|
220
|
+
transitionRun
|
|
221
|
+
};
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { atomicWriteJson, getCurrentRun, loadRun, runsDir } = require('./run-state.js');
|
|
7
|
+
|
|
8
|
+
const STAGE_LIMITS = Object.freeze({
|
|
9
|
+
maker: Object.freeze({ maxFiles: 12, maxFileBytes: 32_000, maxTotalBytes: 96_000, maxEstimatedTokens: 24_000 }),
|
|
10
|
+
'devil-advocate': Object.freeze({ maxFiles: 8, maxFileBytes: 16_000, maxTotalBytes: 64_000, maxEstimatedTokens: 16_000 }),
|
|
11
|
+
judge: Object.freeze({ maxFiles: 3, maxFileBytes: 12_000, maxTotalBytes: 32_000, maxEstimatedTokens: 8_000 })
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const SENSITIVE_NAMES = [
|
|
15
|
+
/^\.env(?:\.|$)/i,
|
|
16
|
+
/^\.npmrc$/i,
|
|
17
|
+
/^\.netrc$/i,
|
|
18
|
+
/^credentials(?:\.json)?$/i,
|
|
19
|
+
/^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.pub)?$/i,
|
|
20
|
+
/\.(?:pem|key|p12|pfx|jks)$/i
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
function contextError(message, code = 'UNSAFE_CONTEXT') {
|
|
24
|
+
const err = new Error(message);
|
|
25
|
+
err.code = code;
|
|
26
|
+
return err;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function inside(parent, child) {
|
|
30
|
+
const relative = path.relative(parent, child);
|
|
31
|
+
return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function isSensitivePath(relativePath) {
|
|
35
|
+
const segments = relativePath.split(/[\\/]/);
|
|
36
|
+
if (segments.includes('.git') || segments.includes('node_modules')) return true;
|
|
37
|
+
return segments.some((segment) => SENSITIVE_NAMES.some((pattern) => pattern.test(segment)));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function resolveSafeRepoFile(rootDir, relativePath) {
|
|
41
|
+
if (!relativePath || path.isAbsolute(relativePath)) {
|
|
42
|
+
throw contextError(`Context path must be repository-relative: ${relativePath || '(empty)'}`);
|
|
43
|
+
}
|
|
44
|
+
const root = fs.realpathSync(rootDir);
|
|
45
|
+
const requested = path.resolve(root, relativePath);
|
|
46
|
+
if (!inside(root, requested)) throw contextError(`Context path escapes the repository: ${relativePath}`);
|
|
47
|
+
const normalized = path.relative(root, requested);
|
|
48
|
+
if (isSensitivePath(normalized)) throw contextError(`Sensitive context path is blocked: ${relativePath}`);
|
|
49
|
+
|
|
50
|
+
let real;
|
|
51
|
+
try {
|
|
52
|
+
real = fs.realpathSync(requested);
|
|
53
|
+
} catch (cause) {
|
|
54
|
+
throw contextError(`Context file is unavailable: ${relativePath} (${cause.message})`, 'CONTEXT_FILE_MISSING');
|
|
55
|
+
}
|
|
56
|
+
if (!inside(root, real)) throw contextError(`Context symlink escapes the repository: ${relativePath}`);
|
|
57
|
+
const stat = fs.statSync(real);
|
|
58
|
+
if (!stat.isFile()) throw contextError(`Context path is not a file: ${relativePath}`);
|
|
59
|
+
return { absolute: real, relative: path.relative(root, real).split(path.sep).join('/') };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function redactSecrets(input) {
|
|
63
|
+
let text = String(input || '');
|
|
64
|
+
const categories = new Set();
|
|
65
|
+
const replace = (pattern, category, replacement) => {
|
|
66
|
+
text = text.replace(pattern, (...args) => {
|
|
67
|
+
categories.add(category);
|
|
68
|
+
return typeof replacement === 'function' ? replacement(...args) : replacement;
|
|
69
|
+
});
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
replace(
|
|
73
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
74
|
+
'private-key',
|
|
75
|
+
'[REDACTED PRIVATE KEY]'
|
|
76
|
+
);
|
|
77
|
+
replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, 'bearer-token', 'Bearer [REDACTED]');
|
|
78
|
+
replace(/\bgithub_pat_[A-Za-z0-9_]{12,}\b/g, 'github-token', '[REDACTED GITHUB TOKEN]');
|
|
79
|
+
replace(/\bgh[pousr]_[A-Za-z0-9]{12,}\b/g, 'github-token', '[REDACTED GITHUB TOKEN]');
|
|
80
|
+
replace(/\bAKIA[A-Z0-9]{16}\b/g, 'aws-access-key', '[REDACTED AWS ACCESS KEY]');
|
|
81
|
+
replace(/\bsk-(?:live-|test-)?[A-Za-z0-9_-]{8,}\b/g, 'api-token', '[REDACTED API TOKEN]');
|
|
82
|
+
replace(
|
|
83
|
+
/(\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|secret|password|authorization)\b\s*[:=]\s*["']?)([^"'\s,;]+)/gi,
|
|
84
|
+
'key-value-secret',
|
|
85
|
+
(_match, prefix) => `${prefix}[REDACTED]`
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
return { text, categories: [...categories].sort() };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function sha256(content) {
|
|
92
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function estimateTokens(text) {
|
|
96
|
+
return Math.ceil(String(text || '').length / 4);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function resolveRun(rootDir, runId) {
|
|
100
|
+
if (runId) return loadRun(rootDir, runId);
|
|
101
|
+
const current = getCurrentRun(rootDir);
|
|
102
|
+
if (!current) throw contextError('No current run. Start one before building context.', 'NO_CURRENT_RUN');
|
|
103
|
+
return current;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function createContextPack(rootDir, { runId, stage, files } = {}) {
|
|
107
|
+
const limits = STAGE_LIMITS[stage];
|
|
108
|
+
if (!limits) {
|
|
109
|
+
throw contextError(`Unknown context stage: ${stage}. Use ${Object.keys(STAGE_LIMITS).join('|')}`, 'UNKNOWN_CONTEXT_STAGE');
|
|
110
|
+
}
|
|
111
|
+
const run = resolveRun(rootDir, runId);
|
|
112
|
+
const uniqueFiles = [...new Set((files || []).map((file) => String(file).trim()).filter(Boolean))];
|
|
113
|
+
if (uniqueFiles.length === 0) throw contextError('At least one context file is required', 'EMPTY_CONTEXT');
|
|
114
|
+
if (uniqueFiles.length > limits.maxFiles) {
|
|
115
|
+
throw contextError(`${stage} context allows at most ${limits.maxFiles} files`, 'CONTEXT_FILE_LIMIT');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const entries = [];
|
|
119
|
+
let totalBytes = 0;
|
|
120
|
+
let estimatedTokens = 0;
|
|
121
|
+
for (const requested of uniqueFiles) {
|
|
122
|
+
const safe = resolveSafeRepoFile(rootDir, requested);
|
|
123
|
+
const raw = fs.readFileSync(safe.absolute);
|
|
124
|
+
if (raw.includes(0)) throw contextError(`Binary context is blocked: ${requested}`, 'BINARY_CONTEXT');
|
|
125
|
+
|
|
126
|
+
const remaining = limits.maxTotalBytes - totalBytes;
|
|
127
|
+
if (remaining <= 0) throw contextError(`${stage} context exceeds its byte budget`, 'CONTEXT_BYTE_LIMIT');
|
|
128
|
+
const allowedBytes = Math.min(raw.length, limits.maxFileBytes, remaining);
|
|
129
|
+
const source = raw.subarray(0, allowedBytes).toString('utf8');
|
|
130
|
+
const redacted = redactSecrets(source);
|
|
131
|
+
const tokens = estimateTokens(redacted.text);
|
|
132
|
+
if (estimatedTokens + tokens > limits.maxEstimatedTokens) {
|
|
133
|
+
throw contextError(`${stage} context exceeds its estimated token budget`, 'CONTEXT_TOKEN_LIMIT');
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
totalBytes += allowedBytes;
|
|
137
|
+
estimatedTokens += tokens;
|
|
138
|
+
entries.push({
|
|
139
|
+
path: safe.relative,
|
|
140
|
+
sourceSha256: sha256(raw),
|
|
141
|
+
sourceBytes: raw.length,
|
|
142
|
+
includedBytes: allowedBytes,
|
|
143
|
+
estimatedTokens: tokens,
|
|
144
|
+
truncated: allowedBytes < raw.length,
|
|
145
|
+
redactions: redacted.categories,
|
|
146
|
+
content: redacted.text
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const pack = {
|
|
151
|
+
schemaVersion: 1,
|
|
152
|
+
runId: run.runId,
|
|
153
|
+
stage,
|
|
154
|
+
createdAt: new Date().toISOString(),
|
|
155
|
+
transcriptInherited: false,
|
|
156
|
+
limits,
|
|
157
|
+
totalBytes,
|
|
158
|
+
estimatedTokens,
|
|
159
|
+
entries
|
|
160
|
+
};
|
|
161
|
+
const packPath = path.join(runsDir(rootDir), run.runId, 'context', `${stage}.json`);
|
|
162
|
+
atomicWriteJson(packPath, pack);
|
|
163
|
+
return {
|
|
164
|
+
pack,
|
|
165
|
+
path: path.relative(rootDir, packPath).split(path.sep).join('/')
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function contextPackSummary(result) {
|
|
170
|
+
return {
|
|
171
|
+
schemaVersion: result.pack.schemaVersion,
|
|
172
|
+
runId: result.pack.runId,
|
|
173
|
+
stage: result.pack.stage,
|
|
174
|
+
path: result.path,
|
|
175
|
+
files: result.pack.entries.length,
|
|
176
|
+
totalBytes: result.pack.totalBytes,
|
|
177
|
+
estimatedTokens: result.pack.estimatedTokens,
|
|
178
|
+
redactions: result.pack.entries.reduce((sum, entry) => sum + entry.redactions.length, 0),
|
|
179
|
+
truncatedFiles: result.pack.entries.filter((entry) => entry.truncated).length
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
module.exports = {
|
|
184
|
+
STAGE_LIMITS,
|
|
185
|
+
isSensitivePath,
|
|
186
|
+
resolveSafeRepoFile,
|
|
187
|
+
redactSecrets,
|
|
188
|
+
estimateTokens,
|
|
189
|
+
createContextPack,
|
|
190
|
+
contextPackSummary
|
|
191
|
+
};
|
package/lib/sync-hosts.js
CHANGED
|
@@ -14,12 +14,13 @@ function homeDir(env = process.env) {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
function applyGrokSkillOverlay(content) {
|
|
17
|
-
const
|
|
18
|
-
|
|
17
|
+
const normalized = String(content).replace(/\r\n?/g, '\n');
|
|
18
|
+
const match = normalized.match(/^---\n([\s\S]*?)\n---\n/);
|
|
19
|
+
if (!match) return normalized;
|
|
19
20
|
const yaml = match[1];
|
|
20
|
-
if (/^user-invocable:\s*true\s*$/m.test(yaml)) return
|
|
21
|
+
if (/^user-invocable:\s*true\s*$/m.test(yaml)) return normalized;
|
|
21
22
|
const nextYaml = `${yaml.replace(/\s*$/, '')}\nuser-invocable: true`;
|
|
22
|
-
return `---\n${nextYaml}\n---\n${
|
|
23
|
+
return `---\n${nextYaml}\n---\n${normalized.slice(match[0].length)}`;
|
|
23
24
|
}
|
|
24
25
|
|
|
25
26
|
function hostFileMap(home) {
|
|
@@ -133,7 +134,7 @@ function planHostSync({ packageRoot: root = packageRoot(), home = homeDir() } =
|
|
|
133
134
|
results.push({ ...item, action: 'skip', reason: 'not-installed' });
|
|
134
135
|
continue;
|
|
135
136
|
}
|
|
136
|
-
let content = fs.readFileSync(srcAbs, 'utf8');
|
|
137
|
+
let content = fs.readFileSync(srcAbs, 'utf8').replace(/\r\n?/g, '\n');
|
|
137
138
|
if (file.grokSkillOverlay) content = applyGrokSkillOverlay(content);
|
|
138
139
|
if (destExists(file.dest) && fs.readFileSync(file.dest, 'utf8') === content) {
|
|
139
140
|
results.push({ ...item, action: 'current' });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-engineering-loop",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "A reusable, framework-agnostic AI Engineering Operating System for autonomous coding agents.",
|
|
5
5
|
"main": "bin/ai-engineering-loop.js",
|
|
6
6
|
"bin": {
|
|
@@ -14,11 +14,8 @@
|
|
|
14
14
|
"policies/",
|
|
15
15
|
"profiles/",
|
|
16
16
|
"adapters/",
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"docs/",
|
|
20
|
-
"tests/",
|
|
21
|
-
"scripts/",
|
|
17
|
+
"schemas/",
|
|
18
|
+
"evals/",
|
|
22
19
|
".agents/",
|
|
23
20
|
".grok/",
|
|
24
21
|
".gemini/skills/",
|
|
@@ -26,12 +23,18 @@
|
|
|
26
23
|
".claude/commands/",
|
|
27
24
|
".claude/skills/",
|
|
28
25
|
"LICENSE",
|
|
29
|
-
"
|
|
30
|
-
"
|
|
26
|
+
"SECURITY.md",
|
|
27
|
+
"SUPPORT.md",
|
|
28
|
+
"README.md"
|
|
31
29
|
],
|
|
32
30
|
"scripts": {
|
|
33
31
|
"init": "node bin/ai-engineering-loop.js",
|
|
34
|
-
"test": "node --test
|
|
32
|
+
"test": "node --test",
|
|
33
|
+
"doctor": "node bin/ai-engineering-loop.js doctor",
|
|
34
|
+
"eval": "node bin/ai-engineering-loop.js eval"
|
|
35
|
+
},
|
|
36
|
+
"engines": {
|
|
37
|
+
"node": ">=18"
|
|
35
38
|
},
|
|
36
39
|
"repository": {
|
|
37
40
|
"type": "git",
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Model Escalation Policy
|
|
2
|
+
|
|
3
|
+
Cheap models are the default Maker, Devil's Advocate, and Judge. Required workflow stages never disappear when the model tier changes.
|
|
4
|
+
|
|
5
|
+
The objective is not to prevent every model error. Unsupported output must fail a deterministic gate, and repeated uncertainty must move to a stronger reviewer.
|
|
6
|
+
|
|
7
|
+
## Deterministic routing
|
|
8
|
+
|
|
9
|
+
Use `CHEAP_MODEL` when all evidence is current, context is complete, and no high-risk path changed.
|
|
10
|
+
|
|
11
|
+
Require `STRONG_MODEL` when any of these signals exist:
|
|
12
|
+
|
|
13
|
+
- Authentication, authorization, payment, billing, database migration, infrastructure, cryptography, or secret-handling paths changed.
|
|
14
|
+
- The model emitted two invalid structured artifacts.
|
|
15
|
+
- A finding cites a path that does not exist.
|
|
16
|
+
- Verification is flaky.
|
|
17
|
+
- A stage context pack was truncated.
|
|
18
|
+
- A critical acceptance criterion remains `UNKNOWN`.
|
|
19
|
+
- Devil's Advocate and Judge disagree about a blocking finding.
|
|
20
|
+
|
|
21
|
+
Require `HUMAN` when:
|
|
22
|
+
|
|
23
|
+
- The iteration ceiling is reached.
|
|
24
|
+
- A credential or evaluation canary reached model-visible output.
|
|
25
|
+
|
|
26
|
+
Diff size alone is not a risk classification. A one-line authorization change can require stronger review.
|
|
27
|
+
|
|
28
|
+
## Verdict metadata
|
|
29
|
+
|
|
30
|
+
Judge writes `modelTier` in `verdict.json`:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{
|
|
34
|
+
"modelTier": "CHEAP_MODEL",
|
|
35
|
+
"humanApproved": false
|
|
36
|
+
}
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`ai-engineering-loop gate judge` computes the required tier. A lower tier may safely issue `ESCALATE`, but it cannot issue `PASS` when stronger review is required.
|
|
40
|
+
|
|
41
|
+
Run `ai-engineering-loop escalation --json` before spawning Judge to select the minimum acceptable tier. Do not invent a provider or model slug; resolve the tier through the active host's model catalog.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/delivery.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Delivery Record",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "destination", "summary"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"destination": { "type": "string", "minLength": 1 },
|
|
11
|
+
"summary": { "type": "string", "minLength": 1 }
|
|
12
|
+
},
|
|
13
|
+
"additionalProperties": true
|
|
14
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/finding-ledger.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Finding Ledger",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "diffHash", "findings"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"diffHash": { "type": "string", "minLength": 1 },
|
|
11
|
+
"findings": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"items": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"required": ["id", "location", "failureScenario", "evidence", "severity", "validity", "disposition"],
|
|
16
|
+
"properties": {
|
|
17
|
+
"axis": { "enum": ["spec", "standards"] },
|
|
18
|
+
"severity": { "enum": ["BLOCKER", "HIGH", "MEDIUM", "LOW"] },
|
|
19
|
+
"validity": { "enum": ["VALID", "INVALID"] },
|
|
20
|
+
"disposition": { "enum": ["STRONG", "ACCEPTABLE", "WEAK"] }
|
|
21
|
+
},
|
|
22
|
+
"additionalProperties": true
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"additionalProperties": true
|
|
27
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/goal-contract.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Goal Contract",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "objective", "acceptanceCriteria"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"objective": { "type": "string", "minLength": 1 },
|
|
11
|
+
"acceptanceCriteria": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"minItems": 1,
|
|
14
|
+
"items": {
|
|
15
|
+
"type": "object",
|
|
16
|
+
"required": ["id", "statement", "evidenceRequired", "failureCases"],
|
|
17
|
+
"properties": {
|
|
18
|
+
"id": { "type": "string", "pattern": "^AC-[0-9]+$" },
|
|
19
|
+
"statement": { "type": "string", "minLength": 1 },
|
|
20
|
+
"evidenceRequired": { "type": "string", "minLength": 1 },
|
|
21
|
+
"failureCases": {
|
|
22
|
+
"type": "array",
|
|
23
|
+
"minItems": 1,
|
|
24
|
+
"items": { "type": "string", "minLength": 1 }
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"additionalProperties": true
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"additionalProperties": true
|
|
32
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/verdict.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Judge Verdict",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "verdict", "reason", "action"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"verdict": { "enum": ["PASS", "ITERATE", "ESCALATE"] },
|
|
11
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
12
|
+
"action": { "type": "string", "minLength": 1 },
|
|
13
|
+
"modelTier": { "enum": ["CHEAP_MODEL", "STRONG_MODEL", "HUMAN"] },
|
|
14
|
+
"humanApproved": { "type": "boolean" }
|
|
15
|
+
},
|
|
16
|
+
"additionalProperties": true
|
|
17
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/egagofur/ai-engineering-loop/schemas/verification.schema.json",
|
|
4
|
+
"title": "AI Engineering Loop Verification Evidence",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "runId", "gitRevision", "diffHash", "commands"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"runId": { "type": "string", "minLength": 1 },
|
|
10
|
+
"gitRevision": { "type": "string", "minLength": 1 },
|
|
11
|
+
"diffHash": { "type": "string", "minLength": 1 },
|
|
12
|
+
"commands": {
|
|
13
|
+
"type": "array",
|
|
14
|
+
"minItems": 1,
|
|
15
|
+
"items": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"required": ["command", "executionIdentity", "startTime", "endTime", "exitCode", "stdout", "timeoutStatus"],
|
|
18
|
+
"properties": {
|
|
19
|
+
"command": { "type": "string", "minLength": 1 },
|
|
20
|
+
"executionIdentity": { "type": "string", "minLength": 1 },
|
|
21
|
+
"startTime": { "type": "string", "minLength": 1 },
|
|
22
|
+
"endTime": { "type": "string", "minLength": 1 },
|
|
23
|
+
"exitCode": { "const": 0 },
|
|
24
|
+
"stdout": { "type": "string", "minLength": 1 },
|
|
25
|
+
"timeoutStatus": { "const": "COMPLETED" }
|
|
26
|
+
},
|
|
27
|
+
"additionalProperties": true
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"additionalProperties": true
|
|
32
|
+
}
|