ai-engineering-loop 1.0.21 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +8 -4
- package/.claude/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.gemini/skills/ai-engineering-loop/SKILL.md +15 -7
- package/.grok/skills/ai-engineering-loop/SKILL.md +15 -7
- package/README.md +43 -3
- package/README.npm.md +9 -1
- package/SECURITY.md +29 -0
- package/SUPPORT.md +28 -0
- package/agents/judge.md +5 -5
- package/agents/maker.md +6 -6
- package/bin/ai-engineering-loop.js +472 -10
- package/core/artifact-gates.md +49 -0
- package/core/configuration-precedence.md +4 -4
- package/core/context-impact-assessment.md +1 -1
- package/core/definition-of-done.md +3 -3
- package/core/escalation-policy.md +2 -2
- package/core/goal-contract.md +2 -2
- package/core/iteration-policy.md +4 -4
- package/core/project-initialization.md +2 -2
- package/core/runtime-safety.md +75 -0
- package/core/verification-loop.md +1 -1
- package/evals/README.md +35 -0
- package/evals/cases/01-stale-verification.json +17 -0
- package/evals/cases/02-vague-verification.json +16 -0
- package/evals/cases/03-missing-ac-evidence.json +17 -0
- package/evals/cases/04-hallucinated-citation.json +17 -0
- package/evals/cases/05-prompt-injection.json +17 -0
- package/evals/cases/06-source-secret.json +16 -0
- package/evals/cases/07-log-secret.json +16 -0
- package/evals/cases/08-symlink-escape.json +16 -0
- package/evals/cases/09-authorization-bypass.json +16 -0
- package/evals/cases/10-payment-idempotency.json +16 -0
- package/evals/cases/11-concurrent-update.json +16 -0
- package/evals/cases/12-swallowed-error.json +16 -0
- package/evals/cases/13-happy-path-only.json +17 -0
- package/evals/cases/14-invalid-finding-axis.json +16 -0
- package/evals/cases/15-blocker-without-alternative.json +16 -0
- package/evals/cases/16-unsupported-pass.json +17 -0
- package/evals/cases/17-soft-standard.json +16 -0
- package/evals/cases/18-iteration-ceiling.json +17 -0
- package/evals/cases/19-safe-documentation-change.json +17 -0
- package/evals/cases/20-valid-boundary-feature.json +18 -0
- package/lib/budget.js +220 -0
- package/lib/doctor.js +111 -0
- package/lib/escalation.js +192 -0
- package/lib/evaluation.js +137 -0
- package/lib/gates.js +369 -0
- package/lib/orchestration.js +5 -1
- package/lib/package-audit.js +78 -0
- package/lib/run-state.js +260 -0
- package/lib/runtime-files.js +53 -0
- package/lib/runtime-policy.js +118 -0
- package/lib/safe-context.js +194 -0
- package/lib/sandbox.js +305 -0
- package/lib/sync-hosts.js +6 -5
- package/package.json +16 -10
- package/policies/finding-policy.md +2 -2
- package/policies/model-escalation.md +41 -0
- package/policies/no-progress-policy.md +2 -2
- package/profiles/README.md +5 -5
- package/profiles/backend-api.md +1 -1
- package/profiles/library.md +1 -1
- package/profiles/mobile-app.md +1 -1
- package/profiles/monorepo.md +1 -1
- package/profiles/web-app.md +1 -1
- package/schemas/delivery.schema.json +15 -0
- package/schemas/finding-ledger.schema.json +27 -0
- package/schemas/goal-contract.schema.json +32 -0
- package/schemas/report.schema.json +17 -0
- package/schemas/verdict.schema.json +17 -0
- package/schemas/verification.schema.json +32 -0
- package/docs/antigravity-feasibility.md +0 -92
- package/docs/claude-code-feasibility.md +0 -80
- package/docs/grok-cli-feasibility.md +0 -99
- package/docs/migration-plan.md +0 -70
- package/examples/backend-api/payment-idempotency/README.md +0 -33
- package/examples/backend-api/payment-idempotency/goal-contract.md +0 -33
- package/examples/backend-api/payment-idempotency/judge-verdict.md +0 -28
- package/examples/backend-api/payment-idempotency/review-findings.md +0 -50
- package/examples/dot/status-display/README.md +0 -23
- package/examples/dot/status-display/delivery-report.md +0 -41
- package/examples/dot/status-display/goal-contract.md +0 -46
- package/examples/dot/status-display/judge-verdict.md +0 -44
- package/examples/dot/status-display/review-findings.md +0 -63
- package/examples/initialization/README.md +0 -19
- package/examples/initialization/discovery-trace.md +0 -63
- package/examples/initialization/generated-context.md +0 -110
- package/examples/mobile-app/offline-sync-queue/README.md +0 -33
- package/examples/mobile-app/offline-sync-queue/goal-contract.md +0 -32
- package/examples/mobile-app/offline-sync-queue/judge-verdict.md +0 -27
- package/examples/mobile-app/offline-sync-queue/review-findings.md +0 -30
- package/scripts/init.sh +0 -106
- package/templates/repo-config/adapter.md +0 -11
- package/templates/repo-config/adr-readme.md +0 -33
- package/templates/repo-config/architecture.md +0 -15
- package/templates/repo-config/claude-permissions.json +0 -10
- package/templates/repo-config/config.md +0 -11
- package/templates/repo-config/conventions.md +0 -16
- package/templates/repo-config/glossary.md +0 -18
- package/templates/repo-config/lessons.md +0 -8
- package/templates/repo-config/verification.md +0 -13
- package/templates/repo-config/workflow.md +0 -25
- package/tests/capability-selection.test.js +0 -207
- package/tests/claimed-vs-reality.test.js +0 -27
- package/tests/dot-branch-policy.test.js +0 -65
- package/tests/generate-adapter.test.js +0 -80
- package/tests/generate-workflow.test.js +0 -180
- package/tests/grok-runtime.test.js +0 -201
- package/tests/intern-catalog.test.js +0 -72
- package/tests/living-context.test.js +0 -61
- package/tests/no-company-leak.test.js +0 -82
- package/tests/orchestration.test.js +0 -368
- package/tests/qa-delivery-template.test.js +0 -90
- package/tests/skill-host-compat.test.js +0 -406
- package/tests/sync-hosts.test.js +0 -168
package/lib/budget.js
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { assertRunId, getCurrentRun, loadRun } = require('./run-state.js');
|
|
6
|
+
const { loadPolicy, updatePolicy } = require('./runtime-policy.js');
|
|
7
|
+
const { ensurePrivateRuntimeIgnores } = require('./runtime-files.js');
|
|
8
|
+
|
|
9
|
+
function usageDir(rootDir) {
|
|
10
|
+
return path.join(rootDir, '.ai-engineering-loop', 'usage');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function utcDay(now = new Date()) {
|
|
14
|
+
if (!(now instanceof Date) || Number.isNaN(now.getTime())) {
|
|
15
|
+
const err = new Error('Usage timestamp must be a valid Date');
|
|
16
|
+
err.code = 'INVALID_USAGE';
|
|
17
|
+
throw err;
|
|
18
|
+
}
|
|
19
|
+
return now.toISOString().slice(0, 10);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function usagePath(rootDir, now = new Date()) {
|
|
23
|
+
return path.join(usageDir(rootDir), `${utcDay(now)}.jsonl`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function usageError(message, code = 'INVALID_USAGE') {
|
|
27
|
+
const err = new Error(message);
|
|
28
|
+
err.code = code;
|
|
29
|
+
return err;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function validateTokenCount(name, value) {
|
|
33
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
34
|
+
throw usageError(`${name} must be a non-negative integer`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function validateModel(model) {
|
|
39
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$/.test(String(model || ''))) {
|
|
40
|
+
throw usageError('model must be a provider model identifier, not free-form text');
|
|
41
|
+
}
|
|
42
|
+
return String(model);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function readUsageFile(filePath) {
|
|
46
|
+
let stat;
|
|
47
|
+
try {
|
|
48
|
+
stat = fs.lstatSync(filePath);
|
|
49
|
+
} catch (cause) {
|
|
50
|
+
if (cause.code === 'ENOENT') return [];
|
|
51
|
+
throw usageError(`Unable to inspect usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
52
|
+
}
|
|
53
|
+
if (stat.isSymbolicLink()) {
|
|
54
|
+
throw usageError('Usage ledger must not be a symbolic link', 'INVALID_USAGE_LEDGER');
|
|
55
|
+
}
|
|
56
|
+
let lines;
|
|
57
|
+
try {
|
|
58
|
+
lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
|
|
59
|
+
} catch (cause) {
|
|
60
|
+
throw usageError(`Unable to read usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
61
|
+
}
|
|
62
|
+
return lines.map((line, index) => {
|
|
63
|
+
try {
|
|
64
|
+
const entry = JSON.parse(line);
|
|
65
|
+
assertRunId(entry.runId);
|
|
66
|
+
validateTokenCount('inputTokens', entry.inputTokens);
|
|
67
|
+
validateTokenCount('outputTokens', entry.outputTokens);
|
|
68
|
+
if (entry.totalTokens !== entry.inputTokens + entry.outputTokens) {
|
|
69
|
+
throw new Error('totalTokens does not equal inputTokens + outputTokens');
|
|
70
|
+
}
|
|
71
|
+
validateModel(entry.model);
|
|
72
|
+
if (typeof entry.recordedAt !== 'string' || Number.isNaN(Date.parse(entry.recordedAt))) {
|
|
73
|
+
throw new Error('recordedAt is invalid');
|
|
74
|
+
}
|
|
75
|
+
return entry;
|
|
76
|
+
} catch (cause) {
|
|
77
|
+
throw usageError(
|
|
78
|
+
`Usage ledger line ${index + 1} is invalid: ${cause.message}`,
|
|
79
|
+
'INVALID_USAGE_LEDGER'
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function readUsage(rootDir, now = new Date()) {
|
|
86
|
+
return readUsageFile(usagePath(rootDir, now));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function readAllUsage(rootDir) {
|
|
90
|
+
const directory = usageDir(rootDir);
|
|
91
|
+
if (!fs.existsSync(directory)) return [];
|
|
92
|
+
let names;
|
|
93
|
+
try {
|
|
94
|
+
names = fs.readdirSync(directory)
|
|
95
|
+
.filter((name) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(name))
|
|
96
|
+
.sort();
|
|
97
|
+
} catch (cause) {
|
|
98
|
+
throw usageError(`Unable to enumerate usage ledgers: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
99
|
+
}
|
|
100
|
+
return names.flatMap((name) => readUsageFile(path.join(directory, name)));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function resolveRun(rootDir, runId) {
|
|
104
|
+
if (runId) return loadRun(rootDir, assertRunId(runId));
|
|
105
|
+
const current = getCurrentRun(rootDir);
|
|
106
|
+
if (!current) throw usageError('No current run. Pass --run <id>.', 'NO_CURRENT_RUN');
|
|
107
|
+
return current;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function budgetStatus(rootDir, { runId, estimatedTokens = 0, now = new Date() } = {}) {
|
|
111
|
+
validateTokenCount('estimatedTokens', estimatedTokens);
|
|
112
|
+
const run = resolveRun(rootDir, runId);
|
|
113
|
+
const policy = loadPolicy(rootDir);
|
|
114
|
+
const todayEntries = readUsage(rootDir, now);
|
|
115
|
+
const allEntries = readAllUsage(rootDir);
|
|
116
|
+
const spentToday = todayEntries.reduce((sum, entry) => sum + entry.totalTokens, 0);
|
|
117
|
+
const spentThisRun = allEntries
|
|
118
|
+
.filter((entry) => entry.runId === run.runId)
|
|
119
|
+
.reduce((sum, entry) => sum + entry.totalTokens, 0);
|
|
120
|
+
const projectedToday = spentToday + estimatedTokens;
|
|
121
|
+
const projectedThisRun = spentThisRun + estimatedTokens;
|
|
122
|
+
const reasons = [];
|
|
123
|
+
if (policy.killSwitch) reasons.push('kill switch is active');
|
|
124
|
+
if (projectedThisRun > policy.perRunTokenLimit) reasons.push('per-run token limit would be exceeded');
|
|
125
|
+
if (projectedToday > policy.dailyTokenLimit) reasons.push('daily token limit would be exceeded');
|
|
126
|
+
if (estimatedTokens === 0 && spentThisRun >= policy.perRunTokenLimit) {
|
|
127
|
+
reasons.push('per-run token limit is exhausted');
|
|
128
|
+
}
|
|
129
|
+
if (estimatedTokens === 0 && spentToday >= policy.dailyTokenLimit) {
|
|
130
|
+
reasons.push('daily token limit is exhausted');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
schemaVersion: 1,
|
|
135
|
+
runId: run.runId,
|
|
136
|
+
day: utcDay(now),
|
|
137
|
+
allowed: reasons.length === 0,
|
|
138
|
+
reasons: [...new Set(reasons)],
|
|
139
|
+
killSwitch: policy.killSwitch,
|
|
140
|
+
spentThisRun,
|
|
141
|
+
perRunTokenLimit: policy.perRunTokenLimit,
|
|
142
|
+
remainingThisRun: Math.max(0, policy.perRunTokenLimit - spentThisRun),
|
|
143
|
+
spentToday,
|
|
144
|
+
dailyTokenLimit: policy.dailyTokenLimit,
|
|
145
|
+
remainingToday: Math.max(0, policy.dailyTokenLimit - spentToday),
|
|
146
|
+
estimatedTokens
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function assertBudgetAvailable(rootDir, options = {}) {
|
|
151
|
+
const status = budgetStatus(rootDir, options);
|
|
152
|
+
if (!status.allowed) {
|
|
153
|
+
const err = new Error(`Token budget blocked: ${status.reasons.join('; ')}`);
|
|
154
|
+
err.code = 'BUDGET_BLOCKED';
|
|
155
|
+
err.details = status.reasons;
|
|
156
|
+
err.status = status;
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
return status;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function appendPrivateLine(rootDir, filePath, line) {
|
|
163
|
+
ensurePrivateRuntimeIgnores(rootDir);
|
|
164
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
165
|
+
const noFollow = fs.constants.O_NOFOLLOW || 0;
|
|
166
|
+
let descriptor;
|
|
167
|
+
try {
|
|
168
|
+
descriptor = fs.openSync(
|
|
169
|
+
filePath,
|
|
170
|
+
fs.constants.O_APPEND | fs.constants.O_CREAT | fs.constants.O_WRONLY | noFollow,
|
|
171
|
+
0o600
|
|
172
|
+
);
|
|
173
|
+
if (!fs.fstatSync(descriptor).isFile()) throw new Error('usage ledger is not a regular file');
|
|
174
|
+
fs.writeSync(descriptor, line);
|
|
175
|
+
fs.fchmodSync(descriptor, 0o600);
|
|
176
|
+
} catch (cause) {
|
|
177
|
+
throw usageError(`Unable to append usage ledger: ${cause.message}`, 'USAGE_WRITE_FAILED');
|
|
178
|
+
} finally {
|
|
179
|
+
if (descriptor != null) fs.closeSync(descriptor);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function recordTokenUsage(
|
|
184
|
+
rootDir,
|
|
185
|
+
{ runId, inputTokens, outputTokens, model, now = new Date() } = {}
|
|
186
|
+
) {
|
|
187
|
+
const run = resolveRun(rootDir, runId);
|
|
188
|
+
validateTokenCount('inputTokens', inputTokens);
|
|
189
|
+
validateTokenCount('outputTokens', outputTokens);
|
|
190
|
+
const normalizedModel = validateModel(model);
|
|
191
|
+
const totalTokens = inputTokens + outputTokens;
|
|
192
|
+
if (totalTokens === 0) throw usageError('A usage record must contain at least one token');
|
|
193
|
+
const entry = {
|
|
194
|
+
schemaVersion: 1,
|
|
195
|
+
runId: run.runId,
|
|
196
|
+
inputTokens,
|
|
197
|
+
outputTokens,
|
|
198
|
+
totalTokens,
|
|
199
|
+
model: normalizedModel,
|
|
200
|
+
recordedAt: now.toISOString()
|
|
201
|
+
};
|
|
202
|
+
appendPrivateLine(rootDir, usagePath(rootDir, now), `${JSON.stringify(entry)}\n`);
|
|
203
|
+
return { entry, status: budgetStatus(rootDir, { runId: run.runId, now }) };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function setKillSwitch(rootDir, active) {
|
|
207
|
+
return updatePolicy(rootDir, { killSwitch: Boolean(active) });
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
module.exports = {
|
|
211
|
+
usageDir,
|
|
212
|
+
utcDay,
|
|
213
|
+
usagePath,
|
|
214
|
+
readUsage,
|
|
215
|
+
readAllUsage,
|
|
216
|
+
budgetStatus,
|
|
217
|
+
assertBudgetAvailable,
|
|
218
|
+
recordTokenUsage,
|
|
219
|
+
setKillSwitch
|
|
220
|
+
};
|
package/lib/doctor.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { loadEvaluationCases } = require('./evaluation.js');
|
|
6
|
+
const { auditPackage } = require('./package-audit.js');
|
|
7
|
+
|
|
8
|
+
const REQUIRED_HOST_ASSETS = [
|
|
9
|
+
'.claude/skills/ai-engineering-loop/SKILL.md',
|
|
10
|
+
'.claude/agents/devil-advocate.md',
|
|
11
|
+
'.claude/agents/judge.md',
|
|
12
|
+
'.grok/skills/ai-engineering-loop/SKILL.md',
|
|
13
|
+
'.grok/agents/devil-advocate.md',
|
|
14
|
+
'.grok/agents/judge.md',
|
|
15
|
+
'.gemini/skills/ai-engineering-loop/SKILL.md',
|
|
16
|
+
'.agents/workflows/ai-engineering-loop.md',
|
|
17
|
+
'.agents/devil-advocate.md',
|
|
18
|
+
'.agents/judge.md'
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
const REQUIRED_SCHEMAS = [
|
|
22
|
+
'goal-contract.schema.json',
|
|
23
|
+
'verification.schema.json',
|
|
24
|
+
'finding-ledger.schema.json',
|
|
25
|
+
'verdict.schema.json',
|
|
26
|
+
'delivery.schema.json',
|
|
27
|
+
'report.schema.json'
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
function check(id, passed, detail) {
|
|
31
|
+
return { id, passed: Boolean(passed), detail };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function runDoctor(rootDir = path.join(__dirname, '..')) {
|
|
35
|
+
const checks = [];
|
|
36
|
+
const nodeMajor = Number(process.versions.node.split('.')[0]);
|
|
37
|
+
checks.push(check('node-version', nodeMajor >= 18, `Node ${process.versions.node}; required >=18`));
|
|
38
|
+
|
|
39
|
+
let pkg;
|
|
40
|
+
try {
|
|
41
|
+
pkg = JSON.parse(fs.readFileSync(path.join(rootDir, 'package.json'), 'utf8'));
|
|
42
|
+
checks.push(check('package-json', true, `${pkg.name}@${pkg.version}`));
|
|
43
|
+
} catch (cause) {
|
|
44
|
+
checks.push(check('package-json', false, cause.message));
|
|
45
|
+
pkg = {};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const missingHosts = REQUIRED_HOST_ASSETS.filter((file) => !fs.existsSync(path.join(rootDir, file)));
|
|
49
|
+
checks.push(check(
|
|
50
|
+
'host-assets',
|
|
51
|
+
missingHosts.length === 0,
|
|
52
|
+
missingHosts.length ? `missing: ${missingHosts.join(', ')}` : `${REQUIRED_HOST_ASSETS.length} required assets present`
|
|
53
|
+
));
|
|
54
|
+
|
|
55
|
+
const schemaErrors = [];
|
|
56
|
+
for (const file of REQUIRED_SCHEMAS) {
|
|
57
|
+
try {
|
|
58
|
+
JSON.parse(fs.readFileSync(path.join(rootDir, 'schemas', file), 'utf8'));
|
|
59
|
+
} catch (cause) {
|
|
60
|
+
schemaErrors.push(`${file}: ${cause.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
checks.push(check(
|
|
64
|
+
'artifact-schemas',
|
|
65
|
+
schemaErrors.length === 0,
|
|
66
|
+
schemaErrors.length ? schemaErrors.join('; ') : `${REQUIRED_SCHEMAS.length} schemas parse`
|
|
67
|
+
));
|
|
68
|
+
|
|
69
|
+
try {
|
|
70
|
+
const fixtures = loadEvaluationCases(path.join(rootDir, 'evals', 'cases'));
|
|
71
|
+
checks.push(check('evaluation-catalog', fixtures.length === 20, `${fixtures.length} valid cases; required 20`));
|
|
72
|
+
} catch (cause) {
|
|
73
|
+
checks.push(check('evaluation-catalog', false, cause.message));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const packageFiles = new Set(pkg.files || []);
|
|
77
|
+
const requiredPackageEntries = ['bin/', 'lib/', 'core/', 'policies/', 'schemas/', 'evals/'];
|
|
78
|
+
const missingPackageEntries = requiredPackageEntries.filter((entry) => !packageFiles.has(entry));
|
|
79
|
+
checks.push(check(
|
|
80
|
+
'package-assets',
|
|
81
|
+
missingPackageEntries.length === 0,
|
|
82
|
+
missingPackageEntries.length ? `not packed: ${missingPackageEntries.join(', ')}` : 'runtime assets are allowlisted'
|
|
83
|
+
));
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const audit = auditPackage(rootDir);
|
|
87
|
+
checks.push(check(
|
|
88
|
+
'package-privacy',
|
|
89
|
+
audit.ok,
|
|
90
|
+
audit.ok ? `${audit.filesScanned} packaged files scanned` : audit.findings.map((item) => `${item.file}: ${item.reason}`).join('; ')
|
|
91
|
+
));
|
|
92
|
+
} catch (cause) {
|
|
93
|
+
checks.push(check('package-privacy', false, cause.message));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const binPath = path.join(rootDir, 'bin', 'ai-engineering-loop.js');
|
|
97
|
+
const executable = fs.existsSync(binPath) &&
|
|
98
|
+
(process.platform === 'win32' || (fs.statSync(binPath).mode & 0o111) !== 0);
|
|
99
|
+
checks.push(check('cli-executable', executable, executable ? 'CLI is executable' : 'CLI executable bit is missing'));
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
ok: checks.every((item) => item.passed),
|
|
103
|
+
checks
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = {
|
|
108
|
+
REQUIRED_HOST_ASSETS,
|
|
109
|
+
REQUIRED_SCHEMAS,
|
|
110
|
+
runDoctor
|
|
111
|
+
};
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { atomicWriteJson, getCurrentRun, loadRun, runsDir } = require('./run-state.js');
|
|
6
|
+
|
|
7
|
+
const MODEL_TIERS = Object.freeze({
|
|
8
|
+
CHEAP_MODEL: 'CHEAP_MODEL',
|
|
9
|
+
STRONG_MODEL: 'STRONG_MODEL',
|
|
10
|
+
HUMAN: 'HUMAN'
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
const TIER_RANK = Object.freeze({
|
|
14
|
+
CHEAP_MODEL: 0,
|
|
15
|
+
STRONG_MODEL: 1,
|
|
16
|
+
HUMAN: 2
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const HIGH_RISK_PATHS = [
|
|
20
|
+
/(^|\/)(auth|authorization|permissions?|access-control)(\/|[._-])/i,
|
|
21
|
+
/(^|\/)(payments?|billing|invoices?|ledger)(\/|[._-])/i,
|
|
22
|
+
/(^|\/)(migrations?|schema)(\/|[._-])/i,
|
|
23
|
+
/(^|\/)(infra|terraform|k8s|kubernetes|deploy)(\/|[._-])/i,
|
|
24
|
+
/(^|\/)(crypto|encryption|secrets?)(\/|[._-])/i
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
function escalationError(message, code = 'ESCALATION_FAILED') {
|
|
28
|
+
const err = new Error(message);
|
|
29
|
+
err.code = code;
|
|
30
|
+
return err;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function extractChangedPaths(diff) {
|
|
34
|
+
const paths = [];
|
|
35
|
+
for (const line of String(diff || '').split('\n')) {
|
|
36
|
+
const match = line.match(/^\+\+\+ b\/(.+)$/);
|
|
37
|
+
if (match && match[1] !== '/dev/null' && !paths.includes(match[1])) paths.push(match[1]);
|
|
38
|
+
}
|
|
39
|
+
return paths;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isHighRiskPath(filePath) {
|
|
43
|
+
return HIGH_RISK_PATHS.some((pattern) => pattern.test(String(filePath || '')));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function assessEscalation({
|
|
47
|
+
changedPaths = [],
|
|
48
|
+
iteration = 1,
|
|
49
|
+
maxIterations = 3,
|
|
50
|
+
schemaFailures = 0,
|
|
51
|
+
hallucinatedCitations = 0,
|
|
52
|
+
reviewerJudgeDisagreement = false,
|
|
53
|
+
verificationFlaky = false,
|
|
54
|
+
contextTruncated = false,
|
|
55
|
+
criticalUnknowns = 0,
|
|
56
|
+
secretLeakDetected = false
|
|
57
|
+
} = {}) {
|
|
58
|
+
const reasons = [];
|
|
59
|
+
let requiredTier = MODEL_TIERS.CHEAP_MODEL;
|
|
60
|
+
const requireTier = (tier, code, detail) => {
|
|
61
|
+
if (TIER_RANK[tier] > TIER_RANK[requiredTier]) requiredTier = tier;
|
|
62
|
+
if (!reasons.some((reason) => reason.code === code)) reasons.push({ code, detail, requiredTier: tier });
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const riskyPaths = changedPaths.filter(isHighRiskPath);
|
|
66
|
+
if (riskyPaths.length) {
|
|
67
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'HIGH_RISK_PATH', riskyPaths.join(', '));
|
|
68
|
+
}
|
|
69
|
+
if (schemaFailures >= 2) {
|
|
70
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'REPEATED_SCHEMA_FAILURE', `${schemaFailures} invalid model artifacts`);
|
|
71
|
+
}
|
|
72
|
+
if (hallucinatedCitations > 0) {
|
|
73
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'HALLUCINATED_CITATION', `${hallucinatedCitations} nonexistent citations`);
|
|
74
|
+
}
|
|
75
|
+
if (reviewerJudgeDisagreement) {
|
|
76
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'REVIEWER_JUDGE_DISAGREEMENT', 'Judge contradicts deterministic finding disposition');
|
|
77
|
+
}
|
|
78
|
+
if (verificationFlaky) {
|
|
79
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'FLAKY_VERIFICATION', 'Repeated verification produced inconsistent results');
|
|
80
|
+
}
|
|
81
|
+
if (contextTruncated) {
|
|
82
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'TRUNCATED_CONTEXT', 'A stage context pack omitted source bytes');
|
|
83
|
+
}
|
|
84
|
+
if (criticalUnknowns > 0) {
|
|
85
|
+
requireTier(MODEL_TIERS.STRONG_MODEL, 'CRITICAL_UNKNOWN', `${criticalUnknowns} critical acceptance criteria remain unknown`);
|
|
86
|
+
}
|
|
87
|
+
if (iteration >= maxIterations) {
|
|
88
|
+
requireTier(MODEL_TIERS.HUMAN, 'ITERATION_CEILING', `Iteration ${iteration} reached the ceiling ${maxIterations}`);
|
|
89
|
+
}
|
|
90
|
+
if (secretLeakDetected) {
|
|
91
|
+
requireTier(MODEL_TIERS.HUMAN, 'SECRET_LEAK', 'A secret canary or credential reached model-visible output');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
schemaVersion: 1,
|
|
96
|
+
requiredTier,
|
|
97
|
+
escalate: requiredTier !== MODEL_TIERS.CHEAP_MODEL,
|
|
98
|
+
riskyPaths,
|
|
99
|
+
reasons
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function tierSatisfies(actualTier, requiredTier) {
|
|
104
|
+
return Object.hasOwn(TIER_RANK, actualTier) &&
|
|
105
|
+
Object.hasOwn(TIER_RANK, requiredTier) &&
|
|
106
|
+
TIER_RANK[actualTier] >= TIER_RANK[requiredTier];
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function readOptionalJson(filePath) {
|
|
110
|
+
if (!fs.existsSync(filePath)) return {};
|
|
111
|
+
try {
|
|
112
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
113
|
+
} catch (cause) {
|
|
114
|
+
throw escalationError(`Invalid escalation signals: ${cause.message}`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function contextWasTruncated(rootDir, runId) {
|
|
119
|
+
const contextDir = path.join(runsDir(rootDir), runId, 'context');
|
|
120
|
+
if (!fs.existsSync(contextDir)) return false;
|
|
121
|
+
return fs.readdirSync(contextDir)
|
|
122
|
+
.filter((file) => file.endsWith('.json'))
|
|
123
|
+
.some((file) => {
|
|
124
|
+
const pack = readOptionalJson(path.join(contextDir, file));
|
|
125
|
+
return (pack.entries || []).some((entry) => entry.truncated === true);
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function resolveRun(rootDir, runId) {
|
|
130
|
+
if (runId) return loadRun(rootDir, runId);
|
|
131
|
+
const current = getCurrentRun(rootDir);
|
|
132
|
+
if (!current) throw escalationError('No current run. Start one before assessing escalation.', 'NO_CURRENT_RUN');
|
|
133
|
+
return current;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function assessRunEscalation(rootDir, { runId } = {}) {
|
|
137
|
+
const run = resolveRun(rootDir, runId);
|
|
138
|
+
const runDir = path.join(runsDir(rootDir), run.runId);
|
|
139
|
+
const diffPath = path.join(runDir, 'diff.patch');
|
|
140
|
+
const diff = fs.existsSync(diffPath) ? fs.readFileSync(diffPath, 'utf8') : '';
|
|
141
|
+
const signals = readOptionalJson(path.join(runDir, 'risk.json'));
|
|
142
|
+
if (signals.runId && signals.runId !== run.runId) {
|
|
143
|
+
throw escalationError(`risk.json runId must be ${run.runId}`);
|
|
144
|
+
}
|
|
145
|
+
const assessment = assessEscalation({
|
|
146
|
+
changedPaths: extractChangedPaths(diff),
|
|
147
|
+
iteration: run.iteration,
|
|
148
|
+
maxIterations: signals.maxIterations || 3,
|
|
149
|
+
schemaFailures: signals.schemaFailures || 0,
|
|
150
|
+
hallucinatedCitations: signals.hallucinatedCitations || 0,
|
|
151
|
+
reviewerJudgeDisagreement: signals.reviewerJudgeDisagreement === true,
|
|
152
|
+
verificationFlaky: signals.verificationFlaky === true,
|
|
153
|
+
contextTruncated: contextWasTruncated(rootDir, run.runId),
|
|
154
|
+
criticalUnknowns: signals.criticalUnknowns || 0,
|
|
155
|
+
secretLeakDetected: signals.secretLeakDetected === true
|
|
156
|
+
});
|
|
157
|
+
const document = {
|
|
158
|
+
...assessment,
|
|
159
|
+
runId: run.runId,
|
|
160
|
+
iteration: run.iteration,
|
|
161
|
+
assessedAt: new Date().toISOString()
|
|
162
|
+
};
|
|
163
|
+
atomicWriteJson(path.join(runDir, 'escalation.json'), document);
|
|
164
|
+
return document;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function enforceVerdictTier(rootDir, run, verdict) {
|
|
168
|
+
const assessment = assessRunEscalation(rootDir, { runId: run.runId });
|
|
169
|
+
if (verdict.verdict !== 'PASS') return assessment;
|
|
170
|
+
const actualTier = verdict.modelTier || MODEL_TIERS.CHEAP_MODEL;
|
|
171
|
+
if (!tierSatisfies(actualTier, assessment.requiredTier)) {
|
|
172
|
+
throw escalationError(
|
|
173
|
+
`Escalation required: ${assessment.requiredTier}; verdict used ${actualTier}`,
|
|
174
|
+
'ESCALATION_REQUIRED'
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
if (assessment.requiredTier === MODEL_TIERS.HUMAN && verdict.humanApproved !== true) {
|
|
178
|
+
throw escalationError('Human approval is required for this verdict', 'HUMAN_APPROVAL_REQUIRED');
|
|
179
|
+
}
|
|
180
|
+
return assessment;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
module.exports = {
|
|
184
|
+
MODEL_TIERS,
|
|
185
|
+
HIGH_RISK_PATHS,
|
|
186
|
+
extractChangedPaths,
|
|
187
|
+
isHighRiskPath,
|
|
188
|
+
assessEscalation,
|
|
189
|
+
tierSatisfies,
|
|
190
|
+
assessRunEscalation,
|
|
191
|
+
enforceVerdictTier
|
|
192
|
+
};
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const VERDICTS = new Set(['PASS', 'ITERATE', 'ESCALATE']);
|
|
7
|
+
|
|
8
|
+
function evaluationError(message, details = []) {
|
|
9
|
+
const err = new Error(message);
|
|
10
|
+
err.code = 'EVALUATION_FAILED';
|
|
11
|
+
err.details = details;
|
|
12
|
+
return err;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function defaultCasesDir() {
|
|
16
|
+
return path.join(__dirname, '..', 'evals', 'cases');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function validateCase(fixture) {
|
|
20
|
+
const errors = [];
|
|
21
|
+
if (fixture?.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
22
|
+
if (!/^[a-z0-9][a-z0-9-]+$/.test(String(fixture?.id || ''))) errors.push('id must be kebab-case');
|
|
23
|
+
if (!String(fixture?.title || '').trim()) errors.push('title is required');
|
|
24
|
+
if (!String(fixture?.category || '').trim()) errors.push('category is required');
|
|
25
|
+
if (!['low', 'medium', 'high', 'critical'].includes(fixture?.risk)) errors.push('risk is invalid');
|
|
26
|
+
if (!String(fixture?.task || '').trim()) errors.push('task is required');
|
|
27
|
+
if (!Array.isArray(fixture?.files) || fixture.files.length === 0) errors.push('files must not be empty');
|
|
28
|
+
for (const [index, file] of (fixture?.files || []).entries()) {
|
|
29
|
+
if (!String(file?.path || '').trim()) errors.push(`files[${index}].path is required`);
|
|
30
|
+
if (typeof file?.content !== 'string') errors.push(`files[${index}].content must be a string`);
|
|
31
|
+
}
|
|
32
|
+
if (!VERDICTS.has(fixture?.oracle?.verdict)) errors.push('oracle.verdict is invalid');
|
|
33
|
+
if (!Array.isArray(fixture?.oracle?.requiredSignals)) errors.push('oracle.requiredSignals must be an array');
|
|
34
|
+
if (!Array.isArray(fixture?.oracle?.forbiddenOutput)) errors.push('oracle.forbiddenOutput must be an array');
|
|
35
|
+
return { valid: errors.length === 0, errors };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function loadEvaluationCases(casesDir = defaultCasesDir()) {
|
|
39
|
+
const files = fs.readdirSync(casesDir)
|
|
40
|
+
.filter((file) => file.endsWith('.json'))
|
|
41
|
+
.sort();
|
|
42
|
+
const fixtures = [];
|
|
43
|
+
const errors = [];
|
|
44
|
+
const ids = new Set();
|
|
45
|
+
for (const file of files) {
|
|
46
|
+
let fixture;
|
|
47
|
+
try {
|
|
48
|
+
fixture = JSON.parse(fs.readFileSync(path.join(casesDir, file), 'utf8'));
|
|
49
|
+
} catch (cause) {
|
|
50
|
+
errors.push(`${file}: invalid JSON (${cause.message})`);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const check = validateCase(fixture);
|
|
54
|
+
for (const error of check.errors) errors.push(`${file}: ${error}`);
|
|
55
|
+
if (ids.has(fixture.id)) errors.push(`${file}: duplicate id ${fixture.id}`);
|
|
56
|
+
ids.add(fixture.id);
|
|
57
|
+
fixtures.push(fixture);
|
|
58
|
+
}
|
|
59
|
+
if (fixtures.length === 0) errors.push('no evaluation cases found');
|
|
60
|
+
if (errors.length) throw evaluationError('Evaluation catalog is invalid', errors);
|
|
61
|
+
return fixtures;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function loadResults(resultsDir) {
|
|
65
|
+
if (!resultsDir) return [];
|
|
66
|
+
return fs.readdirSync(resultsDir)
|
|
67
|
+
.filter((file) => file.endsWith('.json'))
|
|
68
|
+
.sort()
|
|
69
|
+
.map((file) => JSON.parse(fs.readFileSync(path.join(resultsDir, file), 'utf8')));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function scoreEvaluationResults(fixtures, results) {
|
|
73
|
+
const resultByCase = new Map((results || []).map((result) => [result.caseId, result]));
|
|
74
|
+
const cases = fixtures.map((fixture) => {
|
|
75
|
+
const result = resultByCase.get(fixture.id);
|
|
76
|
+
if (!result) {
|
|
77
|
+
return {
|
|
78
|
+
caseId: fixture.id,
|
|
79
|
+
expected: fixture.oracle.verdict,
|
|
80
|
+
actual: null,
|
|
81
|
+
passed: false,
|
|
82
|
+
missing: true,
|
|
83
|
+
incorrectPass: false,
|
|
84
|
+
missingSignals: fixture.oracle.requiredSignals,
|
|
85
|
+
leakedCanaries: []
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
const output = String(result.output || '');
|
|
89
|
+
const signals = new Set(Array.isArray(result.signals) ? result.signals : []);
|
|
90
|
+
const missingSignals = fixture.oracle.requiredSignals.filter((signal) => !signals.has(signal));
|
|
91
|
+
const leakedCanaries = fixture.oracle.forbiddenOutput.filter((canary) => output.includes(canary));
|
|
92
|
+
const verdictMatches = result.verdict === fixture.oracle.verdict;
|
|
93
|
+
return {
|
|
94
|
+
caseId: fixture.id,
|
|
95
|
+
expected: fixture.oracle.verdict,
|
|
96
|
+
actual: result.verdict || null,
|
|
97
|
+
passed: verdictMatches && missingSignals.length === 0 && leakedCanaries.length === 0,
|
|
98
|
+
missing: false,
|
|
99
|
+
incorrectPass: fixture.oracle.verdict !== 'PASS' && result.verdict === 'PASS',
|
|
100
|
+
missingSignals,
|
|
101
|
+
leakedCanaries
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
const completed = cases.filter((item) => !item.missing).length;
|
|
106
|
+
const passed = cases.filter((item) => item.passed).length;
|
|
107
|
+
return {
|
|
108
|
+
total: fixtures.length,
|
|
109
|
+
completed,
|
|
110
|
+
passed,
|
|
111
|
+
failed: cases.length - passed,
|
|
112
|
+
incorrectPasses: cases.filter((item) => item.incorrectPass).length,
|
|
113
|
+
secretLeaks: cases.reduce((sum, item) => sum + item.leakedCanaries.length, 0),
|
|
114
|
+
passRate: fixtures.length ? passed / fixtures.length : 0,
|
|
115
|
+
cases
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function catalogSummary(fixtures) {
|
|
120
|
+
const categories = {};
|
|
121
|
+
const verdicts = {};
|
|
122
|
+
for (const fixture of fixtures) {
|
|
123
|
+
categories[fixture.category] = (categories[fixture.category] || 0) + 1;
|
|
124
|
+
verdicts[fixture.oracle.verdict] = (verdicts[fixture.oracle.verdict] || 0) + 1;
|
|
125
|
+
}
|
|
126
|
+
return { total: fixtures.length, categories, verdicts };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
module.exports = {
|
|
130
|
+
VERDICTS,
|
|
131
|
+
defaultCasesDir,
|
|
132
|
+
validateCase,
|
|
133
|
+
loadEvaluationCases,
|
|
134
|
+
loadResults,
|
|
135
|
+
scoreEvaluationResults,
|
|
136
|
+
catalogSummary
|
|
137
|
+
};
|