ai-engineering-loop 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +5 -5
- package/.claude/skills/ai-engineering-loop/SKILL.md +7 -5
- package/.gemini/skills/ai-engineering-loop/SKILL.md +7 -5
- package/.grok/skills/ai-engineering-loop/SKILL.md +7 -5
- package/README.md +20 -3
- package/README.npm.md +9 -1
- package/SECURITY.md +8 -2
- package/agents/judge.md +5 -5
- package/agents/maker.md +6 -6
- package/bin/ai-engineering-loop.js +223 -13
- package/core/configuration-precedence.md +4 -4
- package/core/context-impact-assessment.md +1 -1
- package/core/definition-of-done.md +3 -3
- package/core/escalation-policy.md +2 -2
- package/core/goal-contract.md +2 -2
- package/core/iteration-policy.md +4 -4
- package/core/project-initialization.md +2 -2
- package/core/runtime-safety.md +75 -0
- package/core/verification-loop.md +1 -1
- package/lib/budget.js +220 -0
- package/lib/doctor.js +14 -1
- package/lib/gates.js +55 -1
- package/lib/package-audit.js +78 -0
- package/lib/run-state.js +47 -8
- package/lib/runtime-files.js +53 -0
- package/lib/runtime-policy.js +118 -0
- package/lib/safe-context.js +3 -0
- package/lib/sandbox.js +305 -0
- package/package.json +6 -3
- package/policies/finding-policy.md +2 -2
- package/policies/no-progress-policy.md +2 -2
- package/profiles/README.md +5 -5
- package/profiles/backend-api.md +1 -1
- package/profiles/library.md +1 -1
- package/profiles/mobile-app.md +1 -1
- package/profiles/monorepo.md +1 -1
- package/profiles/web-app.md +1 -1
- package/schemas/delivery.schema.json +2 -1
- package/schemas/report.schema.json +17 -0
|
@@ -34,10 +34,19 @@ const {
|
|
|
34
34
|
lessonsMarkdown
|
|
35
35
|
} = require('../lib/generate-workflow.js');
|
|
36
36
|
const {
|
|
37
|
+
RUN_MODES,
|
|
38
|
+
TERMINAL_STATES,
|
|
37
39
|
createOrResumeRun,
|
|
38
40
|
getCurrentRun,
|
|
39
41
|
loadRun
|
|
40
42
|
} = require('../lib/run-state.js');
|
|
43
|
+
const {
|
|
44
|
+
DEFAULT_POLICY,
|
|
45
|
+
assertModeAllowed,
|
|
46
|
+
loadPolicy,
|
|
47
|
+
normalizeMode,
|
|
48
|
+
updatePolicy
|
|
49
|
+
} = require('../lib/runtime-policy.js');
|
|
41
50
|
const {
|
|
42
51
|
applyGate
|
|
43
52
|
} = require('../lib/gates.js');
|
|
@@ -58,8 +67,23 @@ const {
|
|
|
58
67
|
const {
|
|
59
68
|
runDoctor
|
|
60
69
|
} = require('../lib/doctor.js');
|
|
70
|
+
const {
|
|
71
|
+
assertBudgetAvailable,
|
|
72
|
+
budgetStatus,
|
|
73
|
+
recordTokenUsage,
|
|
74
|
+
setKillSwitch
|
|
75
|
+
} = require('../lib/budget.js');
|
|
76
|
+
const {
|
|
77
|
+
PRIVATE_RUNTIME_GITIGNORE
|
|
78
|
+
} = require('../lib/runtime-files.js');
|
|
79
|
+
const {
|
|
80
|
+
abortSandbox,
|
|
81
|
+
captureSandbox,
|
|
82
|
+
createSandbox,
|
|
83
|
+
sandboxStatus
|
|
84
|
+
} = require('../lib/sandbox.js');
|
|
61
85
|
|
|
62
|
-
const VERSION = '1.1
|
|
86
|
+
const VERSION = '1.2.1';
|
|
63
87
|
const CWD = process.cwd();
|
|
64
88
|
const CONTEXT_DIR = path.join(CWD, '.ai-engineering-loop');
|
|
65
89
|
|
|
@@ -313,7 +337,7 @@ function generateContextFiles(rootDir, discovery, trigger = 'init', impact = 'IN
|
|
|
313
337
|
|
|
314
338
|
// 0. metadata.json (Baseline)
|
|
315
339
|
const metadataJson = {
|
|
316
|
-
contextVersion: '1.1
|
|
340
|
+
contextVersion: '1.2.1',
|
|
317
341
|
generatedAt: new Date().toISOString(),
|
|
318
342
|
repositoryRevision: currentRevision,
|
|
319
343
|
projectProfile: discovery.profile,
|
|
@@ -453,9 +477,14 @@ What becomes easier, harder, or forbidden.
|
|
|
453
477
|
|
|
454
478
|
writeContextFile(path.join(targetDir, 'workflow.md'), workflowMarkdown(), { overwrite: false });
|
|
455
479
|
writeContextFile(path.join(targetDir, 'lessons.md'), lessonsMarkdown(), { overwrite: false });
|
|
480
|
+
writeContextFile(
|
|
481
|
+
path.join(targetDir, 'runtime-policy.json'),
|
|
482
|
+
`${JSON.stringify(DEFAULT_POLICY, null, 2)}\n`,
|
|
483
|
+
{ overwrite: false }
|
|
484
|
+
);
|
|
456
485
|
writeContextFile(
|
|
457
486
|
path.join(targetDir, '.gitignore'),
|
|
458
|
-
|
|
487
|
+
PRIVATE_RUNTIME_GITIGNORE,
|
|
459
488
|
{ overwrite: false }
|
|
460
489
|
);
|
|
461
490
|
}
|
|
@@ -757,29 +786,54 @@ function handleRun() {
|
|
|
757
786
|
|
|
758
787
|
const runArgs = process.argv.slice(3);
|
|
759
788
|
const forceNew = runArgs.includes('--new');
|
|
760
|
-
const
|
|
789
|
+
const modeArg = argValue('--mode');
|
|
790
|
+
const taskParts = [];
|
|
791
|
+
for (let index = 0; index < runArgs.length; index++) {
|
|
792
|
+
if (runArgs[index] === '--new') continue;
|
|
793
|
+
if (runArgs[index] === '--mode') {
|
|
794
|
+
index += 1;
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
taskParts.push(runArgs[index]);
|
|
798
|
+
}
|
|
799
|
+
const task = taskParts.join(' ').trim();
|
|
761
800
|
let runResult;
|
|
762
801
|
try {
|
|
763
|
-
|
|
802
|
+
const current = getCurrentRun(CWD);
|
|
803
|
+
const policy = loadPolicy(CWD);
|
|
804
|
+
const startsNewRun = forceNew || !current || TERMINAL_STATES.has(current.state);
|
|
805
|
+
const mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
|
|
806
|
+
if (mode) assertModeAllowed(CWD, mode);
|
|
807
|
+
runResult = createOrResumeRun(CWD, { task, forceNew, mode });
|
|
808
|
+
assertBudgetAvailable(CWD, { runId: runResult.run.runId });
|
|
764
809
|
} catch (err) {
|
|
765
810
|
log.error(err.message);
|
|
766
811
|
process.exit(1);
|
|
767
812
|
}
|
|
768
813
|
console.log(`- Run: ${runResult.run.runId} (${runResult.resumed ? 'resumed' : 'created'})`);
|
|
769
814
|
console.log(`- State: ${runResult.run.state}, iteration ${runResult.run.iteration}`);
|
|
815
|
+
console.log(`- Mode: ${runResult.run.mode || RUN_MODES.ASSISTED}`);
|
|
770
816
|
|
|
771
817
|
const grok = detectGrokHost();
|
|
772
818
|
|
|
773
819
|
console.log('\n------------------------------------------------------------');
|
|
774
820
|
log.bold('AI Agent Ready:');
|
|
775
821
|
console.log('1. Grill if needed, then freeze Goal Contract (core/grill-policy.md, core/goal-contract.md)');
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
822
|
+
if (runResult.run.mode === RUN_MODES.REPORT_ONLY) {
|
|
823
|
+
console.log('2. Analyze evidence without changing repository files');
|
|
824
|
+
console.log('3. Write report.json and finish with `ai-engineering-loop gate report`');
|
|
825
|
+
} else {
|
|
826
|
+
console.log('2. Root Cause Analysis (core/root-cause-analysis.md) & Plan');
|
|
827
|
+
console.log('3. Maker TDD at named seams (policies/tdd-policy.md)');
|
|
828
|
+
console.log('4. Run Deterministic Verification');
|
|
829
|
+
console.log('5. Execute Devil\'s Advocate Adversarial Review');
|
|
830
|
+
console.log('6. Judge Agent evaluates DoD and issues PASS verdict');
|
|
831
|
+
console.log('7. Context Impact Assessment (NONE / TARGETED / MAJOR)');
|
|
832
|
+
console.log('8. Delivery Adapter creates MR/PR');
|
|
833
|
+
if (runResult.run.mode === RUN_MODES.ASSISTED) {
|
|
834
|
+
console.log('9. Human approval is required before the delivery gate');
|
|
835
|
+
}
|
|
836
|
+
}
|
|
783
837
|
|
|
784
838
|
if (grok && grok.host === 'grok-cli') {
|
|
785
839
|
console.log('------------------------------------------------------------');
|
|
@@ -826,6 +880,141 @@ function argValue(name) {
|
|
|
826
880
|
return index === -1 ? null : process.argv[index + 1] || null;
|
|
827
881
|
}
|
|
828
882
|
|
|
883
|
+
function parseBooleanArg(name, value) {
|
|
884
|
+
if (!['true', 'false'].includes(String(value))) {
|
|
885
|
+
const err = new Error(`${name} must be true or false`);
|
|
886
|
+
err.code = 'INVALID_POLICY_VALUE';
|
|
887
|
+
throw err;
|
|
888
|
+
}
|
|
889
|
+
return value === 'true';
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function parsePositiveIntegerArg(name, value) {
|
|
893
|
+
const parsed = Number(value);
|
|
894
|
+
if (!Number.isSafeInteger(parsed) || parsed <= 0) {
|
|
895
|
+
const err = new Error(`${name} must be a positive integer`);
|
|
896
|
+
err.code = 'INVALID_POLICY_VALUE';
|
|
897
|
+
throw err;
|
|
898
|
+
}
|
|
899
|
+
return parsed;
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
function parseNonNegativeIntegerArg(name, value) {
|
|
903
|
+
if (value == null) {
|
|
904
|
+
const err = new Error(`${name} is required`);
|
|
905
|
+
err.code = 'INVALID_USAGE';
|
|
906
|
+
throw err;
|
|
907
|
+
}
|
|
908
|
+
const parsed = Number(value);
|
|
909
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
|
910
|
+
const err = new Error(`${name} must be a non-negative integer`);
|
|
911
|
+
err.code = 'INVALID_USAGE';
|
|
912
|
+
throw err;
|
|
913
|
+
}
|
|
914
|
+
return parsed;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
function handlePolicy() {
|
|
918
|
+
const action = process.argv[3] || 'show';
|
|
919
|
+
const json = process.argv.includes('--json');
|
|
920
|
+
try {
|
|
921
|
+
let policy;
|
|
922
|
+
if (action === 'show') {
|
|
923
|
+
policy = loadPolicy(CWD);
|
|
924
|
+
} else if (action === 'set') {
|
|
925
|
+
const changes = {};
|
|
926
|
+
const mode = argValue('--default-mode');
|
|
927
|
+
const allowUnattended = argValue('--allow-unattended');
|
|
928
|
+
const requireSandbox = argValue('--require-sandbox');
|
|
929
|
+
const perRunLimit = argValue('--per-run-token-limit');
|
|
930
|
+
const dailyLimit = argValue('--daily-token-limit');
|
|
931
|
+
if (mode != null) changes.defaultMode = normalizeMode(mode);
|
|
932
|
+
if (allowUnattended != null) {
|
|
933
|
+
changes.allowUnattended = parseBooleanArg('--allow-unattended', allowUnattended);
|
|
934
|
+
}
|
|
935
|
+
if (requireSandbox != null) {
|
|
936
|
+
changes.requireSandboxForUnattended = parseBooleanArg('--require-sandbox', requireSandbox);
|
|
937
|
+
}
|
|
938
|
+
if (perRunLimit != null) {
|
|
939
|
+
changes.perRunTokenLimit = parsePositiveIntegerArg('--per-run-token-limit', perRunLimit);
|
|
940
|
+
}
|
|
941
|
+
if (dailyLimit != null) {
|
|
942
|
+
changes.dailyTokenLimit = parsePositiveIntegerArg('--daily-token-limit', dailyLimit);
|
|
943
|
+
}
|
|
944
|
+
if (Object.keys(changes).length === 0) {
|
|
945
|
+
throw new Error('No policy changes provided');
|
|
946
|
+
}
|
|
947
|
+
policy = updatePolicy(CWD, changes);
|
|
948
|
+
} else {
|
|
949
|
+
throw new Error('Unknown policy action. Use show|set');
|
|
950
|
+
}
|
|
951
|
+
if (json) console.log(JSON.stringify(policy));
|
|
952
|
+
else {
|
|
953
|
+
log.success(action === 'set' ? '✓ Runtime policy updated' : 'Runtime policy');
|
|
954
|
+
console.log(JSON.stringify(policy, null, 2));
|
|
955
|
+
}
|
|
956
|
+
} catch (err) {
|
|
957
|
+
if (json) console.log(JSON.stringify({ ok: false, code: err.code || 'POLICY_ERROR', error: err.message }));
|
|
958
|
+
else log.error(err.message);
|
|
959
|
+
process.exit(1);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
function handleBudget() {
|
|
964
|
+
const action = process.argv[3] || 'status';
|
|
965
|
+
const json = process.argv.includes('--json');
|
|
966
|
+
const runId = argValue('--run');
|
|
967
|
+
try {
|
|
968
|
+
let result;
|
|
969
|
+
if (action === 'status') {
|
|
970
|
+
result = budgetStatus(CWD, { runId });
|
|
971
|
+
} else if (action === 'record') {
|
|
972
|
+
result = recordTokenUsage(CWD, {
|
|
973
|
+
runId,
|
|
974
|
+
inputTokens: parseNonNegativeIntegerArg('--input', argValue('--input')),
|
|
975
|
+
outputTokens: parseNonNegativeIntegerArg('--output', argValue('--output')),
|
|
976
|
+
model: argValue('--model')
|
|
977
|
+
});
|
|
978
|
+
} else if (action === 'pause' || action === 'resume') {
|
|
979
|
+
result = setKillSwitch(CWD, action === 'pause');
|
|
980
|
+
} else {
|
|
981
|
+
throw new Error('Unknown budget action. Use status|record|pause|resume');
|
|
982
|
+
}
|
|
983
|
+
if (json) console.log(JSON.stringify(result));
|
|
984
|
+
else {
|
|
985
|
+
log.success(action === 'record' ? '✓ Token usage recorded' : `Token budget: ${action}`);
|
|
986
|
+
console.log(JSON.stringify(result, null, 2));
|
|
987
|
+
}
|
|
988
|
+
} catch (err) {
|
|
989
|
+
if (json) console.log(JSON.stringify({ ok: false, code: err.code || 'BUDGET_ERROR', error: err.message }));
|
|
990
|
+
else log.error(err.message);
|
|
991
|
+
process.exit(1);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
function handleSandbox() {
|
|
996
|
+
const action = process.argv[3] || 'status';
|
|
997
|
+
const json = process.argv.includes('--json');
|
|
998
|
+
const runId = argValue('--run');
|
|
999
|
+
try {
|
|
1000
|
+
let result;
|
|
1001
|
+
if (action === 'create') result = createSandbox(CWD, { runId });
|
|
1002
|
+
else if (action === 'capture') result = captureSandbox(CWD, { runId });
|
|
1003
|
+
else if (action === 'abort') result = abortSandbox(CWD, { runId });
|
|
1004
|
+
else if (action === 'status') result = sandboxStatus(CWD, { runId });
|
|
1005
|
+
else throw new Error('Unknown sandbox action. Use create|capture|abort|status');
|
|
1006
|
+
if (json) console.log(JSON.stringify(result));
|
|
1007
|
+
else {
|
|
1008
|
+
log.success(`✓ Sandbox ${action}`);
|
|
1009
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1010
|
+
}
|
|
1011
|
+
} catch (err) {
|
|
1012
|
+
if (json) console.log(JSON.stringify({ ok: false, code: err.code || 'SANDBOX_ERROR', error: err.message }));
|
|
1013
|
+
else log.error(err.message);
|
|
1014
|
+
process.exit(1);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
829
1018
|
function handleGate() {
|
|
830
1019
|
const gate = process.argv[3];
|
|
831
1020
|
const runId = argValue('--run');
|
|
@@ -1006,13 +1195,25 @@ Commands:
|
|
|
1006
1195
|
refresh Reconcile drifted context against repository non-destructively
|
|
1007
1196
|
run [task] Start or resume a stateful engineering run, sync hosts, and instruct the agent
|
|
1008
1197
|
--new start a new run even when another run is active
|
|
1198
|
+
--mode report-only|assisted|unattended
|
|
1009
1199
|
state Inspect the current run ledger
|
|
1010
1200
|
--run <id> target a non-current run
|
|
1011
1201
|
--json print the complete machine-readable ledger
|
|
1012
1202
|
gate <name> Validate an artifact and advance the current run
|
|
1013
|
-
goal | maker | verification | review | judge | delivery
|
|
1203
|
+
goal | report | maker | verification | review | judge | delivery
|
|
1014
1204
|
--run <id> target a non-current run
|
|
1015
1205
|
--json print machine-readable gate output
|
|
1206
|
+
policy Inspect or update fail-closed runtime controls
|
|
1207
|
+
show [--json]
|
|
1208
|
+
set [--default-mode <mode>] [--allow-unattended true|false]
|
|
1209
|
+
[--require-sandbox true|false]
|
|
1210
|
+
[--per-run-token-limit <n>] [--daily-token-limit <n>]
|
|
1211
|
+
budget Enforce actual provider-reported token usage and emergency pause
|
|
1212
|
+
status [--run <id>] [--json]
|
|
1213
|
+
record --input <n> --output <n> --model <id> [--run <id>]
|
|
1214
|
+
pause | resume
|
|
1215
|
+
sandbox Isolate Maker changes in a locked disposable Git worktree
|
|
1216
|
+
create | capture | abort | status [--run <id>] [--json]
|
|
1016
1217
|
context <stage> <files...>
|
|
1017
1218
|
Build a bounded, redacted context pack for maker, devil-advocate, or judge
|
|
1018
1219
|
--run <id> target a non-current run
|
|
@@ -1066,6 +1267,15 @@ switch (command) {
|
|
|
1066
1267
|
case 'gate':
|
|
1067
1268
|
handleGate();
|
|
1068
1269
|
break;
|
|
1270
|
+
case 'policy':
|
|
1271
|
+
handlePolicy();
|
|
1272
|
+
break;
|
|
1273
|
+
case 'budget':
|
|
1274
|
+
handleBudget();
|
|
1275
|
+
break;
|
|
1276
|
+
case 'sandbox':
|
|
1277
|
+
handleSandbox();
|
|
1278
|
+
break;
|
|
1069
1279
|
case 'context':
|
|
1070
1280
|
handleContext();
|
|
1071
1281
|
break;
|
|
@@ -35,10 +35,10 @@ flowchart TD
|
|
|
35
35
|
### Layer 2: Engineering Core (`core/`)
|
|
36
36
|
- **Authority**: AI Engineering Operating System specification.
|
|
37
37
|
- **Scope**: Mandatory engineering invariants:
|
|
38
|
-
- No code changes without a formalized [Goal Contract](
|
|
38
|
+
- No code changes without a formalized [Goal Contract](./goal-contract.md).
|
|
39
39
|
- Deterministic checks must pass 100% before adversarial review.
|
|
40
40
|
- Review findings require reproducible evidence (Level 1–3) to be valid.
|
|
41
|
-
- Only the [Judge Agent](
|
|
41
|
+
- Only the [Judge Agent](../agents/judge.md) can issue a `PASS` verdict.
|
|
42
42
|
- **Overridability**: Invariant. Downstream layers cannot skip verification or eliminate agent roles.
|
|
43
43
|
|
|
44
44
|
### Layer 3: Project Type Profile (`profiles/`)
|
|
@@ -55,7 +55,7 @@ flowchart TD
|
|
|
55
55
|
- Layer definitions in `architecture.md`.
|
|
56
56
|
- Architectural constraints and forbidden patterns in `conventions.md`.
|
|
57
57
|
- Configured delivery adapter (DOT, GitHub, GitLab) in `adapter.md`.
|
|
58
|
-
- **Auto-Discovery**: Automatically generated if missing via [Project Initialization](
|
|
58
|
+
- **Auto-Discovery**: Automatically generated if missing via [Project Initialization](./project-initialization.md).
|
|
59
59
|
- **Overridability**: Overrides Layer 3 defaults for this specific codebase.
|
|
60
60
|
|
|
61
61
|
### Layer 5: Task Contract
|
|
@@ -99,4 +99,4 @@ def resolve_config_key(key: str, context: ExecutionContext) -> Any:
|
|
|
99
99
|
- *Refinement (Allowed)*: Profile says "Run unit tests"; Repo config specifies `pnpm run test:unit`. $\rightarrow$ Valid refinement.
|
|
100
100
|
- *Contradiction (Escalate)*: Core requires deterministic verification; Task contract requests skipping tests to merge faster. $\rightarrow$ **Forbidden**. The engine rejects the override and flags a policy violation.
|
|
101
101
|
2. **Unresolvable Contradictions**:
|
|
102
|
-
- If repository configuration directly contradicts an established platform invariant (e.g. a command requires root sudo or drops protected tables), the system halts and triggers [Human Escalation](
|
|
102
|
+
- If repository configuration directly contradicts an established platform invariant (e.g. a command requires root sudo or drops protected tables), the system halts and triggers [Human Escalation](./escalation-policy.md).
|
|
@@ -92,7 +92,7 @@ flowchart TD
|
|
|
92
92
|
|
|
93
93
|
## 4. Devil's Advocate & Judge Drift Interaction
|
|
94
94
|
|
|
95
|
-
During task review, the [Devil's Advocate](
|
|
95
|
+
During task review, the [Devil's Advocate](../agents/devil-advocate.md) may discover that the codebase has drifted from the documented architecture:
|
|
96
96
|
|
|
97
97
|
> *Example Review Finding*: "The implementation assumes `user-service` connects directly to Postgres, but the codebase has migrated to an asynchronous event queue in `src/events/`."
|
|
98
98
|
|
|
@@ -26,7 +26,7 @@ flowchart LR
|
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
### Pillar 1: Contract Satisfaction
|
|
29
|
-
- 100% of Acceptance Criteria defined in the [Goal Contract](
|
|
29
|
+
- 100% of Acceptance Criteria defined in the [Goal Contract](./goal-contract.md) are demonstrably met.
|
|
30
30
|
- Zero out-of-scope files or unauthorized modules were modified.
|
|
31
31
|
- All technical constraints (e.g. backward compatibility, no unapproved dependencies) are preserved.
|
|
32
32
|
|
|
@@ -45,12 +45,12 @@ flowchart LR
|
|
|
45
45
|
- **Null & Boundary Safety**: Explicit handling of `null`, `undefined`, empty collections, and error paths.
|
|
46
46
|
|
|
47
47
|
### Pillar 4: Adversarial Consensus
|
|
48
|
-
- Independent [Devil's Advocate Review](
|
|
48
|
+
- Independent [Devil's Advocate Review](../agents/devil-advocate.md) has been executed across all 6 core review domains.
|
|
49
49
|
- **Zero Unresolved Blocking Findings**: No open `SEV-1 (Critical)` or `SEV-2 (High)` findings.
|
|
50
50
|
- **Evidence-Based Triage**: Every raised finding has been formally triaged with reproducible evidence as `VALID` (and resolved in code) or `INVALID` (with technical proof of why it is a false positive).
|
|
51
51
|
|
|
52
52
|
### Pillar 5: Judge Certified PASS
|
|
53
|
-
- The [Judge Agent](
|
|
53
|
+
- The [Judge Agent](../agents/judge.md) has evaluated the complete execution trace, verified the evidence, confirmed no-progress limits were not violated, and issued a signed `PASS` verdict.
|
|
54
54
|
|
|
55
55
|
---
|
|
56
56
|
|
|
@@ -31,7 +31,7 @@ flowchart TD
|
|
|
31
31
|
- **Rationale**: Prolonged retries indicate either an incorrect fundamental strategy or a hidden architectural obstacle.
|
|
32
32
|
|
|
33
33
|
### Trigger 2: No-Progress / Stagnation Detected
|
|
34
|
-
- **Condition**: Two consecutive iterations produce identical or semantically equivalent finding signatures without demonstrable code convergence (see [No-Progress Policy](
|
|
34
|
+
- **Condition**: Two consecutive iterations produce identical or semantically equivalent finding signatures without demonstrable code convergence (see [No-Progress Policy](../policies/no-progress-policy.md)).
|
|
35
35
|
- **Rationale**: Prevents thrashing where the agent modifies code without resolving the underlying flaw.
|
|
36
36
|
|
|
37
37
|
### Trigger 3: Contradictory or Inconsistent Acceptance Criteria
|
|
@@ -93,7 +93,7 @@ Instead, it must render an **Actionable Escalation Report** following this struc
|
|
|
93
93
|
|
|
94
94
|
## 4. Resumption Protocol
|
|
95
95
|
|
|
96
|
-
Once the human engineer provides guidance or amends the [Goal Contract](
|
|
96
|
+
Once the human engineer provides guidance or amends the [Goal Contract](./goal-contract.md):
|
|
97
97
|
1. The iteration counter is reset: $K \leftarrow 1$.
|
|
98
98
|
2. The agent incorporates the human's decision into the Goal Contract constraints.
|
|
99
99
|
3. The loop resumes at the **Maker Agent** phase with clean validation.
|
package/core/goal-contract.md
CHANGED
|
@@ -11,7 +11,7 @@ The Goal Contract establishes:
|
|
|
11
11
|
- **Where** the boundaries are set (preventing scope creep).
|
|
12
12
|
- **When** the work is strictly considered complete.
|
|
13
13
|
|
|
14
|
-
Alignment before freeze follows [Grill Policy](
|
|
14
|
+
Alignment before freeze follows [Grill Policy](./grill-policy.md). Terms come from `.ai-engineering-loop/glossary.md`.
|
|
15
15
|
|
|
16
16
|
---
|
|
17
17
|
|
|
@@ -77,7 +77,7 @@ Each AC is one row that can **fail**. Happy path alone is not a contract (`polic
|
|
|
77
77
|
|
|
78
78
|
1. **Pre-Implementation Freezing**:
|
|
79
79
|
- The Goal Contract is authored and frozen *before* any production code edits.
|
|
80
|
-
- If the task is ambiguous and a human is present, run the [Grill Policy](
|
|
80
|
+
- If the task is ambiguous and a human is present, run the [Grill Policy](./grill-policy.md) until the design-tree frontier is empty, then freeze.
|
|
81
81
|
- If the task is unambiguous, waived, or headless with testable AC, skip grill and freeze immediately.
|
|
82
82
|
2. **Immutability During Iteration**:
|
|
83
83
|
- Neither the Maker Agent nor the Devil's Advocate Agent may alter Acceptance Criteria during an iteration loop to make tests pass or bypass critique.
|
package/core/iteration-policy.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## 1. Overview & Objectives
|
|
4
4
|
|
|
5
|
-
The **Iteration Policy** governs autonomous retry loops within the AI Engineering Loop. When deterministic tests fail or the [Devil's Advocate](
|
|
5
|
+
The **Iteration Policy** governs autonomous retry loops within the AI Engineering Loop. When deterministic tests fail or the [Devil's Advocate](../agents/devil-advocate.md) identifies valid blocking findings, the system initiates an **autonomous iteration cycle**.
|
|
6
6
|
|
|
7
7
|
The objective of this policy is to enable **reliable self-correction** while strictly guarding against infinite loops, cognitive drift, and resource exhaustion.
|
|
8
8
|
|
|
@@ -54,14 +54,14 @@ stateDiagram-v2
|
|
|
54
54
|
- Increments iteration counter $K \leftarrow K + 1$.
|
|
55
55
|
- Verifies $K \le \text{MAX\_ITERATIONS}$. If $K > \text{MAX\_ITERATIONS}$, immediately triggers `ESCALATE`.
|
|
56
56
|
2. **Maker Refinement**:
|
|
57
|
-
- The [Maker Agent](
|
|
57
|
+
- The [Maker Agent](../agents/maker.md) receives the prior iteration's triaged findings and Judge directives.
|
|
58
58
|
- Maker applies targeted code fixes and updates test suites.
|
|
59
59
|
3. **Deterministic Re-verification**:
|
|
60
60
|
- All tests, typecheck, lint, and build checks are re-executed from scratch.
|
|
61
61
|
4. **Adversarial Differential Review**:
|
|
62
|
-
- The [Devil's Advocate](
|
|
62
|
+
- The [Devil's Advocate](../agents/devil-advocate.md) evaluates the updated diff against previously raised findings and checks for new regressions.
|
|
63
63
|
5. **Judge Evaluation & Progression**:
|
|
64
|
-
- The [Judge Agent](
|
|
64
|
+
- The [Judge Agent](../agents/judge.md) compares findings from Iteration $K$ with Iteration $K-1$.
|
|
65
65
|
- If progress is verified and all criteria met $\rightarrow$ `PASS`.
|
|
66
66
|
- If new valid findings emerge but progress is demonstrated $\rightarrow$ `ITERATE` (if $K < \text{MAX\_ITERATIONS}$).
|
|
67
67
|
- If repeated findings or no progress detected $\rightarrow$ `ESCALATE`.
|
|
@@ -47,9 +47,9 @@ flowchart TD
|
|
|
47
47
|
- **Command**: `npx ai-engineering-loop run` (or `/ai-engineering-loop [task]`).
|
|
48
48
|
- **Boundaries**:
|
|
49
49
|
- The Agent reads `.ai-engineering-loop/` (triggering auto-init if missing).
|
|
50
|
-
- Formulates the [Goal Contract](
|
|
50
|
+
- Formulates the [Goal Contract](./goal-contract.md).
|
|
51
51
|
- Performs Root Cause Analysis (RCA) and Implementation Planning.
|
|
52
|
-
- Executes surgical diffs, runs deterministic test suites, conducts independent adversarial review via the [Devil's Advocate](
|
|
52
|
+
- Executes surgical diffs, runs deterministic test suites, conducts independent adversarial review via the [Devil's Advocate](../agents/devil-advocate.md), and obtains certification from the [Judge Agent](../agents/judge.md).
|
|
53
53
|
|
|
54
54
|
---
|
|
55
55
|
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Runtime Safety Modes
|
|
2
|
+
|
|
3
|
+
AI Engineering Loop separates analysis, human-supervised delivery, and unattended execution. The mode is stored in the run ledger and cannot change during a run.
|
|
4
|
+
|
|
5
|
+
## Modes
|
|
6
|
+
|
|
7
|
+
| Mode | Repository mutation | Delivery | Intended use |
|
|
8
|
+
|---|---|---|---|
|
|
9
|
+
| `REPORT_ONLY` | Forbidden | Ends at `gate report` | Audits, recommendations, and first-time evaluation |
|
|
10
|
+
| `ASSISTED` | Allowed after the Goal Contract gate | `delivery.json` must contain `"humanApproved": true` | Default production workflow |
|
|
11
|
+
| `UNATTENDED` | Allowed only when explicitly enabled; Maker uses a disposable locked Git worktree by default | Deterministic gates still apply | Trusted automation with budget monitoring |
|
|
12
|
+
|
|
13
|
+
`UNATTENDED` is disabled by default. Enable it deliberately:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npx ai-engineering-loop policy set --allow-unattended true
|
|
17
|
+
npx ai-engineering-loop run --mode unattended "task"
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The repository policy lives at `.ai-engineering-loop/runtime-policy.json`. Invalid policy fails closed. Do not change a policy merely because an agent asks to bypass a gate.
|
|
21
|
+
|
|
22
|
+
## Token budget and emergency stop
|
|
23
|
+
|
|
24
|
+
Check the budget before every model call and record provider-reported usage immediately afterward:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npx ai-engineering-loop budget status --run <run-id>
|
|
28
|
+
npx ai-engineering-loop budget record --run <run-id> \
|
|
29
|
+
--input <provider-input-tokens> \
|
|
30
|
+
--output <provider-output-tokens> \
|
|
31
|
+
--model <provider-model-id>
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Never convert byte estimates into "actual" usage. If the host does not expose provider usage, `UNATTENDED` must stop and escalate to a human. In `ASSISTED`, disclose that accounting is unavailable and require the human to decide whether to continue.
|
|
35
|
+
|
|
36
|
+
The per-run limit spans UTC days. The daily limit resets on the next UTC day. Ledgers are append-only JSONL under `.ai-engineering-loop/usage/` and are ignored by Git. A malformed ledger blocks execution rather than undercounting.
|
|
37
|
+
|
|
38
|
+
Emergency controls:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx ai-engineering-loop budget pause
|
|
42
|
+
npx ai-engineering-loop budget resume
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`pause` sets the persistent kill switch. It blocks new run and context/model boundaries, while `budget record` remains available so an in-flight response can still be accounted.
|
|
46
|
+
|
|
47
|
+
## Unattended Maker isolation
|
|
48
|
+
|
|
49
|
+
After `gate goal`, create the worktree:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
npx ai-engineering-loop sandbox create --run <run-id>
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Run Maker with its working directory set to the returned `worktreePath`. Do not let Maker edit the parent checkout. When Maker is done:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
npx ai-engineering-loop sandbox capture --run <run-id>
|
|
59
|
+
npx ai-engineering-loop gate maker --run <run-id>
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Capture writes `diff.patch` plus hash-bound `sandbox-evidence.json`, removes the disposable worktree, and releases the repository-wide Maker lock. The unattended Maker gate rejects a manually supplied diff without matching evidence. Use `sandbox abort` to discard an interrupted Maker worktree.
|
|
63
|
+
|
|
64
|
+
> [!IMPORTANT]
|
|
65
|
+
> A Git worktree is a filesystem and concurrency boundary, not an operating-system sandbox. It does not restrict network access, child processes, credentials inherited by the host, or writes outside the worktree. Run truly untrusted agents in a container or VM with a minimal environment and restricted network.
|
|
66
|
+
|
|
67
|
+
## Host invariant
|
|
68
|
+
|
|
69
|
+
Every supported host follows the same order:
|
|
70
|
+
|
|
71
|
+
1. Read run mode and budget status.
|
|
72
|
+
2. Freeze and gate the Goal Contract.
|
|
73
|
+
3. End with a report in `REPORT_ONLY`, require human delivery approval in `ASSISTED`, or isolate Maker in `UNATTENDED`.
|
|
74
|
+
4. Before each model/subagent call, check budget; after it returns, record actual usage.
|
|
75
|
+
5. Keep context packs, usage ledgers, worktrees, locks, raw logs, and credentials out of commits and remote payloads.
|
|
@@ -6,7 +6,7 @@ The Verification Loop is the deterministic machine gate of the AI Engineering Lo
|
|
|
6
6
|
|
|
7
7
|
> **"Code cannot enter Devil's Advocate review until it achieves 100% green machine verification backed by explicit, verifiable execution evidence."**
|
|
8
8
|
|
|
9
|
-
Maker reaches that green via [TDD Policy](
|
|
9
|
+
Maker reaches that green via [TDD Policy](../policies/tdd-policy.md): red at a named seam, then green. "We did TDD" without logs is not a PASS.
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|