ai-engineering-loop 1.1.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/workflows/ai-engineering-loop.md +5 -5
- package/.claude/skills/ai-engineering-loop/SKILL.md +7 -5
- package/.gemini/skills/ai-engineering-loop/SKILL.md +7 -5
- package/.grok/skills/ai-engineering-loop/SKILL.md +7 -5
- package/README.md +56 -6
- package/README.npm.md +9 -1
- package/SECURITY.md +8 -2
- package/agents/judge.md +5 -5
- package/agents/maker.md +6 -6
- package/agents/recipe-builder.md +28 -0
- package/bin/ai-engineering-loop.js +521 -16
- package/core/configuration-precedence.md +4 -4
- package/core/context-impact-assessment.md +1 -1
- package/core/controlled-workflow-runtime.md +77 -0
- package/core/definition-of-done.md +3 -3
- package/core/escalation-policy.md +2 -2
- package/core/goal-contract.md +2 -2
- package/core/iteration-policy.md +4 -4
- package/core/project-initialization.md +2 -2
- package/core/runtime-safety.md +75 -0
- package/core/verification-loop.md +1 -1
- package/core/workflow-recipes.md +140 -0
- package/lib/budget.js +258 -0
- package/lib/doctor.js +38 -2
- package/lib/gates.js +55 -1
- package/lib/node-registry.js +155 -0
- package/lib/package-audit.js +78 -0
- package/lib/recipe-builder.js +200 -0
- package/lib/recipe.js +495 -0
- package/lib/run-state.js +94 -8
- package/lib/runtime-files.js +54 -0
- package/lib/runtime-policy.js +118 -0
- package/lib/safe-context.js +3 -0
- package/lib/sandbox.js +305 -0
- package/lib/workflow-runtime.js +692 -0
- package/package.json +7 -3
- package/policies/finding-policy.md +2 -2
- package/policies/no-progress-policy.md +2 -2
- package/profiles/README.md +5 -5
- package/profiles/backend-api.md +1 -1
- package/profiles/library.md +1 -1
- package/profiles/mobile-app.md +1 -1
- package/profiles/monorepo.md +1 -1
- package/profiles/web-app.md +1 -1
- package/recipes/audit.json +32 -0
- package/recipes/bugfix.json +63 -0
- package/recipes/default.json +49 -0
- package/recipes/docs-light.json +63 -0
- package/recipes/high-risk.json +69 -0
- package/recipes/refactor.json +63 -0
- package/schemas/compiled-plan.schema.json +19 -0
- package/schemas/delivery.schema.json +2 -1
- package/schemas/node-definition.schema.json +33 -0
- package/schemas/recipe.schema.json +26 -0
- package/schemas/report.schema.json +17 -0
- package/schemas/workflow-event.schema.json +28 -0
- package/schemas/workflow-state.schema.json +29 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Controlled Workflow Runtime
|
|
2
|
+
|
|
3
|
+
The controlled runtime binds a validated recipe to one run without replacing the existing artifact gates. It is opt-in through `run --recipe`; runs started without that option retain the legacy behavior.
|
|
4
|
+
|
|
5
|
+
## Immutable run binding
|
|
6
|
+
|
|
7
|
+
At run creation the CLI compiles the selected recipe and writes private runtime files under `.ai-engineering-loop/runs/<run-id>/`:
|
|
8
|
+
|
|
9
|
+
- `plan.json`: canonical compiled DAG with source and graph hashes
|
|
10
|
+
- `workflow-state.json`: reconstructable scheduler cache
|
|
11
|
+
- `events.jsonl`: append-only, hash-chained transition log
|
|
12
|
+
|
|
13
|
+
The run ledger stores the recipe ID, recipe version, source hash, and graph hash. Resume rejects a different graph. Every load recalculates the plan hash and verifies the full event chain. The state cache is reconstructed from events, so a stale or edited cache cannot grant a node permission to run.
|
|
14
|
+
|
|
15
|
+
## Scheduler rules
|
|
16
|
+
|
|
17
|
+
Node states are:
|
|
18
|
+
|
|
19
|
+
`PENDING → READY → RUNNING → PASSED`
|
|
20
|
+
|
|
21
|
+
Additional outcomes are `FAILED`, `SKIPPED`, and `BLOCKED`.
|
|
22
|
+
|
|
23
|
+
- A node becomes `READY` only after every dependency is `PASSED` or conditionally `SKIPPED`.
|
|
24
|
+
- A failed dependency makes descendants `BLOCKED`.
|
|
25
|
+
- `retry` returns a failed or interrupted custom node to `READY` and recalculates descendants.
|
|
26
|
+
- Structured conditions read only allowlisted run metadata.
|
|
27
|
+
- Legacy gates execute atomically under the workflow lock and mark their corresponding node passed only after existing artifact validation succeeds.
|
|
28
|
+
- Judge `ITERATE` resets Maker and its descendants while preserving upstream analysis.
|
|
29
|
+
- Judge `ESCALATE` blocks unfinished nodes.
|
|
30
|
+
|
|
31
|
+
## Commands
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# Create an immutable recipe-bound run
|
|
35
|
+
ai-engineering-loop run --recipe bugfix --mode assisted "repair retry race"
|
|
36
|
+
|
|
37
|
+
# Inspect scheduler state
|
|
38
|
+
ai-engineering-loop node status --json
|
|
39
|
+
|
|
40
|
+
# Custom agent lifecycle
|
|
41
|
+
ai-engineering-loop node start root-cause
|
|
42
|
+
ai-engineering-loop budget status --node root-cause --estimate 4000
|
|
43
|
+
ai-engineering-loop budget record --node root-cause \
|
|
44
|
+
--input 3000 --output 700 --model provider/model-id
|
|
45
|
+
ai-engineering-loop node complete root-cause \
|
|
46
|
+
--artifact .ai-engineering-loop/runs/<run-id>/root-cause.json
|
|
47
|
+
|
|
48
|
+
# Failure and crash recovery
|
|
49
|
+
ai-engineering-loop node fail root-cause --reason provider-timeout
|
|
50
|
+
ai-engineering-loop node retry root-cause
|
|
51
|
+
|
|
52
|
+
# Explicit human checkpoint
|
|
53
|
+
ai-engineering-loop node approve delivery-approval --yes --by account-id
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Existing gate commands remain authoritative for built-in stages:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
ai-engineering-loop gate goal
|
|
60
|
+
ai-engineering-loop gate maker
|
|
61
|
+
ai-engineering-loop gate verification
|
|
62
|
+
ai-engineering-loop gate review
|
|
63
|
+
ai-engineering-loop gate judge
|
|
64
|
+
ai-engineering-loop gate delivery
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Artifact and token boundaries
|
|
68
|
+
|
|
69
|
+
Agent completion requires a regular JSON file inside the repository, rejects symlink escape, binds `runId`, validates the declared repository-relative schema, and records the artifact hash. Artifact content is never copied into the event log.
|
|
70
|
+
|
|
71
|
+
Provider-reported usage can be attributed to a node. `budget status --node --estimate` must be called before a model request; it combines kill switch, daily, run, node-state, and node token limits. `budget record --node` records actual provider usage after the response.
|
|
72
|
+
|
|
73
|
+
## Intentionally disabled
|
|
74
|
+
|
|
75
|
+
The runtime does not execute `command` nodes. Command nodes are conservatively classified as repository mutation, rejected in `REPORT_ONLY`, and fail with `COMMAND_EXECUTION_DISABLED`. Enabling them later requires a separate sandboxed executor with executable allowlists, environment filtering, bounded output, cancellation, and auditable process isolation.
|
|
76
|
+
|
|
77
|
+
The CLI also does not invoke a model itself. Host agents perform model work, record real token usage, and submit structured artifacts through the runtime boundary.
|
|
@@ -26,7 +26,7 @@ flowchart LR
|
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
### Pillar 1: Contract Satisfaction
|
|
29
|
-
- 100% of Acceptance Criteria defined in the [Goal Contract](
|
|
29
|
+
- 100% of Acceptance Criteria defined in the [Goal Contract](./goal-contract.md) are demonstrably met.
|
|
30
30
|
- Zero out-of-scope files or unauthorized modules were modified.
|
|
31
31
|
- All technical constraints (e.g. backward compatibility, no unapproved dependencies) are preserved.
|
|
32
32
|
|
|
@@ -45,12 +45,12 @@ flowchart LR
|
|
|
45
45
|
- **Null & Boundary Safety**: Explicit handling of `null`, `undefined`, empty collections, and error paths.
|
|
46
46
|
|
|
47
47
|
### Pillar 4: Adversarial Consensus
|
|
48
|
-
- Independent [Devil's Advocate Review](
|
|
48
|
+
- Independent [Devil's Advocate Review](../agents/devil-advocate.md) has been executed across all 6 core review domains.
|
|
49
49
|
- **Zero Unresolved Blocking Findings**: No open `SEV-1 (Critical)` or `SEV-2 (High)` findings.
|
|
50
50
|
- **Evidence-Based Triage**: Every raised finding has been formally triaged with reproducible evidence as `VALID` (and resolved in code) or `INVALID` (with technical proof of why it is a false positive).
|
|
51
51
|
|
|
52
52
|
### Pillar 5: Judge Certified PASS
|
|
53
|
-
- The [Judge Agent](
|
|
53
|
+
- The [Judge Agent](../agents/judge.md) has evaluated the complete execution trace, verified the evidence, confirmed no-progress limits were not violated, and issued a signed `PASS` verdict.
|
|
54
54
|
|
|
55
55
|
---
|
|
56
56
|
|
|
@@ -31,7 +31,7 @@ flowchart TD
|
|
|
31
31
|
- **Rationale**: Prolonged retries indicate either an incorrect fundamental strategy or a hidden architectural obstacle.
|
|
32
32
|
|
|
33
33
|
### Trigger 2: No-Progress / Stagnation Detected
|
|
34
|
-
- **Condition**: Two consecutive iterations produce identical or semantically equivalent finding signatures without demonstrable code convergence (see [No-Progress Policy](
|
|
34
|
+
- **Condition**: Two consecutive iterations produce identical or semantically equivalent finding signatures without demonstrable code convergence (see [No-Progress Policy](../policies/no-progress-policy.md)).
|
|
35
35
|
- **Rationale**: Prevents thrashing where the agent modifies code without resolving the underlying flaw.
|
|
36
36
|
|
|
37
37
|
### Trigger 3: Contradictory or Inconsistent Acceptance Criteria
|
|
@@ -93,7 +93,7 @@ Instead, it must render an **Actionable Escalation Report** following this struc
|
|
|
93
93
|
|
|
94
94
|
## 4. Resumption Protocol
|
|
95
95
|
|
|
96
|
-
Once the human engineer provides guidance or amends the [Goal Contract](
|
|
96
|
+
Once the human engineer provides guidance or amends the [Goal Contract](./goal-contract.md):
|
|
97
97
|
1. The iteration counter is reset: $K \leftarrow 1$.
|
|
98
98
|
2. The agent incorporates the human's decision into the Goal Contract constraints.
|
|
99
99
|
3. The loop resumes at the **Maker Agent** phase with clean validation.
|
package/core/goal-contract.md
CHANGED
|
@@ -11,7 +11,7 @@ The Goal Contract establishes:
|
|
|
11
11
|
- **Where** the boundaries are set (preventing scope creep).
|
|
12
12
|
- **When** the work is strictly considered complete.
|
|
13
13
|
|
|
14
|
-
Alignment before freeze follows [Grill Policy](
|
|
14
|
+
Alignment before freeze follows [Grill Policy](./grill-policy.md). Terms come from `.ai-engineering-loop/glossary.md`.
|
|
15
15
|
|
|
16
16
|
---
|
|
17
17
|
|
|
@@ -77,7 +77,7 @@ Each AC is one row that can **fail**. Happy path alone is not a contract (`polic
|
|
|
77
77
|
|
|
78
78
|
1. **Pre-Implementation Freezing**:
|
|
79
79
|
- The Goal Contract is authored and frozen *before* any production code edits.
|
|
80
|
-
- If the task is ambiguous and a human is present, run the [Grill Policy](
|
|
80
|
+
- If the task is ambiguous and a human is present, run the [Grill Policy](./grill-policy.md) until the design-tree frontier is empty, then freeze.
|
|
81
81
|
- If the task is unambiguous, waived, or headless with testable AC, skip grill and freeze immediately.
|
|
82
82
|
2. **Immutability During Iteration**:
|
|
83
83
|
- Neither the Maker Agent nor the Devil's Advocate Agent may alter Acceptance Criteria during an iteration loop to make tests pass or bypass critique.
|
package/core/iteration-policy.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## 1. Overview & Objectives
|
|
4
4
|
|
|
5
|
-
The **Iteration Policy** governs autonomous retry loops within the AI Engineering Loop. When deterministic tests fail or the [Devil's Advocate](
|
|
5
|
+
The **Iteration Policy** governs autonomous retry loops within the AI Engineering Loop. When deterministic tests fail or the [Devil's Advocate](../agents/devil-advocate.md) identifies valid blocking findings, the system initiates an **autonomous iteration cycle**.
|
|
6
6
|
|
|
7
7
|
The objective of this policy is to enable **reliable self-correction** while strictly guarding against infinite loops, cognitive drift, and resource exhaustion.
|
|
8
8
|
|
|
@@ -54,14 +54,14 @@ stateDiagram-v2
|
|
|
54
54
|
- Increments iteration counter $K \leftarrow K + 1$.
|
|
55
55
|
- Verifies $K \le \text{MAX\_ITERATIONS}$. If $K > \text{MAX\_ITERATIONS}$, immediately triggers `ESCALATE`.
|
|
56
56
|
2. **Maker Refinement**:
|
|
57
|
-
- The [Maker Agent](
|
|
57
|
+
- The [Maker Agent](../agents/maker.md) receives the prior iteration's triaged findings and Judge directives.
|
|
58
58
|
- Maker applies targeted code fixes and updates test suites.
|
|
59
59
|
3. **Deterministic Re-verification**:
|
|
60
60
|
- All tests, typecheck, lint, and build checks are re-executed from scratch.
|
|
61
61
|
4. **Adversarial Differential Review**:
|
|
62
|
-
- The [Devil's Advocate](
|
|
62
|
+
- The [Devil's Advocate](../agents/devil-advocate.md) evaluates the updated diff against previously raised findings and checks for new regressions.
|
|
63
63
|
5. **Judge Evaluation & Progression**:
|
|
64
|
-
- The [Judge Agent](
|
|
64
|
+
- The [Judge Agent](../agents/judge.md) compares findings from Iteration $K$ with Iteration $K-1$.
|
|
65
65
|
- If progress is verified and all criteria met $\rightarrow$ `PASS`.
|
|
66
66
|
- If new valid findings emerge but progress is demonstrated $\rightarrow$ `ITERATE` (if $K < \text{MAX\_ITERATIONS}$).
|
|
67
67
|
- If repeated findings or no progress detected $\rightarrow$ `ESCALATE`.
|
|
@@ -47,9 +47,9 @@ flowchart TD
|
|
|
47
47
|
- **Command**: `npx ai-engineering-loop run` (or `/ai-engineering-loop [task]`).
|
|
48
48
|
- **Boundaries**:
|
|
49
49
|
- The Agent reads `.ai-engineering-loop/` (triggering auto-init if missing).
|
|
50
|
-
- Formulates the [Goal Contract](
|
|
50
|
+
- Formulates the [Goal Contract](./goal-contract.md).
|
|
51
51
|
- Performs Root Cause Analysis (RCA) and Implementation Planning.
|
|
52
|
-
- Executes surgical diffs, runs deterministic test suites, conducts independent adversarial review via the [Devil's Advocate](
|
|
52
|
+
- Executes surgical diffs, runs deterministic test suites, conducts independent adversarial review via the [Devil's Advocate](../agents/devil-advocate.md), and obtains certification from the [Judge Agent](../agents/judge.md).
|
|
53
53
|
|
|
54
54
|
---
|
|
55
55
|
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Runtime Safety Modes
|
|
2
|
+
|
|
3
|
+
AI Engineering Loop separates analysis, human-supervised delivery, and unattended execution. The mode is stored in the run ledger and cannot change during a run.
|
|
4
|
+
|
|
5
|
+
## Modes
|
|
6
|
+
|
|
7
|
+
| Mode | Repository mutation | Delivery | Intended use |
|
|
8
|
+
|---|---|---|---|
|
|
9
|
+
| `REPORT_ONLY` | Forbidden | Ends at `gate report` | Audits, recommendations, and first-time evaluation |
|
|
10
|
+
| `ASSISTED` | Allowed after the Goal Contract gate | `delivery.json` must contain `"humanApproved": true` | Default production workflow |
|
|
11
|
+
| `UNATTENDED` | Allowed only when explicitly enabled; Maker uses a disposable locked Git worktree by default | Deterministic gates still apply | Trusted automation with budget monitoring |
|
|
12
|
+
|
|
13
|
+
`UNATTENDED` is disabled by default. Enable it deliberately:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npx ai-engineering-loop policy set --allow-unattended true
|
|
17
|
+
npx ai-engineering-loop run --mode unattended "task"
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The repository policy lives at `.ai-engineering-loop/runtime-policy.json`. Invalid policy fails closed. Do not change a policy merely because an agent asks to bypass a gate.
|
|
21
|
+
|
|
22
|
+
## Token budget and emergency stop
|
|
23
|
+
|
|
24
|
+
Check the budget before every model call and record provider-reported usage immediately afterward:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npx ai-engineering-loop budget status --run <run-id>
|
|
28
|
+
npx ai-engineering-loop budget record --run <run-id> \
|
|
29
|
+
--input <provider-input-tokens> \
|
|
30
|
+
--output <provider-output-tokens> \
|
|
31
|
+
--model <provider-model-id>
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Never convert byte estimates into "actual" usage. If the host does not expose provider usage, `UNATTENDED` must stop and escalate to a human. In `ASSISTED`, disclose that accounting is unavailable and require the human to decide whether to continue.
|
|
35
|
+
|
|
36
|
+
The per-run limit spans UTC days. The daily limit resets on the next UTC day. Ledgers are append-only JSONL under `.ai-engineering-loop/usage/` and are ignored by Git. A malformed ledger blocks execution rather than undercounting.
|
|
37
|
+
|
|
38
|
+
Emergency controls:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx ai-engineering-loop budget pause
|
|
42
|
+
npx ai-engineering-loop budget resume
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`pause` sets the persistent kill switch. It blocks new run and context/model boundaries, while `budget record` remains available so an in-flight response can still be accounted.
|
|
46
|
+
|
|
47
|
+
## Unattended Maker isolation
|
|
48
|
+
|
|
49
|
+
After `gate goal`, create the worktree:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
npx ai-engineering-loop sandbox create --run <run-id>
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Run Maker with its working directory set to the returned `worktreePath`. Do not let Maker edit the parent checkout. When Maker is done:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
npx ai-engineering-loop sandbox capture --run <run-id>
|
|
59
|
+
npx ai-engineering-loop gate maker --run <run-id>
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Capture writes `diff.patch` plus hash-bound `sandbox-evidence.json`, removes the disposable worktree, and releases the repository-wide Maker lock. The unattended Maker gate rejects a manually supplied diff without matching evidence. Use `sandbox abort` to discard an interrupted Maker worktree.
|
|
63
|
+
|
|
64
|
+
> [!IMPORTANT]
|
|
65
|
+
> A Git worktree is a filesystem and concurrency boundary, not an operating-system sandbox. It does not restrict network access, child processes, credentials inherited by the host, or writes outside the worktree. Run truly untrusted agents in a container or VM with a minimal environment and restricted network.
|
|
66
|
+
|
|
67
|
+
## Host invariant
|
|
68
|
+
|
|
69
|
+
Every supported host follows the same order:
|
|
70
|
+
|
|
71
|
+
1. Read run mode and budget status.
|
|
72
|
+
2. Freeze and gate the Goal Contract.
|
|
73
|
+
3. End with a report in `REPORT_ONLY`, require human delivery approval in `ASSISTED`, or isolate Maker in `UNATTENDED`.
|
|
74
|
+
4. Before each model/subagent call, check budget; after it returns, record actual usage.
|
|
75
|
+
5. Keep context packs, usage ledgers, worktrees, locks, raw logs, and credentials out of commits and remote payloads.
|
|
@@ -6,7 +6,7 @@ The Verification Loop is the deterministic machine gate of the AI Engineering Lo
|
|
|
6
6
|
|
|
7
7
|
> **"Code cannot enter Devil's Advocate review until it achieves 100% green machine verification backed by explicit, verifiable execution evidence."**
|
|
8
8
|
|
|
9
|
-
Maker reaches that green via [TDD Policy](
|
|
9
|
+
Maker reaches that green via [TDD Policy](../policies/tdd-policy.md): red at a named seam, then green. "We did TDD" without logs is not a PASS.
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Declarative Workflow Recipes
|
|
2
|
+
|
|
3
|
+
Workflow recipes describe a directed acyclic graph of engineering stages in JSON. They are an additive authoring layer over the existing AI Engineering Loop: `recipe` commands provide deterministic validation and inspection, while `run --recipe` opts into the controlled runtime. Runs without a recipe keep their established behavior.
|
|
4
|
+
|
|
5
|
+
This separation is deliberate. A visual editor or an AI agent may eventually author the JSON, but the compiler—not the authoring interface—owns safety invariants.
|
|
6
|
+
|
|
7
|
+
## Sources and precedence
|
|
8
|
+
|
|
9
|
+
Recipes are loaded from two locations:
|
|
10
|
+
|
|
11
|
+
1. Package presets in `recipes/*.json`
|
|
12
|
+
2. Project recipes in `.ai-engineering-loop/recipes/*.json`
|
|
13
|
+
|
|
14
|
+
Built-in IDs are reserved and cannot be shadowed by a project file. Recipe IDs and node IDs use lowercase letters, numbers, and hyphens. Recipe files and the project recipe directory must be regular filesystem entries; symlinks are rejected.
|
|
15
|
+
|
|
16
|
+
The package includes:
|
|
17
|
+
|
|
18
|
+
| Recipe | Purpose | Modes |
|
|
19
|
+
| --- | --- | --- |
|
|
20
|
+
| `default` | Existing production safety loop | `ASSISTED`, `UNATTENDED` |
|
|
21
|
+
| `audit` | Read-only structured analysis | `REPORT_ONLY` |
|
|
22
|
+
| `bugfix` | Root-cause-first repair | `ASSISTED`, `UNATTENDED` |
|
|
23
|
+
| `docs-light` | Bounded documentation change | `ASSISTED`, `UNATTENDED` |
|
|
24
|
+
| `high-risk` | Blast-radius analysis and pre-mutation approval | `ASSISTED`, `UNATTENDED` |
|
|
25
|
+
| `refactor` | Behavior-baseline-first refactor | `ASSISTED`, `UNATTENDED` |
|
|
26
|
+
|
|
27
|
+
## Safety backbone
|
|
28
|
+
|
|
29
|
+
The graph compiler fails closed.
|
|
30
|
+
|
|
31
|
+
- `REPORT_ONLY` requires `goal-contract → ... → report` and rejects every mutation or remote-side-effect node.
|
|
32
|
+
- `ASSISTED` and `UNATTENDED` require the ordered backbone `goal-contract → maker → verification → devil-advocate → judge → delivery`.
|
|
33
|
+
- `ASSISTED` additionally requires human approval after Judge and before delivery.
|
|
34
|
+
- Required safety nodes cannot be conditional.
|
|
35
|
+
- `report` and `delivery` are graph terminals. Every node must lead to one terminal.
|
|
36
|
+
- Cycles, duplicate singleton safety nodes, missing dependencies, unknown node types, and incompatible modes are invalid.
|
|
37
|
+
|
|
38
|
+
Dependencies may include additional nodes between required stages. A custom analysis node before `maker`, for example, does not weaken the mandatory backbone.
|
|
39
|
+
|
|
40
|
+
## Node catalog
|
|
41
|
+
|
|
42
|
+
| Type | Intended use | Model | Repository mutation | Remote side effect |
|
|
43
|
+
| --- | --- | ---: | ---: | ---: |
|
|
44
|
+
| `goal-contract` | Acceptance contract | No | No | No |
|
|
45
|
+
| `agent` | Bounded structured analysis | Yes | No | No |
|
|
46
|
+
| `command` | Trusted repository executable | No | No | No |
|
|
47
|
+
| `condition` | Structured branch predicate | No | No | No |
|
|
48
|
+
| `approval` | Human checkpoint | No | No | No |
|
|
49
|
+
| `artifact-check` | Deterministic artifact assertion | No | No | No |
|
|
50
|
+
| `maker` | Implementation | Yes | Yes | No |
|
|
51
|
+
| `verification` | Evidence collection | No | No | No |
|
|
52
|
+
| `devil-advocate` | Adversarial review | Yes | No | No |
|
|
53
|
+
| `judge` | Independent verdict | Yes | No | No |
|
|
54
|
+
| `report` | Read-only terminal | No | No | No |
|
|
55
|
+
| `delivery` | Delivery terminal | No | No | Yes |
|
|
56
|
+
|
|
57
|
+
An `agent` node must declare a role, bounded `maxFiles` and `maxTokens`, and a repository-relative output schema. A `command` node must use a bare executable name, an argument array, a bounded timeout, and expected exit codes. Shells and general-purpose interpreters are rejected; command nodes are for trusted repository tools, not arbitrary script text.
|
|
58
|
+
|
|
59
|
+
## Authoring example
|
|
60
|
+
|
|
61
|
+
Project recipe:
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"schemaVersion": 1,
|
|
66
|
+
"id": "api-bugfix",
|
|
67
|
+
"version": 1,
|
|
68
|
+
"description": "Analyze an API regression before entering the standard repair loop.",
|
|
69
|
+
"compatibleModes": ["ASSISTED"],
|
|
70
|
+
"nodes": [
|
|
71
|
+
{ "id": "goal", "type": "goal-contract" },
|
|
72
|
+
{
|
|
73
|
+
"id": "api-analysis",
|
|
74
|
+
"type": "agent",
|
|
75
|
+
"dependsOn": ["goal"],
|
|
76
|
+
"role": "api-regression-analyst",
|
|
77
|
+
"context": { "maxFiles": 25, "maxTokens": 8000 },
|
|
78
|
+
"output": {
|
|
79
|
+
"artifact": "api-analysis",
|
|
80
|
+
"schema": "schemas/report.schema.json"
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
{ "id": "maker", "type": "maker", "dependsOn": ["api-analysis"] },
|
|
84
|
+
{ "id": "verification", "type": "verification", "dependsOn": ["maker"] },
|
|
85
|
+
{ "id": "review", "type": "devil-advocate", "dependsOn": ["verification"] },
|
|
86
|
+
{ "id": "judge", "type": "judge", "dependsOn": ["review"] },
|
|
87
|
+
{
|
|
88
|
+
"id": "approval",
|
|
89
|
+
"type": "approval",
|
|
90
|
+
"dependsOn": ["judge"],
|
|
91
|
+
"message": "Approve delivery."
|
|
92
|
+
},
|
|
93
|
+
{ "id": "delivery", "type": "delivery", "dependsOn": ["approval"] }
|
|
94
|
+
]
|
|
95
|
+
}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Validate and inspect it:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
ai-engineering-loop recipe validate api-bugfix --mode assisted
|
|
102
|
+
ai-engineering-loop recipe explain api-bugfix --mode assisted
|
|
103
|
+
ai-engineering-loop recipe graph api-bugfix --mode assisted
|
|
104
|
+
ai-engineering-loop recipe simulate api-bugfix --mode assisted
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`simulate` compiles the graph and reports capability counts. It never calls a model, mutates the repository, executes a command, or performs delivery.
|
|
108
|
+
|
|
109
|
+
## AI-assisted authoring protocol
|
|
110
|
+
|
|
111
|
+
Until a visual editor exists, an AI agent can safely act as the recipe builder:
|
|
112
|
+
|
|
113
|
+
1. Ask the user for the outcome, allowed run modes, required checkpoints, context limits, and delivery risk.
|
|
114
|
+
2. Start from the closest built-in preset rather than a blank graph.
|
|
115
|
+
3. Add only nodes that change the workflow meaningfully.
|
|
116
|
+
4. Write the candidate to `.ai-engineering-loop/recipes/<id>.json`.
|
|
117
|
+
5. Run `recipe validate`, `recipe explain`, `recipe graph`, and `recipe simulate`.
|
|
118
|
+
6. Present the explanation, graph hash, warnings, and capability counts for human confirmation.
|
|
119
|
+
7. Do not claim execution from validation or simulation. Execution is proven only by a recipe-bound run and its validated event chain.
|
|
120
|
+
|
|
121
|
+
Machine consumers should use `--json`. The compiled plan includes a canonical source hash and graph hash so execution and audit logs bind to the exact graph reviewed by the user. Runtime behavior is specified in `core/controlled-workflow-runtime.md`.
|
|
122
|
+
|
|
123
|
+
The CLI supports safe scaffolding and installation:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
ai-engineering-loop recipe create api-bugfix --from bugfix
|
|
127
|
+
ai-engineering-loop recipe inspect api-bugfix --json
|
|
128
|
+
ai-engineering-loop recipe diff bugfix api-bugfix
|
|
129
|
+
ai-engineering-loop recipe install candidate-v2.json --replace
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Creation never overwrites an existing recipe. Installation validates before writing, reserves built-in IDs, requires `--replace` plus a strictly higher version, and stores the previous document under `.ai-engineering-loop/recipes/.history/<id>/`. Candidate paths and recipe stores reject filesystem escape and symlink traversal.
|
|
133
|
+
|
|
134
|
+
## Schemas
|
|
135
|
+
|
|
136
|
+
- `schemas/recipe.schema.json`: top-level authoring document
|
|
137
|
+
- `schemas/node-definition.schema.json`: node shape and type catalog
|
|
138
|
+
- `schemas/compiled-plan.schema.json`: deterministic compiler output
|
|
139
|
+
|
|
140
|
+
Semantic safety checks are stricter than JSON Schema and are enforced by `recipe validate` and the compiler.
|
package/lib/budget.js
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { assertRunId, getCurrentRun, loadRun } = require('./run-state.js');
|
|
6
|
+
const { loadPolicy, updatePolicy } = require('./runtime-policy.js');
|
|
7
|
+
const { ensurePrivateRuntimeIgnores } = require('./runtime-files.js');
|
|
8
|
+
|
|
9
|
+
function usageDir(rootDir) {
|
|
10
|
+
return path.join(rootDir, '.ai-engineering-loop', 'usage');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function utcDay(now = new Date()) {
|
|
14
|
+
if (!(now instanceof Date) || Number.isNaN(now.getTime())) {
|
|
15
|
+
const err = new Error('Usage timestamp must be a valid Date');
|
|
16
|
+
err.code = 'INVALID_USAGE';
|
|
17
|
+
throw err;
|
|
18
|
+
}
|
|
19
|
+
return now.toISOString().slice(0, 10);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function usagePath(rootDir, now = new Date()) {
|
|
23
|
+
return path.join(usageDir(rootDir), `${utcDay(now)}.jsonl`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function usageError(message, code = 'INVALID_USAGE') {
|
|
27
|
+
const err = new Error(message);
|
|
28
|
+
err.code = code;
|
|
29
|
+
return err;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function validateTokenCount(name, value) {
|
|
33
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
34
|
+
throw usageError(`${name} must be a non-negative integer`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function validateModel(model) {
|
|
39
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$/.test(String(model || ''))) {
|
|
40
|
+
throw usageError('model must be a provider model identifier, not free-form text');
|
|
41
|
+
}
|
|
42
|
+
return String(model);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function readUsageFile(filePath) {
|
|
46
|
+
let stat;
|
|
47
|
+
try {
|
|
48
|
+
stat = fs.lstatSync(filePath);
|
|
49
|
+
} catch (cause) {
|
|
50
|
+
if (cause.code === 'ENOENT') return [];
|
|
51
|
+
throw usageError(`Unable to inspect usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
52
|
+
}
|
|
53
|
+
if (stat.isSymbolicLink()) {
|
|
54
|
+
throw usageError('Usage ledger must not be a symbolic link', 'INVALID_USAGE_LEDGER');
|
|
55
|
+
}
|
|
56
|
+
let lines;
|
|
57
|
+
try {
|
|
58
|
+
lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
|
|
59
|
+
} catch (cause) {
|
|
60
|
+
throw usageError(`Unable to read usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
61
|
+
}
|
|
62
|
+
return lines.map((line, index) => {
|
|
63
|
+
try {
|
|
64
|
+
const entry = JSON.parse(line);
|
|
65
|
+
assertRunId(entry.runId);
|
|
66
|
+
validateTokenCount('inputTokens', entry.inputTokens);
|
|
67
|
+
validateTokenCount('outputTokens', entry.outputTokens);
|
|
68
|
+
if (entry.totalTokens !== entry.inputTokens + entry.outputTokens) {
|
|
69
|
+
throw new Error('totalTokens does not equal inputTokens + outputTokens');
|
|
70
|
+
}
|
|
71
|
+
validateModel(entry.model);
|
|
72
|
+
if (entry.nodeId != null && !/^[a-z][a-z0-9-]{0,63}$/.test(entry.nodeId)) {
|
|
73
|
+
throw new Error('nodeId is invalid');
|
|
74
|
+
}
|
|
75
|
+
if (typeof entry.recordedAt !== 'string' || Number.isNaN(Date.parse(entry.recordedAt))) {
|
|
76
|
+
throw new Error('recordedAt is invalid');
|
|
77
|
+
}
|
|
78
|
+
return entry;
|
|
79
|
+
} catch (cause) {
|
|
80
|
+
throw usageError(
|
|
81
|
+
`Usage ledger line ${index + 1} is invalid: ${cause.message}`,
|
|
82
|
+
'INVALID_USAGE_LEDGER'
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function readUsage(rootDir, now = new Date()) {
|
|
89
|
+
return readUsageFile(usagePath(rootDir, now));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function readAllUsage(rootDir) {
|
|
93
|
+
const directory = usageDir(rootDir);
|
|
94
|
+
if (!fs.existsSync(directory)) return [];
|
|
95
|
+
let names;
|
|
96
|
+
try {
|
|
97
|
+
names = fs.readdirSync(directory)
|
|
98
|
+
.filter((name) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(name))
|
|
99
|
+
.sort();
|
|
100
|
+
} catch (cause) {
|
|
101
|
+
throw usageError(`Unable to enumerate usage ledgers: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
102
|
+
}
|
|
103
|
+
return names.flatMap((name) => readUsageFile(path.join(directory, name)));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function resolveRun(rootDir, runId) {
|
|
107
|
+
if (runId) return loadRun(rootDir, assertRunId(runId));
|
|
108
|
+
const current = getCurrentRun(rootDir);
|
|
109
|
+
if (!current) throw usageError('No current run. Pass --run <id>.', 'NO_CURRENT_RUN');
|
|
110
|
+
return current;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function budgetStatus(rootDir, { runId, nodeId, estimatedTokens = 0, now = new Date() } = {}) {
|
|
114
|
+
validateTokenCount('estimatedTokens', estimatedTokens);
|
|
115
|
+
const run = resolveRun(rootDir, runId);
|
|
116
|
+
const policy = loadPolicy(rootDir);
|
|
117
|
+
const todayEntries = readUsage(rootDir, now);
|
|
118
|
+
const allEntries = readAllUsage(rootDir);
|
|
119
|
+
const spentToday = todayEntries.reduce((sum, entry) => sum + entry.totalTokens, 0);
|
|
120
|
+
const spentThisRun = allEntries
|
|
121
|
+
.filter((entry) => entry.runId === run.runId)
|
|
122
|
+
.reduce((sum, entry) => sum + entry.totalTokens, 0);
|
|
123
|
+
const projectedToday = spentToday + estimatedTokens;
|
|
124
|
+
const projectedThisRun = spentThisRun + estimatedTokens;
|
|
125
|
+
const reasons = [];
|
|
126
|
+
let node = null;
|
|
127
|
+
let spentThisNode = 0;
|
|
128
|
+
let nodeTokenLimit = null;
|
|
129
|
+
if (nodeId) {
|
|
130
|
+
const { workflowStatus, NODE_STATES } = require('./workflow-runtime.js');
|
|
131
|
+
const workflow = workflowStatus(rootDir, { runId: run.runId });
|
|
132
|
+
node = workflow.plan.nodes.find((candidate) => candidate.id === nodeId);
|
|
133
|
+
if (!node) throw usageError(`Unknown workflow node: ${nodeId}`);
|
|
134
|
+
if (!node.capabilities.modelRequired) reasons.push(`node ${nodeId} does not permit model usage`);
|
|
135
|
+
const status = workflow.state.nodes[nodeId].status;
|
|
136
|
+
if (![NODE_STATES.READY, NODE_STATES.RUNNING].includes(status)) {
|
|
137
|
+
reasons.push(`node ${nodeId} is ${status}, not READY`);
|
|
138
|
+
}
|
|
139
|
+
nodeTokenLimit = node.context?.maxTokens || null;
|
|
140
|
+
spentThisNode = allEntries
|
|
141
|
+
.filter((entry) => entry.runId === run.runId && entry.nodeId === nodeId)
|
|
142
|
+
.reduce((sum, entry) => sum + entry.totalTokens, 0);
|
|
143
|
+
if (nodeTokenLimit != null && spentThisNode + estimatedTokens > nodeTokenLimit) {
|
|
144
|
+
reasons.push(`node ${nodeId} token limit would be exceeded`);
|
|
145
|
+
}
|
|
146
|
+
if (nodeTokenLimit != null && estimatedTokens === 0 && spentThisNode >= nodeTokenLimit) {
|
|
147
|
+
reasons.push(`node ${nodeId} token limit is exhausted`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (policy.killSwitch) reasons.push('kill switch is active');
|
|
151
|
+
if (projectedThisRun > policy.perRunTokenLimit) reasons.push('per-run token limit would be exceeded');
|
|
152
|
+
if (projectedToday > policy.dailyTokenLimit) reasons.push('daily token limit would be exceeded');
|
|
153
|
+
if (estimatedTokens === 0 && spentThisRun >= policy.perRunTokenLimit) {
|
|
154
|
+
reasons.push('per-run token limit is exhausted');
|
|
155
|
+
}
|
|
156
|
+
if (estimatedTokens === 0 && spentToday >= policy.dailyTokenLimit) {
|
|
157
|
+
reasons.push('daily token limit is exhausted');
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
schemaVersion: 1,
|
|
162
|
+
runId: run.runId,
|
|
163
|
+
day: utcDay(now),
|
|
164
|
+
allowed: reasons.length === 0,
|
|
165
|
+
reasons: [...new Set(reasons)],
|
|
166
|
+
killSwitch: policy.killSwitch,
|
|
167
|
+
spentThisRun,
|
|
168
|
+
perRunTokenLimit: policy.perRunTokenLimit,
|
|
169
|
+
remainingThisRun: Math.max(0, policy.perRunTokenLimit - spentThisRun),
|
|
170
|
+
spentToday,
|
|
171
|
+
dailyTokenLimit: policy.dailyTokenLimit,
|
|
172
|
+
remainingToday: Math.max(0, policy.dailyTokenLimit - spentToday),
|
|
173
|
+
estimatedTokens,
|
|
174
|
+
...(node ? {
|
|
175
|
+
nodeId,
|
|
176
|
+
spentThisNode,
|
|
177
|
+
nodeTokenLimit,
|
|
178
|
+
remainingThisNode: nodeTokenLimit == null ? null : Math.max(0, nodeTokenLimit - spentThisNode)
|
|
179
|
+
} : {})
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function assertBudgetAvailable(rootDir, options = {}) {
|
|
184
|
+
const status = budgetStatus(rootDir, options);
|
|
185
|
+
if (!status.allowed) {
|
|
186
|
+
const err = new Error(`Token budget blocked: ${status.reasons.join('; ')}`);
|
|
187
|
+
err.code = 'BUDGET_BLOCKED';
|
|
188
|
+
err.details = status.reasons;
|
|
189
|
+
err.status = status;
|
|
190
|
+
throw err;
|
|
191
|
+
}
|
|
192
|
+
return status;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function appendPrivateLine(rootDir, filePath, line) {
|
|
196
|
+
ensurePrivateRuntimeIgnores(rootDir);
|
|
197
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
198
|
+
const noFollow = fs.constants.O_NOFOLLOW || 0;
|
|
199
|
+
let descriptor;
|
|
200
|
+
try {
|
|
201
|
+
descriptor = fs.openSync(
|
|
202
|
+
filePath,
|
|
203
|
+
fs.constants.O_APPEND | fs.constants.O_CREAT | fs.constants.O_WRONLY | noFollow,
|
|
204
|
+
0o600
|
|
205
|
+
);
|
|
206
|
+
if (!fs.fstatSync(descriptor).isFile()) throw new Error('usage ledger is not a regular file');
|
|
207
|
+
fs.writeSync(descriptor, line);
|
|
208
|
+
fs.fchmodSync(descriptor, 0o600);
|
|
209
|
+
} catch (cause) {
|
|
210
|
+
throw usageError(`Unable to append usage ledger: ${cause.message}`, 'USAGE_WRITE_FAILED');
|
|
211
|
+
} finally {
|
|
212
|
+
if (descriptor != null) fs.closeSync(descriptor);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function recordTokenUsage(
|
|
217
|
+
rootDir,
|
|
218
|
+
{ runId, nodeId, inputTokens, outputTokens, model, now = new Date() } = {}
|
|
219
|
+
) {
|
|
220
|
+
const run = resolveRun(rootDir, runId);
|
|
221
|
+
validateTokenCount('inputTokens', inputTokens);
|
|
222
|
+
validateTokenCount('outputTokens', outputTokens);
|
|
223
|
+
const normalizedModel = validateModel(model);
|
|
224
|
+
const totalTokens = inputTokens + outputTokens;
|
|
225
|
+
if (totalTokens === 0) throw usageError('A usage record must contain at least one token');
|
|
226
|
+
const entry = {
|
|
227
|
+
schemaVersion: 1,
|
|
228
|
+
runId: run.runId,
|
|
229
|
+
inputTokens,
|
|
230
|
+
outputTokens,
|
|
231
|
+
totalTokens,
|
|
232
|
+
model: normalizedModel,
|
|
233
|
+
...(nodeId ? { nodeId } : {}),
|
|
234
|
+
recordedAt: now.toISOString()
|
|
235
|
+
};
|
|
236
|
+
if (nodeId) budgetStatus(rootDir, { runId: run.runId, nodeId, now });
|
|
237
|
+
appendPrivateLine(rootDir, usagePath(rootDir, now), `${JSON.stringify(entry)}\n`);
|
|
238
|
+
return {
|
|
239
|
+
entry,
|
|
240
|
+
status: budgetStatus(rootDir, { runId: run.runId, nodeId, now })
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function setKillSwitch(rootDir, active) {
|
|
245
|
+
return updatePolicy(rootDir, { killSwitch: Boolean(active) });
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
module.exports = {
|
|
249
|
+
usageDir,
|
|
250
|
+
utcDay,
|
|
251
|
+
usagePath,
|
|
252
|
+
readUsage,
|
|
253
|
+
readAllUsage,
|
|
254
|
+
budgetStatus,
|
|
255
|
+
assertBudgetAvailable,
|
|
256
|
+
recordTokenUsage,
|
|
257
|
+
setKillSwitch
|
|
258
|
+
};
|