@tangle-network/agent-runtime 0.126.0 → 0.128.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +2 -2
- package/dist/{environment-provider-PM9PeW_J.d.ts → environment-provider-CUFsyymu.d.ts} +7 -1
- package/dist/environment-provider.d.ts +1 -1
- package/dist/{index-EdjCQBV9.d.ts → index-BhZhQw77.d.ts} +198 -20
- package/dist/{index-DIV33AF5.d.ts → index-BhuzfG2r.d.ts} +3 -3
- package/dist/{index-D_M4d1_B.d.ts → index-DLM0W1h1.d.ts} +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +1 -1
- package/dist/kernel.d.ts +3 -3
- package/dist/kernel.js +3 -3
- package/dist/{knowledge-EnuEqm_Y.js → knowledge-DF63xPr4.js} +2 -2
- package/dist/{knowledge-EnuEqm_Y.js.map → knowledge-DF63xPr4.js.map} +1 -1
- package/dist/knowledge.d.ts +1 -1
- package/dist/knowledge.js +1 -1
- package/dist/{loop-runner-bin-qwT_4F5I.js → loop-runner-bin-CWqOpCEw.js} +3 -3
- package/dist/{loop-runner-bin-qwT_4F5I.js.map → loop-runner-bin-CWqOpCEw.js.map} +1 -1
- package/dist/{loop-runner-bin-Bo29_fiD.d.ts → loop-runner-bin-Ckp_9tmD.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +1 -1
- package/dist/mcp/index.d.ts +3 -7
- package/dist/mcp/index.js +3 -3
- package/dist/{openai-tools-Bp1KSkP6.js → openai-tools-D3XfrrQ6.js} +2 -2
- package/dist/{openai-tools-Bp1KSkP6.js.map → openai-tools-D3XfrrQ6.js.map} +1 -1
- package/dist/primeintellect/index.d.ts +1 -1
- package/dist/{runtime-BzXz7OjS.js → runtime-5uDVVfER.js} +362 -11
- package/dist/runtime-5uDVVfER.js.map +1 -0
- package/dist/{supervise-Ds8FtyI9.js → supervise-CsTKbH9R.js} +100 -18
- package/dist/supervise-CsTKbH9R.js.map +1 -0
- package/dist/testing.js +8 -8
- package/package.json +1 -1
- package/skills/agent-graphs/IMPROVE.md +58 -0
- package/skills/agent-graphs/SKILL.md +140 -0
- package/skills/agent-graphs/cases/artifact-mission-release-notes.json +10 -0
- package/skills/agent-graphs/cases/audited-single-writer.json +9 -0
- package/skills/agent-graphs/cases/cap-as-stop-mistake.json +8 -0
- package/skills/agent-graphs/cases/floor-trap-pi.json +11 -0
- package/skills/agent-graphs/cases/mission-in-deliverable.json +8 -0
- package/skills/agent-graphs/cases/review-pipeline.json +14 -0
- package/skills/agent-graphs/cases/runtime-discovered-fanout.json +8 -0
- package/skills/agent-graphs/cases/single-agent-suffices.json +7 -0
- package/skills/agent-graphs/cases/steer-heavy-drafting.json +9 -0
- package/skills/agent-graphs/cases/unmeasured-harness.json +9 -0
- package/skills/agent-graphs/generations/gen1-baseline.json +248 -0
- package/skills/agent-graphs/generations/gen2.json +375 -0
- package/skills/agent-graphs/generations/gen3.json +702 -0
- package/skills/build-with-agent-runtime/SKILL.md +1 -0
- package/dist/runtime-BzXz7OjS.js.map +0 -1
- package/dist/supervise-Ds8FtyI9.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -4,7 +4,7 @@ import { SANDBOX_SIZE_PRESET_NAMES } from "@tangle-network/agent-interface";
|
|
|
4
4
|
//#region src/testing/fixtures/agent-improvement-proposal.json
|
|
5
5
|
var agent_improvement_proposal_default = {
|
|
6
6
|
changedSurfaces: ["prompt"],
|
|
7
|
-
digest: "sha256:
|
|
7
|
+
digest: "sha256:5feb51aa14c16c29e4b06a4f344dec42ecc7c13f0a876c17ed743014580dc93a",
|
|
8
8
|
evaluation: {
|
|
9
9
|
"decision": {
|
|
10
10
|
"contributingChecks": [
|
|
@@ -4515,7 +4515,7 @@ var agent_improvement_proposal_default = {
|
|
|
4515
4515
|
],
|
|
4516
4516
|
"metadata": {
|
|
4517
4517
|
"fixture": "agent-improvement-proposal",
|
|
4518
|
-
"runtimeVersion": "0.
|
|
4518
|
+
"runtimeVersion": "0.128.0"
|
|
4519
4519
|
},
|
|
4520
4520
|
"objectives": [
|
|
4521
4521
|
{
|
|
@@ -4626,8 +4626,8 @@ var agent_improvement_proposal_default = {
|
|
|
4626
4626
|
"baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09",
|
|
4627
4627
|
"candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693",
|
|
4628
4628
|
"kind": "agent-eval-loop",
|
|
4629
|
-
"recordDigest": "sha256:
|
|
4630
|
-
"runId": "agent-runtime-0.
|
|
4629
|
+
"recordDigest": "sha256:21aec498ac5cf1fec87690d5216cf4edb811e0db19b841f8be84c1bea8a9365e",
|
|
4630
|
+
"runId": "agent-runtime-0.128.0-proposal-fixture",
|
|
4631
4631
|
"schema": "agent-candidate-experiment"
|
|
4632
4632
|
}
|
|
4633
4633
|
},
|
|
@@ -4650,13 +4650,13 @@ var agent_improvement_proposal_default = {
|
|
|
4650
4650
|
}],
|
|
4651
4651
|
kind: "agent-improvement-proposal",
|
|
4652
4652
|
proposedAt: "2026-07-10T01:00:00.000Z",
|
|
4653
|
-
runId: "agent-runtime-0.
|
|
4653
|
+
runId: "agent-runtime-0.128.0-proposal-fixture"
|
|
4654
4654
|
};
|
|
4655
4655
|
//#endregion
|
|
4656
4656
|
//#region src/testing/fixtures/agent-profile-improvement-proposal.json
|
|
4657
4657
|
var agent_profile_improvement_proposal_default = {
|
|
4658
4658
|
changedSurfaces: ["prompt", "skills"],
|
|
4659
|
-
digest: "sha256:
|
|
4659
|
+
digest: "sha256:481b9db8d4e8e1834114e3b922876eaa0d68ac007c094ce9b2bd9aa7a14d812e",
|
|
4660
4660
|
evaluation: {
|
|
4661
4661
|
"decision": {
|
|
4662
4662
|
"contributingChecks": [
|
|
@@ -6290,7 +6290,7 @@ var agent_profile_improvement_proposal_default = {
|
|
|
6290
6290
|
],
|
|
6291
6291
|
"metadata": {
|
|
6292
6292
|
"fixture": "agent-profile-improvement-proposal",
|
|
6293
|
-
"runtimeVersion": "0.
|
|
6293
|
+
"runtimeVersion": "0.128.0"
|
|
6294
6294
|
},
|
|
6295
6295
|
"objectives": [
|
|
6296
6296
|
{
|
|
@@ -6401,7 +6401,7 @@ var agent_profile_improvement_proposal_default = {
|
|
|
6401
6401
|
"baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704",
|
|
6402
6402
|
"candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9",
|
|
6403
6403
|
"kind": "agent-eval-loop",
|
|
6404
|
-
"recordDigest": "sha256:
|
|
6404
|
+
"recordDigest": "sha256:9c0bf8c61092567c90d1d40e48ba3f44f70433c80dcd6d28cc14b14a80bcd295",
|
|
6405
6405
|
"runId": "profile-improvement-1",
|
|
6406
6406
|
"schema": "agent-profile-improvement-experiment"
|
|
6407
6407
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-runtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.128.0",
|
|
4
4
|
"description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-runtime#readme",
|
|
6
6
|
"repository": {
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Improving the agent-graphs skill: the loop, mapped onto what exists
|
|
2
|
+
|
|
3
|
+
The improving artifact is `skills/agent-graphs/SKILL.md`'s text.
|
|
4
|
+
Nothing below is a new framework; every step names the existing agent-eval primitive it composes, per the adopt-or-improve rule.
|
|
5
|
+
The only code this loop owns is two closures and a case set — the slots the machinery deliberately leaves to the caller.
|
|
6
|
+
|
|
7
|
+
## The loop
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
case (idea brief) ──► author agent + skill-vN ──► graph ──► runGraph OFFLINE ──► score ──► revise skill ──► gate ──► vN+1
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
| Step | Primitive | What the loop supplies |
|
|
14
|
+
| --- | --- | --- |
|
|
15
|
+
| Skill text as candidate surface | `MutableSurface = string` (`campaign/types.ts:210`); read `skills/agent-graphs/SKILL.md` → string | one line |
|
|
16
|
+
| Generate graph from case | caller-owned `dispatchWithSurface(surface, scenario, ctx)` in `runCampaign` | **closure A**: run an author agent carrying skill-vN + the case brief; return the authored graph module |
|
|
17
|
+
| Execute offline | `runGraph` with scripted `brain`, stub leaf seam, in-memory journal/blobs (the `examples/graphs/` pattern) | part of closure A |
|
|
18
|
+
| Deterministic scoring | a `JudgeConfig` closure (the `golden-matcher`/`completion-verifier` pattern) | **closure B**: score from `GraphResult` — validation passed; expected edges present with >0 traversals; ledger outcomes match the case's expectations; `exhaustedEdges` empty unless expected; deliverable verdict correct on both a passing and a failing scripted run |
|
|
19
|
+
| Semantic scoring (only what mechanics can't see) | `judge-panel.ts` `ensembleJudge` (cross-family, fail-loud) | rubric: role decomposition quality, directive clarity |
|
|
20
|
+
| Revision | `skillOptOptimizationMethod` (requires a **string** surface — skill text is first-class) or `gepa-optimization-method`; trace-conditioned diffs via `reflective-mutation.ts` | config only |
|
|
21
|
+
| Generations + incumbent | `runOptimization` (retains every generation's surfaces and campaigns) | config only |
|
|
22
|
+
| Gated promotion | `runImprovementLoop` — disjoint train/holdout enforced, no-op winner forced to hold, `autoOnPromote: 'pr'` writes the winner back as a PR | config only |
|
|
23
|
+
| Audit trail | `search-ledger.ts` hash-chained JSONL | free |
|
|
24
|
+
|
|
25
|
+
## Cases
|
|
26
|
+
|
|
27
|
+
`cases/` seeds eight idea-briefs, each with `expect`: the edges a correct graph must have, ledger outcomes, whether analysts are warranted, and a floor-trap flag (the case is under-budgetable and a correct author must budget above the floor).
|
|
28
|
+
Case briefs are deliberately loose — "loose context in, correct graph out" is the skill's whole claim, so tidy specs would test the wrong thing.
|
|
29
|
+
|
|
30
|
+
Holdout discipline: at least 3 of the 8 held out, never trained on; `runImprovementLoop` throws on overlap.
|
|
31
|
+
|
|
32
|
+
## What is deliberately NOT built
|
|
33
|
+
|
|
34
|
+
- No graph-diff scorer beyond the ledger checks — a graph is correct if it *runs* correctly offline, not if it textually matches a golden.
|
|
35
|
+
- No new optimizer, campaign runner, judge plumbing, or ledger — all named above.
|
|
36
|
+
- No live-backend scoring in the loop. Live runs are pursuit work, not skill-improvement work; the loop stays offline and free.
|
|
37
|
+
|
|
38
|
+
## Orchestration layering (doctrine, gates gen4)
|
|
39
|
+
|
|
40
|
+
Two layers, different jobs.
|
|
41
|
+
Foundation harnesses ship trained orchestration — Claude Code subagents, codex goal-mode, pi extensions — and prose is that layer's native API: instructing a claude-code node to "fan out subagents over these files" invokes an in-distribution capability, not vibes.
|
|
42
|
+
The graph/script layer exists for what no single harness provides: cross-harness composition, one conserved budget across the whole tree, durable ledger evidence, resume, and heterogeneous model placement.
|
|
43
|
+
Rule: outer layer coarse, inner layer maximal — one harness-sized node told to use its native fan-out beats N externally-choreographed thin nodes that each pay the harness floor while suppressing the orchestration the model was trained for.
|
|
44
|
+
Which harnesses qualify for native fan-out is a supervisor-lab harness-KB row, not a guess.
|
|
45
|
+
|
|
46
|
+
## Version history
|
|
47
|
+
|
|
48
|
+
The live tree carries only the current `SKILL.md`; every prior surface text is recoverable from git history via the pinned sha256s below, and each generation's full measurement record lives in `generations/`.
|
|
49
|
+
|
|
50
|
+
| gen | date | surface sha256 (short) | holdout mean | verdict |
|
|
51
|
+
| --- | --- | --- | --- | --- |
|
|
52
|
+
| 1 | 2026-08-03 | `582429a1` | 0.444 (k=3 re-measure in `generations/gen2.json`, n=9 holdout cells) | baseline |
|
|
53
|
+
| 2 | 2026-08-03 | `4c6615b6` | 0.611 (k=3, n=9 holdout cells); 0.600 at the gen3 k=5 re-measure | SHIP (#722) |
|
|
54
|
+
| 3 | 2026-08-03 | `54e7b38b` (not promoted; v2 stays live) | 0.900 — invalidated | **HOLD**: verifier found case-design contamination and scorer leniency; reasons + gen4 requirements in `generations/gen3.json` `verifierHold` |
|
|
55
|
+
|
|
56
|
+
## Known upstream gap this loop will hit
|
|
57
|
+
|
|
58
|
+
`OptimizationMethodResult` returns `winnerSurface` only — full candidate history is an owed upstream extension (recorded in discovery docs 22/25). Workaround needing no code: `runOptimization` already retains every generation's surfaces.
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: agent-graphs
|
|
3
|
+
description: Author runGraph programs from AgentProfiles and versioned prompt directives.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Agent graphs
|
|
7
|
+
|
|
8
|
+
Use this skill when every role is known before execution and the relationship between roles must be reviewable as data.
|
|
9
|
+
The output is an `AgentGraph` executed by `runGraph`, not a new coordinator or workflow framework.
|
|
10
|
+
|
|
11
|
+
## Choose the existing entry point
|
|
12
|
+
|
|
13
|
+
| Need | Use |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| Known roles with versioned work and analysis instructions | `runGraph` |
|
|
16
|
+
| A standard fixed shape such as parallel attempts, a chain, or a review panel | `fanout`, `pipeline`, `verify`, or `panel` |
|
|
17
|
+
| A model decides which workers to create while it works | `supervise` |
|
|
18
|
+
| One profile can complete the task directly | Run that profile without composition |
|
|
19
|
+
|
|
20
|
+
Do not force a dynamic task into a static graph.
|
|
21
|
+
Do not use a graph when a smaller shipped primitive already expresses the work.
|
|
22
|
+
|
|
23
|
+
### Strict authoring decisions (Do not under-graph)
|
|
24
|
+
|
|
25
|
+
- **Cheapness is not the dialect test:** Do not bail to `single-agent` just because a brief sounds trivial (e.g., "write a one-line file"). If the brief implies roles, observers, or a specific tight budget, author the graph.
|
|
26
|
+
- **Budget Floor Traps:** If a brief demands an impossibly "tight" budget (e.g., a few thousand tokens), do not dodge it by dropping to `single-agent`. Author the graph and explicitly set `budget` to the valid measured executor floor.
|
|
27
|
+
- **Identical-Role Parallelism:** If a brief requests N parallel instances of the same role, you MUST create N distinct worker nodes and N `delegates` edges. Do not collapse identical parallel workers into a single node.
|
|
28
|
+
- **Mandatory Analysts:** If a brief requires independent observation, review, or post-settle findings (e.g., "neutral decider", "review by two perspectives", "watch the worker"), you MUST author `analyzes` edges. Do not omit analysts and attempt to merge their logic into the root's prompt.
|
|
29
|
+
- **Caps are not stops:** Do not use an analysis edge `maxTraversals` cap as a global stop condition. To stop after N findings, use `deliverable.check` or `maxTraversals` on a `delegates` edge.
|
|
30
|
+
|
|
31
|
+
## Author the complete contract
|
|
32
|
+
|
|
33
|
+
An `AgentGraph` has four required fields: `nodes`, `edges`, `deliverable`, and `budget`.
|
|
34
|
+
`runGraph(graph, options)` validates graph structure and prompt references before it spends compute.
|
|
35
|
+
|
|
36
|
+
### Nodes
|
|
37
|
+
|
|
38
|
+
Each node is `{ id, profile }`, where `profile` is a complete canonical `AgentProfile`.
|
|
39
|
+
Set `profile.name` equal to `id` because Runtime uses that value to select and route the node.
|
|
40
|
+
Put the standing role in `profile.prompt.systemPrompt` and capabilities in the profile's tools, MCP, resources, hooks, and subagents.
|
|
41
|
+
Do not rebuild profile materialization in graph code.
|
|
42
|
+
|
|
43
|
+
### Delegation edges
|
|
44
|
+
|
|
45
|
+
A delegation edge is `{ kind: 'delegates', from, to, directive, maxTraversals? }`.
|
|
46
|
+
The directive is a registered, versioned `PromptHandle`, such as `promptHandle('delegates/research-brief/v1')`.
|
|
47
|
+
Each spawn and each later steer over the same edge consumes one traversal.
|
|
48
|
+
The default cap is `defaultEdgeTraversalCap`; exhaustion refuses further delegation.
|
|
49
|
+
|
|
50
|
+
The current graph form has one root and a static set of worker nodes.
|
|
51
|
+
Every delegation edge starts at the root, and each worker has exactly one incoming delegation edge.
|
|
52
|
+
Use a new directive version to change a brief instead of adding a second edge to the same worker.
|
|
53
|
+
|
|
54
|
+
### Analysis edges
|
|
55
|
+
|
|
56
|
+
An analysis edge is `{ kind: 'analyzes', analyst, over, to, directive, maxTraversals? }`.
|
|
57
|
+
It runs after a listed worker settles and routes findings to one node.
|
|
58
|
+
|
|
59
|
+
`analyst` has two supported forms:
|
|
60
|
+
|
|
61
|
+
- A lens id from `options.analysts` runs a caller-supplied analysis function.
|
|
62
|
+
- A graph node id runs that node's pinned `AgentProfile` as a tool-equipped analyst.
|
|
63
|
+
|
|
64
|
+
An analyst node has no incoming delegation edge, so the root cannot hand it ordinary work.
|
|
65
|
+
An id cannot be both a registered lens and an analyst node.
|
|
66
|
+
`over` lists delegated worker nodes only; Runtime refuses the root and analyst nodes because neither settles as an ordinary worker.
|
|
67
|
+
An analysis traversal cap records excess findings as `unpropagated`; it does not stop the run.
|
|
68
|
+
|
|
69
|
+
### Completion and budget
|
|
70
|
+
|
|
71
|
+
`deliverable.check(output)` is the independent completion test.
|
|
72
|
+
It must accept a genuinely complete result and reject junk.
|
|
73
|
+
Put the concrete mission in `deliverable.describe`; Runtime uses that text as the root's task.
|
|
74
|
+
|
|
75
|
+
`budget` is one conserved pool for the full graph.
|
|
76
|
+
Set `options.perWorker` explicitly from the actual executor cost.
|
|
77
|
+
For Pi, `WORKER_TOKEN_FLOOR.pi` is 31,211 input tokens before useful work, so a worker allocation below that value is refused. If a brief asks for a budget lower than the floor, do not switch to `single-agent`; output the graph with the floor allocation.
|
|
78
|
+
Treat an unmeasured executor floor as unknown rather than zero.
|
|
79
|
+
Analyst nodes spend from the same pool and need the same honest accounting as ordinary workers.
|
|
80
|
+
|
|
81
|
+
## Authoring procedure
|
|
82
|
+
|
|
83
|
+
1. **Classify correctly:** Verify if this needs `single-agent`, `dynamic-workflow`, or a static `runGraph`. If independent review or parallel workers are requested, use `runGraph`.
|
|
84
|
+
2. **Define completion first:** Write the completion test and its description.
|
|
85
|
+
3. **Select entry point:** Choose the smallest shipped entry point from the table above.
|
|
86
|
+
4. **Define Roles:** Give every distinct role one complete `AgentProfile`. If N parallel instances of a role are requested, create N nodes. Merge roles only if their standing prompts and capabilities are identical.
|
|
87
|
+
5. **Register directives:** Register a versioned directive for every edge.
|
|
88
|
+
6. **Delegate work:** Add one delegation edge per ordinary worker from the root.
|
|
89
|
+
7. **Attach analysts:** Add `analyzes` edges only when findings must be produced independently after a worker settles. Do not skip this if the brief asked for a watcher/reviewer.
|
|
90
|
+
8. **Size the pool:** Set budget, per-worker allocation, traversal caps, time, and concurrency from measured executor behavior. Ensure budgets meet the executor floor.
|
|
91
|
+
9. **Prove and inspect:** Run the structure offline, then run the real backend and inspect its result.
|
|
92
|
+
|
|
93
|
+
## Prove the graph before spending
|
|
94
|
+
|
|
95
|
+
Use an injected `brain` plus `makeWorkerAgent` to exercise graph structure without a network call.
|
|
96
|
+
Cover invalid profiles, unknown directives, impossible analysis routes, traversal exhaustion, successful completion, and rejected junk.
|
|
97
|
+
Start from the runnable programs in `examples/graphs/` rather than creating a second graph runner.
|
|
98
|
+
|
|
99
|
+
Offline execution proves control flow only.
|
|
100
|
+
A real task must still use the intended backend, profiles, tools, completion test, and budget before claiming the graph solves that task.
|
|
101
|
+
|
|
102
|
+
## Read the complete result
|
|
103
|
+
|
|
104
|
+
| Field | Meaning |
|
|
105
|
+
| --- | --- |
|
|
106
|
+
| `result.result.kind` and `reason` | Whether a result won and why execution ended |
|
|
107
|
+
| `result.result.spentTotal` | Tokens and money, including whether each total is known |
|
|
108
|
+
| `result.ledger` | Every delivered, stripped, empty, or unpropagated edge traversal with byte counts |
|
|
109
|
+
| `result.exhaustedEdges` | Every edge whose cap was reached, including normal lifecycle endings |
|
|
110
|
+
| Journal `edge` events | Durable copies of traversal evidence |
|
|
111
|
+
|
|
112
|
+
Zero traversals on an expected edge means the graph did not exercise that relationship.
|
|
113
|
+
`usdKnown: false` means cost is missing, not free.
|
|
114
|
+
A passing completion test proves only what that test checks.
|
|
115
|
+
|
|
116
|
+
## Common mistakes
|
|
117
|
+
|
|
118
|
+
- Bailing to `single-agent` because a brief sounds trivial, instead of respecting requested roles or applying budget floors.
|
|
119
|
+
- Collapsing N requested parallel identical roles into a single worker node.
|
|
120
|
+
- Skipping `analyzes` edges when an observer or reviewer is explicitly requested.
|
|
121
|
+
- Putting the task only in a spawn prompt instead of `deliverable.describe`.
|
|
122
|
+
- Giving a node a `profile.name` different from its id.
|
|
123
|
+
- Delegating ordinary work to an analyst node.
|
|
124
|
+
- Listing the root or an analyst node in `analyzes.over`.
|
|
125
|
+
- Using an analysis cap as a stop condition.
|
|
126
|
+
- Allowing a driver-authored spawn profile to add capabilities instead of defining them on the pinned node profile.
|
|
127
|
+
- Reading only thrown cap errors and missing `result.exhaustedEdges` on budget or cancellation endings.
|
|
128
|
+
- Treating unknown spend as zero.
|
|
129
|
+
- Claiming recursive or runtime-discovered structure when the current graph is a static root with workers and analysts.
|
|
130
|
+
|
|
131
|
+
## Improve only after measurement
|
|
132
|
+
|
|
133
|
+
Runtime already optimizes one inline skill through `improve(profile, { surface: 'skills', skills: { resourceName }, ... })`.
|
|
134
|
+
Put the exact skill bytes in `profile.resources.skills`, set `profile.resources.failOnError: true`, supply disjoint development and final-test tasks, and pass a complete Agent Eval optimization method.
|
|
135
|
+
Do not create a graph-specific optimizer, campaign runner, candidate store, or promotion path.
|
|
136
|
+
|
|
137
|
+
## Then consider
|
|
138
|
+
|
|
139
|
+
- `loop-writer` when the required dynamic structure still cannot be expressed by `supervise` or another shipped primitive; pass the exact missing behavior and the completion test.
|
|
140
|
+
- `verify` before publishing a graph consumer; pass the real backend command, expected result fields, and failure cases.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "artifact-mission-release-notes",
|
|
3
|
+
"brief": "Produce a release-notes file for version 2.0 that passes our repo format checker.",
|
|
4
|
+
"expect": {
|
|
5
|
+
"correctAnswerIsGraph": true,
|
|
6
|
+
"nodes": 1,
|
|
7
|
+
"deliverableDescribeCarriesMission": true,
|
|
8
|
+
"checkIsMechanical": true
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "audited-single-writer",
|
|
3
|
+
"brief": "Have a separate worker write the summary document so I can audit exactly what it did afterwards.",
|
|
4
|
+
"expect": {
|
|
5
|
+
"correctAnswerIsGraph": true,
|
|
6
|
+
"nodes": 1,
|
|
7
|
+
"reason": "auditability warrants delegation despite a cheap-sounding task"
|
|
8
|
+
}
|
|
9
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "floor-trap-pi",
|
|
3
|
+
"brief": "Cheap little task: have a pi agent write a one-line file. Keep the budget tight, like a few thousand tokens.",
|
|
4
|
+
"expect": {
|
|
5
|
+
"nodes": 1,
|
|
6
|
+
"analyzesWarranted": false,
|
|
7
|
+
"floorTrap": true,
|
|
8
|
+
"mustBudgetAtLeast": 31211,
|
|
9
|
+
"correctAuthorOverridesBrief": true
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "review-pipeline",
|
|
3
|
+
"brief": "I want code changes reviewed by two different perspectives before anything merges, and someone neutral deciding.",
|
|
4
|
+
"expect": {
|
|
5
|
+
"nodes": 3,
|
|
6
|
+
"analyzesWarranted": true,
|
|
7
|
+
"floorTrap": false,
|
|
8
|
+
"edges": [
|
|
9
|
+
"delegates to each reviewer",
|
|
10
|
+
"analyzes routing findings to root"
|
|
11
|
+
],
|
|
12
|
+
"wrongIfAnalystIsNode": true
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "steer-heavy-drafting",
|
|
3
|
+
"brief": "One writer drafts, I want the coordinator to redirect it up to five times based on how the draft evolves.",
|
|
4
|
+
"expect": {
|
|
5
|
+
"nodes": 1,
|
|
6
|
+
"maxTraversalsAtLeast": 6,
|
|
7
|
+
"reason": "spawns and steers share the traversal count"
|
|
8
|
+
}
|
|
9
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skillVersion": "v1",
|
|
3
|
+
"surfaceSource": "working-tree",
|
|
4
|
+
"authorModel": "glm-5.2",
|
|
5
|
+
"temperature": 0.2,
|
|
6
|
+
"date": "2026-08-03T17:42:45.828Z",
|
|
7
|
+
"n": 8,
|
|
8
|
+
"aggregate": {
|
|
9
|
+
"mean": 0.6375,
|
|
10
|
+
"median": 0.8,
|
|
11
|
+
"min": 0,
|
|
12
|
+
"max": 1,
|
|
13
|
+
"n": 8,
|
|
14
|
+
"correction": "first publication reported median 1.0 from an upper-middle indexing bug; corrected to the interpolated median",
|
|
15
|
+
"samplePerCase": 1,
|
|
16
|
+
"varianceNote": "decision-level variance observed at temp 0.2 (cap-as-stop-mistake flipped graph->single-agent on rerun); treat per-case scores as n=1 draws, not stable properties. Reps belong to the v2 gate round."
|
|
17
|
+
},
|
|
18
|
+
"cases": [
|
|
19
|
+
{
|
|
20
|
+
"id": "cap-as-stop-mistake",
|
|
21
|
+
"decision": "graph",
|
|
22
|
+
"score": 1,
|
|
23
|
+
"reasons": [
|
|
24
|
+
"PASS trapIsAnalyzesCapAsStop: analyzes caps=true delegates caps=true",
|
|
25
|
+
"PASS correctStopIsDelegatesCapOrDeliverable: delegates caps=true, reason mentions deliverable=true"
|
|
26
|
+
],
|
|
27
|
+
"reason": "The brief asks for an analyst to watch a worker \u2014 a legitimate unaddressable observation lens \u2014 so a static graph is the right dialect. But the brief's stated stop mechanism is the textbook pothole: an analyzes edge's maxTraversals is observability-only; on exhaustion the traversal is ledgered 'unpropagated' and it never refuses, so it cannot terminate a run. Only a delegates cap or the deliverable check can. I therefore encode 'three findings' in the deliverable (the real termination oracle) and let the worker settle once three findings exist; the findings-lens analyst observes and routes to the root with maxTraversals=3 as the intended observation budget, but it is NOT load-bearing for termination. If the brief truly needs hard termination at exactly three analyst findings regardless of worker output, that is not expressible in the current P0 topology and must be re-scoped before running.",
|
|
28
|
+
"authoredGraph": {
|
|
29
|
+
"nodes": [
|
|
30
|
+
{
|
|
31
|
+
"id": "root",
|
|
32
|
+
"systemPrompt": "You are the root orchestrator. Hold the mission: produce at least three distinct findings on the delegated subject. Delegate to the worker, accept the findings-lens observations routed back to you, and stop spawning the moment the worker's settled output contains three findings. Do not delegate further once three are present."
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "worker",
|
|
36
|
+
"systemPrompt": "You are the findings worker. Investigate the delegated brief and emit discrete, numbered findings, one paragraph each. Continue until you have produced at least three distinct findings, then settle. Do not pad or repeat."
|
|
37
|
+
}
|
|
38
|
+
],
|
|
39
|
+
"edges": [
|
|
40
|
+
{
|
|
41
|
+
"kind": "delegates",
|
|
42
|
+
"from": "root",
|
|
43
|
+
"to": "worker",
|
|
44
|
+
"maxTraversals": 6
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"kind": "analyzes",
|
|
48
|
+
"analyst": "findings-lens",
|
|
49
|
+
"over": [
|
|
50
|
+
"worker"
|
|
51
|
+
],
|
|
52
|
+
"to": "root",
|
|
53
|
+
"maxTraversals": 3
|
|
54
|
+
}
|
|
55
|
+
],
|
|
56
|
+
"budget": {
|
|
57
|
+
"maxIterations": 8,
|
|
58
|
+
"maxTokens": 240000
|
|
59
|
+
},
|
|
60
|
+
"perWorker": {
|
|
61
|
+
"maxIterations": 4,
|
|
62
|
+
"maxTokens": 80000
|
|
63
|
+
},
|
|
64
|
+
"deliverableDescribe": "Investigate the delegated subject and report at least three distinct, numbered findings. The run terminates when the worker's settled output contains three findings; the findings-lens analyst observes the worker and routes its observations to the root (observation only \u2014 it does not stop the run)."
|
|
65
|
+
},
|
|
66
|
+
"runResultKind": "winner",
|
|
67
|
+
"ledgerRows": 2,
|
|
68
|
+
"exhaustedEdges": []
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "floor-trap-pi",
|
|
72
|
+
"decision": "single-agent",
|
|
73
|
+
"score": 0,
|
|
74
|
+
"reasons": [
|
|
75
|
+
"FAIL mustBudgetAtLeast: no graph authored (decision=single-agent)",
|
|
76
|
+
"FAIL nodes: no graph authored",
|
|
77
|
+
"FAIL analyzesWarranted: no graph authored"
|
|
78
|
+
],
|
|
79
|
+
"reason": "The task is trivially one-shot-able by a single pi agent (writing a one-line file) without requiring parallel fan-out or independent verification. Furthermore, the brief's tight budget constraint of a few thousand tokens falls drastically below the measured 31,211 input token floor (`WORKER_TOKEN_FLOOR`) required for a `runGraph` spawn, meaning any graph attempt would fail with a `below-runtime-floor` refusal."
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"id": "mission-in-deliverable",
|
|
83
|
+
"decision": "single-agent",
|
|
84
|
+
"score": 0,
|
|
85
|
+
"reasons": [
|
|
86
|
+
"FAIL deliverableDescribeCarriesMission: no graph authored"
|
|
87
|
+
],
|
|
88
|
+
"reason": "A single agent can author a CHANGELOG entry that satisfies a format check in a single pass, requiring no topology, parallelism, or independent verification."
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"id": "review-pipeline",
|
|
92
|
+
"decision": "graph",
|
|
93
|
+
"score": 0.6,
|
|
94
|
+
"reasons": [
|
|
95
|
+
"PASS nodes: workers=2 total=3 expected=3",
|
|
96
|
+
"FAIL analyzesWarranted: analyzes edges=0 warranted=true",
|
|
97
|
+
"PASS wrongIfAnalystIsNode: no analyst id collides with a node id",
|
|
98
|
+
"PASS edge:delegates to each reviewer: workers [security-reviewer, maintainability-reviewer] each delegated-to with >0 ledger traversals: true",
|
|
99
|
+
"FAIL edge:analyzes routing findings to root: analyzes\u2192root edges=0, fired with >0 traversals=false"
|
|
100
|
+
],
|
|
101
|
+
"reason": "The case calls for two genuinely distinct review perspectives (security/correctness vs maintainability) that must settle independently, then a neutral arbiter synthesizing both \u2014 a fixed topology with independent verification that earns composition. One agent cannot maintain two addressable, independently-settled roles plus a neutral third.",
|
|
102
|
+
"authoredGraph": {
|
|
103
|
+
"nodes": [
|
|
104
|
+
{
|
|
105
|
+
"id": "merge-arbiter",
|
|
106
|
+
"systemPrompt": "You are the neutral merge arbiter. You delegate code changes to two independent reviewers \u2014 one for security/correctness, one for maintainability \u2014 then collect both reviews and render a final merge decision: APPROVE, REQUEST-CHANGES, or REJECT. You must cite specific findings from each reviewer in your decision rationale. You do not review code yourself; you synthesize and decide."
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"id": "security-reviewer",
|
|
110
|
+
"systemPrompt": "You are a security and correctness reviewer. Examine code changes for vulnerabilities, input-validation gaps, race conditions, data-integrity risks, logic errors, and unhandled edge cases. Report each finding with a severity rating (critical, high, medium, low) and a concrete location in the diff. Do not comment on style or naming unless it introduces a correctness risk."
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"id": "maintainability-reviewer",
|
|
114
|
+
"systemPrompt": "You are a maintainability and clarity reviewer. Examine code changes for naming quality, structural clarity, test coverage, documentation gaps, convention adherence, and future changeability. Report each finding with a priority rating (blocker, important, minor) and a concrete location in the diff. Do not comment on security unless a pattern directly blocks maintainability."
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"edges": [
|
|
118
|
+
{
|
|
119
|
+
"kind": "delegates",
|
|
120
|
+
"from": "merge-arbiter",
|
|
121
|
+
"to": "security-reviewer",
|
|
122
|
+
"maxTraversals": 2
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"kind": "delegates",
|
|
126
|
+
"from": "merge-arbiter",
|
|
127
|
+
"to": "maintainability-reviewer",
|
|
128
|
+
"maxTraversals": 2
|
|
129
|
+
}
|
|
130
|
+
],
|
|
131
|
+
"budget": {
|
|
132
|
+
"maxIterations": 6,
|
|
133
|
+
"maxTokens": 300000
|
|
134
|
+
},
|
|
135
|
+
"perWorker": {
|
|
136
|
+
"maxIterations": 3,
|
|
137
|
+
"maxTokens": 80000
|
|
138
|
+
},
|
|
139
|
+
"deliverableDescribe": "Review the provided code change from two independent perspectives \u2014 security/correctness and maintainability \u2014 then produce a merge decision (APPROVE, REQUEST-CHANGES, or REJECT) with a rationale that cites specific findings from each reviewer."
|
|
140
|
+
},
|
|
141
|
+
"runResultKind": "winner",
|
|
142
|
+
"ledgerRows": 2,
|
|
143
|
+
"exhaustedEdges": []
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"id": "runtime-discovered-fanout",
|
|
147
|
+
"decision": "dynamic-workflow",
|
|
148
|
+
"score": 1,
|
|
149
|
+
"reasons": [
|
|
150
|
+
"PASS correctAnswerIsDynamicWorkflow: decision=dynamic-workflow"
|
|
151
|
+
],
|
|
152
|
+
"reason": "The task requires identifying an unknown number of failing tests at runtime and fixing them in parallel, which means the topology is discovered while working. A static runGraph cannot add nodes at runtime."
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"id": "single-agent-suffices",
|
|
156
|
+
"decision": "single-agent",
|
|
157
|
+
"score": 1,
|
|
158
|
+
"reasons": [
|
|
159
|
+
"PASS correctAnswerIsNoGraph: decision=single-agent"
|
|
160
|
+
],
|
|
161
|
+
"reason": "Summarizing a document into five bullets is a one-shot task that a single strong agent can fully complete. There is no parallelism, no independent verification, and no topology to express \u2014 the deliverable check (five bullet points) is trivially satisfiable by one pass. Per the skill, if one agent could produce bytes that pass the check, you do not need a graph; a harness-driven root costs ~11\u00d7 a plain router loop, so composition does not earn its cost here."
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"id": "steer-heavy-drafting",
|
|
165
|
+
"decision": "graph",
|
|
166
|
+
"score": 1,
|
|
167
|
+
"reasons": [
|
|
168
|
+
"PASS nodes: workers=1 total=2 expected=1",
|
|
169
|
+
"PASS maxTraversalsAtLeast: effective delegates cap 6 (default 32 when unset) vs \u22656"
|
|
170
|
+
],
|
|
171
|
+
"reason": "Topology is fully known before running: one coordinator root delegates to one writer worker in a cyclic steer loop. The requirement is an explicit five-redirect cap with a ledgered audit trail \u2014 exactly what a static runGraph provides. This is not a single-agent case (the coordinator and writer have distinct, differentiable standing prompts and the coordinator must independently verify the writer's output) and not a dynamic-workflow case (no fan-out is discovered mid-run; the topology is fixed at author time).",
|
|
172
|
+
"authoredGraph": {
|
|
173
|
+
"nodes": [
|
|
174
|
+
{
|
|
175
|
+
"id": "coordinator",
|
|
176
|
+
"systemPrompt": "You are the coordinator supervising a single writer. On each cycle you receive the writer's latest draft, evaluate it against the mission, and either accept the draft as final or issue a single concise redirect directive that names exactly what the writer must change and why. You may redirect at most five times \u2014 every redirect consumes one of those five, so be specific and decisive. Declare the draft done only when it satisfies the mission; never accept a draft that still misses a stated requirement. A vague or incomplete redirect wastes a traversal you cannot get back."
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
"id": "writer",
|
|
180
|
+
"systemPrompt": "You are the writer. On first contact produce a full draft from the mission. On every subsequent contact revise the existing draft to address the coordinator's redirect directive precisely, then return the complete updated draft. Never return a diff, a summary, or a placeholder \u2014 always the full text."
|
|
181
|
+
}
|
|
182
|
+
],
|
|
183
|
+
"edges": [
|
|
184
|
+
{
|
|
185
|
+
"kind": "delegates",
|
|
186
|
+
"from": "coordinator",
|
|
187
|
+
"to": "writer",
|
|
188
|
+
"maxTraversals": 6
|
|
189
|
+
}
|
|
190
|
+
],
|
|
191
|
+
"budget": {
|
|
192
|
+
"maxIterations": 12,
|
|
193
|
+
"maxTokens": 600000
|
|
194
|
+
},
|
|
195
|
+
"perWorker": {
|
|
196
|
+
"maxIterations": 8,
|
|
197
|
+
"maxTokens": 400000
|
|
198
|
+
},
|
|
199
|
+
"deliverableDescribe": "Supervise the writer to produce a finished draft that fully satisfies the mission. After each draft the writer returns, evaluate it and, if it falls short, redirect the writer with a specific change directive. You may redirect up to five times \u2014 each redirect must state concrete, actionable changes, not vague preferences. Declare the draft done only when every mission requirement is met. The deliverable is the accepted final draft text."
|
|
200
|
+
},
|
|
201
|
+
"runResultKind": "winner",
|
|
202
|
+
"ledgerRows": 1,
|
|
203
|
+
"exhaustedEdges": []
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"id": "unmeasured-harness",
|
|
207
|
+
"decision": "graph",
|
|
208
|
+
"score": 0.5,
|
|
209
|
+
"reasons": [
|
|
210
|
+
"FAIL nodes: workers=1 total=2 expected=3",
|
|
211
|
+
"PASS generousBudgetsBecauseFloorUnknown: per-child tokens 150000 vs generous line 50000"
|
|
212
|
+
],
|
|
213
|
+
"reason": "The topology is known ahead of time (root executing three probes via a worker) and needs an audit trail with strict budget caps, especially given the unmeasured harness. A single worker node represents the probe role, and capping its traversals enforces exactly three probe executions.",
|
|
214
|
+
"authoredGraph": {
|
|
215
|
+
"nodes": [
|
|
216
|
+
{
|
|
217
|
+
"id": "root",
|
|
218
|
+
"systemPrompt": "You are the root orchestrator. Your role is to coordinate the execution of distinct probes and aggregate their outputs."
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
"id": "probe-worker",
|
|
222
|
+
"systemPrompt": "You are a probe worker running on an unmeasured harness. Execute the specific probe you are assigned and return the raw output."
|
|
223
|
+
}
|
|
224
|
+
],
|
|
225
|
+
"edges": [
|
|
226
|
+
{
|
|
227
|
+
"kind": "delegates",
|
|
228
|
+
"from": "root",
|
|
229
|
+
"to": "probe-worker",
|
|
230
|
+
"maxTraversals": 3
|
|
231
|
+
}
|
|
232
|
+
],
|
|
233
|
+
"budget": {
|
|
234
|
+
"maxIterations": 10,
|
|
235
|
+
"maxTokens": 600000
|
|
236
|
+
},
|
|
237
|
+
"perWorker": {
|
|
238
|
+
"maxIterations": 5,
|
|
239
|
+
"maxTokens": 150000
|
|
240
|
+
},
|
|
241
|
+
"deliverableDescribe": "Run three probes on claude-code workers and collect what they output."
|
|
242
|
+
},
|
|
243
|
+
"runResultKind": "winner",
|
|
244
|
+
"ledgerRows": 1,
|
|
245
|
+
"exhaustedEdges": []
|
|
246
|
+
}
|
|
247
|
+
]
|
|
248
|
+
}
|