opencode-harness-kit 0.2.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +301 -0
- package/GLOBAL.md +50 -13
- package/MIGRATION.md +140 -0
- package/README.md +276 -122
- package/bin/oc-harness.mjs +571 -135
- package/dist/index.js +453 -159
- package/docs/KNOWN-LIMITS.md +189 -4
- package/docs/RESEARCH.md +80 -5
- package/docs/ROUTING.md +254 -0
- package/lanes.json +136 -0
- package/lanes.schema.json +55 -0
- package/package.json +13 -11
- package/profiles/beam-free.json +51 -0
- package/profiles/big-pickle.json +45 -0
- package/profiles/deepseek-v4.1-flash-free.json +49 -0
- package/profiles/fleet-all.json +139 -0
- package/profiles/fleet-safe.json +139 -0
- package/profiles/glm-5.3-flash-free.json +49 -0
- package/profiles/index.json +120 -37
- package/profiles/longcat-2.5-preview-free.json +14 -10
- package/profiles/mimo-v2.6-flash-free.json +44 -0
- package/profiles/minimax-m3.1-flash-free.json +51 -0
- package/profiles/nemotron-3-ultra-free.json +44 -0
- package/profiles/nemotron-3.5-lightning-free.json +44 -0
- package/profiles/nemotron-3.5-super-vl-free.json +52 -0
- package/profiles/schema.json +30 -11
- package/{agents → prompts}/build.md +2 -35
- package/prompts/doc.md +42 -0
- package/{agents → prompts}/locate.md +1 -16
- package/prompts/longform.md +34 -0
- package/{agents → prompts}/plan.md +2 -25
- package/{agents → prompts}/reviewer.md +2 -22
- package/prompts/schema.md +30 -0
- package/prompts/sweep.md +29 -0
- package/{agents → prompts}/verify.md +6 -29
- package/prompts/vision.md +33 -0
- package/retention.yml +75 -57
- package/skills/space-build/SKILL.md +16 -0
- package/skills/space-finish/SKILL.md +0 -10
- package/skills/space-review/SKILL.md +8 -0
- package/src/config.js +94 -69
- package/src/driver/run.mjs +237 -0
- package/src/effort.js +59 -22
- package/src/index.ts +15 -2
- package/src/lane-usage.js +142 -0
- package/src/profile.js +615 -221
- package/src/router.ts +148 -39
- package/src/tasks.js +248 -0
- package/src/telemetry.ts +98 -0
- package/docs/demo/demo.gif +0 -0
- package/docs/demo/demo.mp4 +0 -0
- package/docs/demo/render.py +0 -146
- package/examples/muse-spark-1.3-contributor-free.json +0 -41
- package/profiles/deepseek-v4-flash-free.json +0 -41
- package/profiles/muse-split-free.json +0 -64
- package/profiles/space-bunny-free.json +0 -42
- package/profiles/x-preview-f-free.json +0 -41
package/src/config.js
CHANGED
|
@@ -3,8 +3,31 @@
|
|
|
3
3
|
// Config that does not depend on the model - permission allowlist, pruning
|
|
4
4
|
// defaults, experimental flags - is fixed here. Everything model-specific is
|
|
5
5
|
// derived from the profile, so switching models changes only the profile file.
|
|
6
|
+
//
|
|
7
|
+
// A fleet profile turns several models on at once, and there is one hard
|
|
8
|
+
// constraint that shapes the whole file:
|
|
9
|
+
//
|
|
10
|
+
// The model lives in the CONFIG, never in a markdown agent file.
|
|
11
|
+
//
|
|
12
|
+
// An agent file of the same name replaces the config entry wholesale, and a
|
|
13
|
+
// file without a `model:` line makes the agent inherit the caller's model
|
|
14
|
+
// instead. Both were verified against OpenCode 1.19: a conflicting file won,
|
|
15
|
+
// and a file with no model ran on the parent session's model. So the lanes
|
|
16
|
+
// live in `agent.<lane>` here, and their prompt bodies live in prompts/ and
|
|
17
|
+
// are referenced with a path opencode can resolve.
|
|
6
18
|
|
|
7
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
pruneThreshold,
|
|
21
|
+
reservedTokens,
|
|
22
|
+
inputCap,
|
|
23
|
+
effectiveRetention,
|
|
24
|
+
resolveRoleModel,
|
|
25
|
+
resolveRoles,
|
|
26
|
+
resolveDefaultVariant,
|
|
27
|
+
roleSplitModels,
|
|
28
|
+
promptRef,
|
|
29
|
+
loadLanes,
|
|
30
|
+
} from "./profile.js"
|
|
8
31
|
|
|
9
32
|
import { join } from "node:path"
|
|
10
33
|
import { homedir } from "node:os"
|
|
@@ -67,8 +90,8 @@ export const BASH_ALLOWLIST = [
|
|
|
67
90
|
"git rev-parse*", "git config --get*", "git remote*", "git blame*",
|
|
68
91
|
"git stash list*", "git ls-files*", "git describe*", "git shortlog*", "git --version*",
|
|
69
92
|
// `git -C <dir> <read-verb>` only. A bare `git -C *` would allow `git -C x push`.
|
|
70
|
-
"git -C * status*", "git -C * log*", "git -C * diff*",
|
|
71
|
-
"git -C * ls-files*", "git -C * show*",
|
|
93
|
+
"git -C * status*", "git -C * log*", "git -C * diff*",
|
|
94
|
+
"git -C * rev-parse*", "git -C * ls-files*", "git -C * show*",
|
|
72
95
|
// Staging is local and reversible; publishing is not and stays gated.
|
|
73
96
|
"git add*",
|
|
74
97
|
// `chmod 777` is denied below. `+x` is ordinary build work.
|
|
@@ -102,66 +125,62 @@ export const BASH_DENYLIST = [
|
|
|
102
125
|
"gh repo delete*",
|
|
103
126
|
]
|
|
104
127
|
|
|
105
|
-
/**
|
|
106
|
-
* Iteration cap per role, written as `agent.<role>.steps`.
|
|
107
|
-
*
|
|
108
|
-
* `steps` is a loop breaker rather than a quality knob. On exhaustion OpenCode
|
|
109
|
-
* forces a text-only response, so an agent that has stopped making progress
|
|
110
|
-
* reports what it has instead of silently looping until the context is gone.
|
|
111
|
-
* The caps are sized to the role's real work: build gets room because it runs
|
|
112
|
-
* the edit-verify-fix cycle, verify gets room because a full suite is many
|
|
113
|
-
* commands, and the read-only roles are capped tight because they have one job.
|
|
114
|
-
*/
|
|
115
|
-
export const AGENT_STEPS = /** @type {Record<string, number>} */ ({
|
|
116
|
-
build: 40,
|
|
117
|
-
plan: 25,
|
|
118
|
-
reviewer: 15,
|
|
119
|
-
locate: 12,
|
|
120
|
-
verify: 20,
|
|
121
|
-
})
|
|
122
|
-
|
|
123
|
-
/** Roles the kit writes, with the permissions that keep them honest. */
|
|
124
|
-
const AGENT_PERMISSIONS = /** @type {Record<string, Record<string, string>>} */ ({
|
|
125
|
-
plan: { edit: "deny" },
|
|
126
|
-
reviewer: { edit: "deny" },
|
|
127
|
-
locate: { edit: "deny", bash: "deny" },
|
|
128
|
-
verify: { edit: "deny" },
|
|
129
|
-
})
|
|
130
|
-
|
|
131
128
|
/**
|
|
132
129
|
* @param {Profile} p
|
|
133
|
-
* @param {{ base?: Record<string, any>, comments?: boolean }} [opts]
|
|
130
|
+
* @param {{ base?: Record<string, any>, comments?: boolean, unhealthy?: Set<string> }} [opts]
|
|
134
131
|
* @returns {string}
|
|
135
132
|
*/
|
|
136
133
|
export function generateConfig(p, opts = {}) {
|
|
137
134
|
const base = opts.base ?? {}
|
|
135
|
+
const unhealthy = opts.unhealthy
|
|
138
136
|
const model = `${p.provider}/${p.model}`
|
|
139
|
-
const roles = resolveRoles(p)
|
|
137
|
+
const roles = resolveRoles(p, { unhealthy })
|
|
140
138
|
const instructionsPath = globalInstructionsPath()
|
|
139
|
+
const lanes = loadLanes()
|
|
141
140
|
|
|
142
141
|
const agent = { ...(base.agent ?? {}) }
|
|
142
|
+
/** Lanes the primary agents are allowed to delegate to. */
|
|
143
|
+
const delegable = Object.keys(roles).filter(
|
|
144
|
+
(r) => lanes[r]?.mode !== "primary",
|
|
145
|
+
)
|
|
146
|
+
|
|
143
147
|
for (const [role, level] of Object.entries(roles)) {
|
|
144
148
|
if (level === undefined) continue
|
|
149
|
+
const lane = lanes[role]
|
|
145
150
|
const existing = { ...(agent[role] ?? {}) }
|
|
146
|
-
|
|
151
|
+
|
|
147
152
|
// The model is pinned, never inherited. Preserving a model already written
|
|
148
153
|
// on the agent is how `build` ended up on a `may-train` model while the rest
|
|
149
154
|
// of the file claimed the profile's zero-retention one: the compiler was
|
|
150
155
|
// asked to re-run over a hand-edited config and kept the hand-edit.
|
|
151
|
-
|
|
152
|
-
// `roleModels` is the one sanctioned way a role differs, because it is
|
|
153
|
-
// declared in the profile next to the model it points at and validated
|
|
154
|
-
// against retention.yml. Still an assignment, never a preserved value.
|
|
156
|
+
const rm = resolveRoleModel(p, role, { unhealthy })
|
|
155
157
|
existing.model = rm ? `${p.provider}/${rm.model}` : model
|
|
158
|
+
|
|
156
159
|
if (level === null) delete existing.variant
|
|
157
160
|
else existing.variant = level
|
|
158
|
-
|
|
161
|
+
|
|
162
|
+
if (lane) {
|
|
163
|
+
if (lane.description) existing.description = lane.description
|
|
164
|
+
if (lane.mode) existing.mode = lane.mode
|
|
165
|
+
if (lane.steps) existing.steps = lane.steps
|
|
166
|
+
existing.prompt = promptRef(role)
|
|
167
|
+
if (lane.permission) {
|
|
168
|
+
existing.permission = {
|
|
169
|
+
...(existing.permission ?? {}),
|
|
170
|
+
...lane.permission,
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// A primary agent may hand work to a lane. The rule is per-agent, and it
|
|
176
|
+
// is last-match-wins, so the allow list has to come after the deny.
|
|
177
|
+
if (lane?.mode === "primary" && delegable.length) {
|
|
159
178
|
existing.permission = {
|
|
160
179
|
...(existing.permission ?? {}),
|
|
161
|
-
...
|
|
180
|
+
task: { "*": "deny", ...Object.fromEntries(delegable.map((r) => [r, "allow"])) },
|
|
162
181
|
}
|
|
163
182
|
}
|
|
164
|
-
|
|
183
|
+
|
|
165
184
|
agent[role] = existing
|
|
166
185
|
}
|
|
167
186
|
|
|
@@ -179,7 +198,7 @@ export function generateConfig(p, opts = {}) {
|
|
|
179
198
|
delete compaction.tail_turns
|
|
180
199
|
compaction.auto = true
|
|
181
200
|
compaction.prune = true
|
|
182
|
-
compaction.reserved = reservedTokens(p)
|
|
201
|
+
compaction.reserved = reservedTokens(p, { unhealthy })
|
|
183
202
|
|
|
184
203
|
const base_ = { ...base }
|
|
185
204
|
// Two keys are dropped from the inherited config outright rather than merged.
|
|
@@ -202,7 +221,7 @@ export function generateConfig(p, opts = {}) {
|
|
|
202
221
|
// No top-level `variant`. The 1.18.35 Config schema sets
|
|
203
222
|
// `additionalProperties: false` and has no `variant` key, so OpenCode
|
|
204
223
|
// strips it from the resolved config - verified with `opencode debug
|
|
205
|
-
// config`. The per-
|
|
224
|
+
// config`. The per-lane tiering is carried by `agent.<lane>.variant`,
|
|
206
225
|
// which is a real schema key, and `effort.js` sets the session default at
|
|
207
226
|
// the `config` hook where the runtime does still see it.
|
|
208
227
|
default_agent: base.default_agent ?? "build",
|
|
@@ -240,11 +259,11 @@ export function generateConfig(p, opts = {}) {
|
|
|
240
259
|
// `allow` here is load-bearing in a way that is easy to get backwards, and
|
|
241
260
|
// it is pinned rather than inherited for the same reason the model is: an
|
|
242
261
|
// inherited `ask` survives regeneration and re-breaks every read-only
|
|
243
|
-
//
|
|
262
|
+
// lane.
|
|
244
263
|
//
|
|
245
264
|
// OpenCode appends the global rule AFTER the agent's own, and last match
|
|
246
265
|
// wins, so a global `edit: "ask"` silently overrides every agent's
|
|
247
|
-
// `edit: "deny"` and the read-only
|
|
266
|
+
// `edit: "deny"` and the read-only lanes can edit files after a prompt.
|
|
248
267
|
// `allow` is the implicit default and emits no rule at all, which leaves
|
|
249
268
|
// the per-agent `deny` intact. Writes outside the project are still
|
|
250
269
|
// gated by `external_directory`, which defaults to `ask`.
|
|
@@ -265,33 +284,37 @@ export function generateConfig(p, opts = {}) {
|
|
|
265
284
|
harness: {
|
|
266
285
|
profile: p.id,
|
|
267
286
|
...(p.fallback?.length ? { fallback: p.fallback[0] } : {}),
|
|
268
|
-
threshold: pruneThreshold(p),
|
|
287
|
+
threshold: pruneThreshold(p, { unhealthy }),
|
|
269
288
|
},
|
|
270
289
|
}
|
|
271
290
|
|
|
272
|
-
return render(out, p, { comments: opts.comments ?? true, instructionsPath })
|
|
291
|
+
return render(out, p, { comments: opts.comments ?? true, instructionsPath, unhealthy })
|
|
273
292
|
}
|
|
274
293
|
|
|
275
294
|
/**
|
|
276
295
|
* @param {Record<string, any>} cfg
|
|
277
296
|
* @param {Profile} p
|
|
278
|
-
* @param {{ comments: boolean, instructionsPath?: string }} opts
|
|
297
|
+
* @param {{ comments: boolean, instructionsPath?: string, unhealthy?: Set<string> }} opts
|
|
279
298
|
* @returns {string}
|
|
280
299
|
*/
|
|
281
300
|
function render(cfg, p, opts) {
|
|
282
301
|
const c = opts.comments ? "// " : ""
|
|
283
302
|
const instructionsPath = opts.instructionsPath ?? globalInstructionsPath()
|
|
284
|
-
const
|
|
303
|
+
const unhealthy = opts.unhealthy
|
|
304
|
+
const lanes = loadLanes()
|
|
285
305
|
const levels = p.effort?.levels ?? []
|
|
286
|
-
const roles = resolveRoles(p)
|
|
287
|
-
|
|
288
|
-
const effective = effectiveRetention(p)
|
|
289
|
-
// The input cap that actually governs context management, which on a
|
|
290
|
-
// profile is the
|
|
291
|
-
// it, or `reserved` and `prune` read as arithmetic against a number that
|
|
292
|
-
// not the one they used.
|
|
293
|
-
const effectiveInput = inputCap(p)
|
|
306
|
+
const roles = resolveRoles(p, { unhealthy })
|
|
307
|
+
|
|
308
|
+
const effective = effectiveRetention(p, { unhealthy })
|
|
309
|
+
// The input cap that actually governs context management, which on a fleet
|
|
310
|
+
// profile is the NARROWEST lane model rather than the base. The header must
|
|
311
|
+
// name it, or `reserved` and `prune` read as arithmetic against a number that
|
|
312
|
+
// is not the one they used.
|
|
313
|
+
const effectiveInput = inputCap(p, { unhealthy })
|
|
294
314
|
const splits = Object.keys(p.roleModels ?? {})
|
|
315
|
+
const laneRows = roleSplitModels(p, { unhealthy })
|
|
316
|
+
const isFleet = splits.length > 0
|
|
317
|
+
|
|
295
318
|
const head = [
|
|
296
319
|
`${c}Generated by opencode-harness-kit. Edit profiles/${p.id}.json instead`,
|
|
297
320
|
`${c}of this file, then re-run: oc-harness use ${p.id}`,
|
|
@@ -300,35 +323,37 @@ function render(cfg, p, opts) {
|
|
|
300
323
|
`${c}model ${p.provider}/${p.model}`,
|
|
301
324
|
...(splits.length
|
|
302
325
|
? [
|
|
303
|
-
`${c}
|
|
304
|
-
`${c} (roles not listed above run on the base model above)`,
|
|
326
|
+
`${c}lanes ${splits.length} enabled, each on the first model its chain allows`,
|
|
305
327
|
]
|
|
306
328
|
: []),
|
|
307
|
-
`${c}context ${p.limits.context.toLocaleString("en-US")} tokens`,
|
|
308
|
-
`${c}input cap ${
|
|
329
|
+
`${c}context ${p.limits.context.toLocaleString("en-US")} tokens${isFleet ? " (base model)" : ""}`,
|
|
330
|
+
`${c}input cap ${effectiveInput.toLocaleString("en-US")} tokens${effectiveInput !== (p.limits.input ?? p.limits.context) ? " (narrowest lane; governs the shared reserve)" : ""}`,
|
|
309
331
|
`${c}output cap ${p.limits.output.toLocaleString("en-US")} tokens`,
|
|
310
332
|
`${c}reasoning ${levels.length ? levels.join(" < ") : "(not tunable)"}`,
|
|
311
|
-
// The header must name the retention that actually applies, which with a
|
|
312
|
-
// split is the least private model in the file, not the base one. Printing
|
|
313
|
-
// `p.retention` here is the same class of bug as the inherited `build`
|
|
314
|
-
// model: the file claims a privacy property the agents do not have.
|
|
315
333
|
`${c}retention ${effective.retention}${effective.retention === "may-train" ? " <-- prompts may train a model" : ""}`,
|
|
316
334
|
...(effective.retention !== p.retention
|
|
317
335
|
? [
|
|
318
336
|
`${c} base ${p.provider}/${p.model} is ${p.retention}, but ${effective.worst} makes the whole file ${effective.retention}`,
|
|
319
337
|
]
|
|
320
338
|
: []),
|
|
339
|
+
...(p.maxRetention && p.maxRetention !== p.retention
|
|
340
|
+
? [
|
|
341
|
+
`${c}ceiling ${p.maxRetention} - a lane may only resolve to a model at or below this`,
|
|
342
|
+
]
|
|
343
|
+
: []),
|
|
321
344
|
`${c}`,
|
|
322
345
|
`${c}Derived numbers, do not hand-edit:`,
|
|
323
|
-
`${c} compaction.reserved ${reservedTokens(p).toLocaleString("en-US")} (${Math.round((p.effort?.reserveShare ?? 0.12) * 100)}% of the
|
|
324
|
-
`${c} prune threshold ${pruneThreshold(p).toLocaleString("en-US")} (
|
|
346
|
+
`${c} compaction.reserved ${reservedTokens(p, { unhealthy }).toLocaleString("en-US")} (${Math.round((p.effort?.reserveShare ?? 0.12) * 100)}% of the narrowest input cap in play, held back for reasoning growth)`,
|
|
347
|
+
`${c} prune threshold ${pruneThreshold(p, { unhealthy }).toLocaleString("en-US")} (narrowest input cap minus reserve)`,
|
|
325
348
|
`${c} role effort ${Object.entries(roles).map(([k, v]) => `${k}=${v ?? "unset"}`).join(" ")}`,
|
|
326
|
-
...(
|
|
349
|
+
...(laneRows.length
|
|
327
350
|
? [
|
|
328
|
-
`${c}
|
|
351
|
+
`${c} lane models ${laneRows.map((r) => `${r.role}=${r.model}${r.limits ? ` (out ${r.limits.output.toLocaleString("en-US")})` : ""}`).join(" ")}`,
|
|
329
352
|
]
|
|
330
353
|
: []),
|
|
331
|
-
`${c} agent steps ${Object.entries(
|
|
354
|
+
`${c} agent steps ${Object.entries(roles)
|
|
355
|
+
.map(([k]) => `${k}=${lanes[k]?.steps ?? 0}`)
|
|
356
|
+
.join(" ")}`,
|
|
332
357
|
]
|
|
333
358
|
|
|
334
359
|
if (!levels.length) {
|
|
@@ -336,7 +361,7 @@ function render(cfg, p, opts) {
|
|
|
336
361
|
} else {
|
|
337
362
|
head.push(
|
|
338
363
|
`${c}`,
|
|
339
|
-
`${c}Effort is per-
|
|
364
|
+
`${c}Effort is per-lane above. There is no top-level "variant" key: the 1.18.35`,
|
|
340
365
|
`${c}Config schema rejects it, and the router sets the session default at runtime.`,
|
|
341
366
|
)
|
|
342
367
|
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// `oc-harness run` -- drive the space-plan -> build -> review -> finish loop
|
|
3
|
+
// without a human at the keyboard.
|
|
4
|
+
//
|
|
5
|
+
// The four skills already know how to do their own job. What was missing was
|
|
6
|
+
// the thing that decides *which* one runs next and *when to stop*. That is
|
|
7
|
+
// here, and it is deliberately small: it reads `.space/tasks.md`, hands one
|
|
8
|
+
// task at a time to `opencode run --agent <role>`, records the real verify
|
|
9
|
+
// output, and stops.
|
|
10
|
+
//
|
|
11
|
+
// What it will not do, ever:
|
|
12
|
+
// - push, open a PR, or cut a release. Publishing is a human decision, and an
|
|
13
|
+
// unattended loop that can do it is an unattended loop that will.
|
|
14
|
+
// - retry a task it already blocked, or start a task with no verify command.
|
|
15
|
+
// A task it cannot check is a task it cannot claim is done.
|
|
16
|
+
// - loop forever. Every stop condition is explicit and every one of them is
|
|
17
|
+
// reported, so a run that ends is never a mystery.
|
|
18
|
+
|
|
19
|
+
import { spawnSync } from "node:child_process"
|
|
20
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs"
|
|
21
|
+
import { join } from "node:path"
|
|
22
|
+
|
|
23
|
+
import {
|
|
24
|
+
parseTasks,
|
|
25
|
+
nextPending,
|
|
26
|
+
markTask,
|
|
27
|
+
taskPrompt,
|
|
28
|
+
summarize,
|
|
29
|
+
isBlocked,
|
|
30
|
+
} from "../tasks.js"
|
|
31
|
+
|
|
32
|
+
const TASKS_PATH = join(".space", "tasks.md")
|
|
33
|
+
|
|
34
|
+
const bold = (s) => `\x1b[1m${s}\x1b[0m`
|
|
35
|
+
const stamp = (s) => `\x1b[2m${s}\x1b[0m`
|
|
36
|
+
const green = (s) => `\x1b[32m${s}\x1b[0m`
|
|
37
|
+
const yellow = (s) => `\x1b[33m${s}\x1b[0m`
|
|
38
|
+
const red = (s) => `\x1b[31m${s}\x1b[0m`
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Run one command to completion and report what happened.
|
|
42
|
+
*
|
|
43
|
+
* Never throws: the caller has to be able to write the failure into the task
|
|
44
|
+
* file, which is the only record anyone gets of why a task did not land.
|
|
45
|
+
*/
|
|
46
|
+
export function sh(command, { cwd, timeoutMs = 900_000 } = {}) {
|
|
47
|
+
const started = Date.now()
|
|
48
|
+
const res = spawnSync(command, {
|
|
49
|
+
shell: true,
|
|
50
|
+
cwd,
|
|
51
|
+
timeout: timeoutMs,
|
|
52
|
+
encoding: "utf8",
|
|
53
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
54
|
+
env: { ...process.env, NO_COLOR: process.env.NO_COLOR ?? "" },
|
|
55
|
+
})
|
|
56
|
+
const stdout = `${res.stdout ?? ""}${res.stderr ?? ""}`.trimEnd()
|
|
57
|
+
return {
|
|
58
|
+
ok: res.status === 0,
|
|
59
|
+
status: res.status,
|
|
60
|
+
// A spawn that never started has no stderr; say so instead of implying the
|
|
61
|
+
// command ran and said nothing.
|
|
62
|
+
output: stdout || (res.error ? String(res.error.message) : ""),
|
|
63
|
+
seconds: Math.round((Date.now() - started) / 1000),
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The tail of a verify run -- the last few lines are the verdict. */
|
|
68
|
+
export function tail(text, n = 12) {
|
|
69
|
+
const lines = String(text).split("\n").filter((l) => l.trim())
|
|
70
|
+
return lines.slice(-n).join("\n")
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Ask `opencode` to do the task. Returns the transcript so a human can read what
|
|
75
|
+
* the agent actually did rather than being told about it.
|
|
76
|
+
*/
|
|
77
|
+
export function invokeAgent(role, prompt, { cwd, timeoutMs }) {
|
|
78
|
+
const res = spawnSync(
|
|
79
|
+
"opencode",
|
|
80
|
+
["run", "--agent", role, "--format", "json", prompt],
|
|
81
|
+
{
|
|
82
|
+
cwd,
|
|
83
|
+
encoding: "utf8",
|
|
84
|
+
timeout: timeoutMs,
|
|
85
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
return {
|
|
89
|
+
ok: res.status === 0,
|
|
90
|
+
status: res.status,
|
|
91
|
+
output: `${res.stdout ?? ""}${res.stderr ?? ""}`.trimEnd(),
|
|
92
|
+
error: res.error ? String(res.error.message) : "",
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The agent's own instructions for one task, plus what the loop expects back. */
|
|
97
|
+
export function buildPrompt(task) {
|
|
98
|
+
return [
|
|
99
|
+
`You are the space-build agent. Do exactly this one task and stop.`,
|
|
100
|
+
``,
|
|
101
|
+
taskPrompt(task),
|
|
102
|
+
``,
|
|
103
|
+
task.verify
|
|
104
|
+
? `When the change is done, run this command yourself: ${task.verify}`
|
|
105
|
+
: `There is no verify command for this task. Do the work, say what you changed, and stop.`,
|
|
106
|
+
``,
|
|
107
|
+
`Rules for this run:`,
|
|
108
|
+
` - Change only the files named above. If the task is wrong or blocked, say so and stop.`,
|
|
109
|
+
` - Never run git push, never create a PR, never publish. A human does that.`,
|
|
110
|
+
` - Report the real output of the verify command. Never invent a passing result.`,
|
|
111
|
+
].join("\n")
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Run the loop.
|
|
116
|
+
*
|
|
117
|
+
* Stops at the first of: nothing pending, nothing verifiable, a blocked task, a
|
|
118
|
+
* failing verify, the round cap, or the time budget. Whichever it was gets
|
|
119
|
+
* printed, so a run that stops early always says why.
|
|
120
|
+
*/
|
|
121
|
+
export function runLoop({
|
|
122
|
+
cwd = process.cwd(),
|
|
123
|
+
role = "build",
|
|
124
|
+
maxRounds = 20,
|
|
125
|
+
budgetMinutes = 120,
|
|
126
|
+
runAgent = invokeAgent,
|
|
127
|
+
runVerify = sh,
|
|
128
|
+
} = {}) {
|
|
129
|
+
const started = Date.now()
|
|
130
|
+
const budgetMs = budgetMinutes * 60_000
|
|
131
|
+
const log = []
|
|
132
|
+
|
|
133
|
+
if (!existsSync(join(cwd, TASKS_PATH))) {
|
|
134
|
+
return stop(log, {
|
|
135
|
+
reason: `no task file at ${TASKS_PATH}. Run space-plan first -- this loop needs a plan to walk.`,
|
|
136
|
+
code: 1,
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
let text = readFileSync(join(cwd, TASKS_PATH), "utf8")
|
|
141
|
+
let rounds = 0
|
|
142
|
+
|
|
143
|
+
for (;;) {
|
|
144
|
+
const parsed = parseTasks(text)
|
|
145
|
+
const task = nextPending(parsed)
|
|
146
|
+
|
|
147
|
+
if (!task) {
|
|
148
|
+
const s = summarize(parsed)
|
|
149
|
+
return stop(log, {
|
|
150
|
+
reason:
|
|
151
|
+
s.blocked > 0
|
|
152
|
+
? `nothing left to run: ${s.done} done, ${s.blocked} blocked.`
|
|
153
|
+
: `every task is done (${s.done}/${s.total}).`,
|
|
154
|
+
code: s.blocked > 0 ? 2 : 0,
|
|
155
|
+
summary: s,
|
|
156
|
+
})
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (isBlocked(task)) {
|
|
160
|
+
return stop(log, { reason: `task is blocked and needs a person: ${task.title}`, code: 2 })
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (!task.verify) {
|
|
164
|
+
text = markTask(text, task, {
|
|
165
|
+
status: "blocked",
|
|
166
|
+
verifyOutput: "blocked by the loop: this task has no verify command, so nothing could prove it finished.",
|
|
167
|
+
})
|
|
168
|
+
writeFileSync(join(cwd, TASKS_PATH), text)
|
|
169
|
+
return stop(log, {
|
|
170
|
+
reason: `"${task.title}" has no verify command. It cannot be checked, so it cannot be run unattended. Add one, or do it yourself.`,
|
|
171
|
+
code: 2,
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
rounds++
|
|
176
|
+
if (rounds > maxRounds) {
|
|
177
|
+
return stop(log, { reason: `stopped at the round cap (${maxRounds}). Raise it with --max-rounds, or run fewer tasks.`, code: 2 })
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const elapsed = Date.now() - started
|
|
181
|
+
if (elapsed >= budgetMs) {
|
|
182
|
+
return stop(log, { reason: `stopped at the time budget (${budgetMinutes} min, ${Math.round(elapsed / 60000)} min used).`, code: 2 })
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
console.log(`\n ${bold(`[${rounds}]`)} ${task.title}`)
|
|
186
|
+
if (task.files) console.log(` ${stamp(task.files)}`)
|
|
187
|
+
|
|
188
|
+
const agent = runAgent(role, buildPrompt(task), {
|
|
189
|
+
cwd,
|
|
190
|
+
timeoutMs: Math.max(60_000, budgetMs - (Date.now() - started)),
|
|
191
|
+
})
|
|
192
|
+
if (!agent.ok) {
|
|
193
|
+
const why = agent.error || agent.output || `exit ${agent.status}`
|
|
194
|
+
text = markTask(text, task, {
|
|
195
|
+
status: "blocked",
|
|
196
|
+
verifyOutput: `blocked by the loop: the ${role} agent failed.\n${tail(why)}`,
|
|
197
|
+
})
|
|
198
|
+
writeFileSync(join(cwd, TASKS_PATH), text)
|
|
199
|
+
return stop(log, {
|
|
200
|
+
reason: `the ${role} agent failed on "${task.title}": ${tail(why, 3).split("\n")[0]}`,
|
|
201
|
+
code: 3,
|
|
202
|
+
})
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// The agent's claim is not the evidence. Re-run the verify here, because a
|
|
206
|
+
// loop that trusts the agent's own word about whether it passed is a loop
|
|
207
|
+
// that reports success it never checked.
|
|
208
|
+
console.log(` ${stamp(`verify: ${task.verify}`)}`)
|
|
209
|
+
const check = runVerify(task.verify, { cwd, timeoutMs: 1800_000 })
|
|
210
|
+
log.push({ round: rounds, task: task.title, ok: check.ok, seconds: check.seconds })
|
|
211
|
+
|
|
212
|
+
if (!check.ok) {
|
|
213
|
+
text = markTask(text, task, {
|
|
214
|
+
status: "blocked",
|
|
215
|
+
verifyOutput: `verify failed (exit ${check.status}, ${check.seconds}s)\n${tail(check.output)}`,
|
|
216
|
+
})
|
|
217
|
+
writeFileSync(join(cwd, TASKS_PATH), text)
|
|
218
|
+
return stop(log, {
|
|
219
|
+
reason: `verify failed on "${task.title}". Marked blocked -- the real output is in ${TASKS_PATH}.`,
|
|
220
|
+
code: 4,
|
|
221
|
+
})
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
text = markTask(text, task, {
|
|
225
|
+
status: "done",
|
|
226
|
+
verifyOutput: `exit ${check.status} in ${check.seconds}s\n${tail(check.output)}`,
|
|
227
|
+
})
|
|
228
|
+
writeFileSync(join(cwd, TASKS_PATH), text)
|
|
229
|
+
console.log(` ${green("done")} ${stamp(`${check.seconds}s`)}`)
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function stop(log, { reason, code, summary }) {
|
|
234
|
+
return { reason, code: code ?? 0, rounds: log.length, log, summary }
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export { TASKS_PATH }
|
package/src/effort.js
CHANGED
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
// against 7.8s for `xhigh` on a different model, while 77% of responses ended
|
|
8
8
|
// on a tool call rather than prose.
|
|
9
9
|
//
|
|
10
|
-
// A level is never emitted raw. It is resolved against the
|
|
11
|
-
//
|
|
12
|
-
// the nearest level that actually exists.
|
|
10
|
+
// A level is never emitted raw. It is resolved against the levels of the model
|
|
11
|
+
// the lane actually runs on, so a lane asking for `max` on a model without
|
|
12
|
+
// `max` gets the nearest level that actually exists.
|
|
13
13
|
|
|
14
|
-
import { loadProfile, resolveRoles, resolveDefaultVariant } from "./profile.js"
|
|
14
|
+
import { loadProfile, resolveRoles, resolveDefaultVariant, resolveLaneModel, LANE_NAMES } from "./profile.js"
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* @typedef {import("./profile.js").Profile} Profile
|
|
@@ -22,39 +22,56 @@ import { loadProfile, resolveRoles, resolveDefaultVariant } from "./profile.js"
|
|
|
22
22
|
* @property {string} model
|
|
23
23
|
* @property {string|null} defaultVariant
|
|
24
24
|
* @property {Record<string, string|null>} roles
|
|
25
|
+
* @property {Record<string, string>} laneModels
|
|
25
26
|
* @property {string[]} notes
|
|
26
27
|
*/
|
|
27
28
|
|
|
28
|
-
/**
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The lanes a profile manages. A lane a profile does not mention is left alone.
|
|
31
|
+
*
|
|
32
|
+
* This used to be a hand-written list of nine lanes, and it drifted: the
|
|
33
|
+
* fleet has ten, and `longform` was missing. A second list of lane names is a
|
|
34
|
+
* second source of truth, and the lane table in `lanes.json` is the first one.
|
|
35
|
+
*
|
|
36
|
+
* @type {string[]}
|
|
37
|
+
*/
|
|
38
|
+
export const MANAGED_ROLES = LANE_NAMES
|
|
30
39
|
|
|
31
40
|
/**
|
|
32
41
|
* @param {string} profileId
|
|
42
|
+
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
33
43
|
* @returns {EffortResult}
|
|
34
44
|
*/
|
|
35
|
-
export function planEffort(profileId) {
|
|
45
|
+
export function planEffort(profileId, opts = {}) {
|
|
36
46
|
const p = loadProfile(profileId)
|
|
37
47
|
const levels = p.effort?.levels ?? []
|
|
38
|
-
const roles = resolveRoles(p)
|
|
48
|
+
const roles = resolveRoles(p, opts)
|
|
39
49
|
const notes = []
|
|
40
50
|
|
|
41
51
|
if (levels.length === 0) {
|
|
42
52
|
notes.push(
|
|
43
|
-
`${p.model} publishes no tunable reasoning effort; variant will be omitted for every
|
|
53
|
+
`${p.model} publishes no tunable reasoning effort; variant will be omitted for every lane.`,
|
|
44
54
|
)
|
|
45
55
|
}
|
|
46
56
|
for (const [role, want] of Object.entries(p.roles ?? {})) {
|
|
47
57
|
const got = roles[role]
|
|
48
58
|
if (want && got && want !== got) {
|
|
49
|
-
notes.push(`
|
|
59
|
+
notes.push(`lane ${role} asked for ${want}; its model tops out at ${got}.`)
|
|
50
60
|
}
|
|
51
61
|
}
|
|
52
62
|
|
|
63
|
+
/** @type {Record<string, string>} */
|
|
64
|
+
const laneModels = {}
|
|
65
|
+
for (const role of Object.keys(p.roles ?? {})) {
|
|
66
|
+
laneModels[role] = resolveLaneModel(p, role, opts).model
|
|
67
|
+
}
|
|
68
|
+
|
|
53
69
|
return {
|
|
54
70
|
profileId,
|
|
55
71
|
model: `${p.provider}/${p.model}`,
|
|
56
72
|
defaultVariant: resolveDefaultVariant(p),
|
|
57
73
|
roles,
|
|
74
|
+
laneModels,
|
|
58
75
|
notes,
|
|
59
76
|
}
|
|
60
77
|
}
|
|
@@ -65,34 +82,54 @@ export function planEffort(profileId) {
|
|
|
65
82
|
*
|
|
66
83
|
* `AgentConfig.variant` only applies to the agent's own configured model, so
|
|
67
84
|
* the model is written alongside the variant instead of relying on the global
|
|
68
|
-
* default. That is also what makes per-
|
|
85
|
+
* default. That is also what makes per-lane effort work at all.
|
|
86
|
+
*
|
|
87
|
+
* The model is only filled in when the agent does not have one. A fleet puts
|
|
88
|
+
* ten different models behind ten agents, and overwriting them with the
|
|
89
|
+
* profile's base model here would collapse the whole fleet onto one route.
|
|
69
90
|
*
|
|
70
91
|
* @param {Config} config
|
|
71
92
|
* @param {EffortResult} result
|
|
72
|
-
* @param {{ fallbackProfileId?: string }} [opts]
|
|
93
|
+
* @param {{ fallbackProfileId?: string, sessionModel?: string }} [opts]
|
|
73
94
|
* @returns {Config}
|
|
74
95
|
*/
|
|
75
96
|
export function applyEffort(config, result, opts = {}) {
|
|
76
97
|
const cfg = /** @type {Record<string, any>} */ (config)
|
|
77
98
|
|
|
78
|
-
//
|
|
79
|
-
// and
|
|
80
|
-
|
|
99
|
+
// The session model is the profile's base, unless the router moved it. A
|
|
100
|
+
// fleet whose base route is spent keeps its lanes and moves the session
|
|
101
|
+
// onto a fallback; pinning the base back here would undo that failover and
|
|
102
|
+
// send the session back to the dead route.
|
|
103
|
+
const sessionModel =
|
|
104
|
+
typeof opts.sessionModel === "string" && opts.sessionModel
|
|
105
|
+
? opts.sessionModel
|
|
106
|
+
: result.model
|
|
107
|
+
// Always pin the session model. When the model has no tunable effort we
|
|
108
|
+
// still set it and simply omit `variant` below.
|
|
109
|
+
cfg.model = sessionModel
|
|
81
110
|
|
|
82
111
|
cfg.agent = cfg.agent ?? {}
|
|
83
|
-
for (const role of
|
|
84
|
-
|
|
85
|
-
if (level === undefined) continue // role not described by this profile
|
|
112
|
+
for (const [role, level] of Object.entries(result.roles)) {
|
|
113
|
+
if (level === undefined) continue
|
|
86
114
|
const existing = cfg.agent[role] ?? {}
|
|
87
|
-
if (!existing.model)
|
|
115
|
+
if (!existing.model) {
|
|
116
|
+
existing.model = result.laneModels?.[role]
|
|
117
|
+
? `${result.model.split("/")[0]}/${result.laneModels[role]}`
|
|
118
|
+
: result.model
|
|
119
|
+
}
|
|
88
120
|
if (level === null) delete existing.variant
|
|
89
121
|
else existing.variant = level
|
|
90
122
|
cfg.agent[role] = existing
|
|
91
123
|
}
|
|
92
124
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
125
|
+
// The router publishes `sessionModel` and `threshold` on the same object.
|
|
126
|
+
// Replacing the whole thing here dropped them, which is how a fleet that
|
|
127
|
+
// had failed over ended up with a session model nobody chose.
|
|
128
|
+
cfg.harness = {
|
|
129
|
+
...(cfg.harness ?? {}),
|
|
130
|
+
profile: result.profileId,
|
|
131
|
+
...(opts.fallbackProfileId ? { fallback: opts.fallbackProfileId } : {}),
|
|
132
|
+
}
|
|
96
133
|
|
|
97
134
|
return config
|
|
98
135
|
}
|