opencode-harness-kit 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +544 -19
- package/CONTRIBUTING.md +242 -0
- package/GLOBAL.md +34 -5
- package/MIGRATION.md +151 -0
- package/README.md +294 -37
- package/SECURITY.md +30 -0
- package/bin/baseline.mjs +15 -7
- package/bin/git-hooks/pre-commit +41 -9
- package/bin/oc-harness.mjs +1265 -82
- package/dist/config.d.ts +127 -0
- package/dist/denylist.d.ts +33 -0
- package/dist/effort.d.ts +64 -0
- package/dist/guard.d.ts +23 -0
- package/dist/health.d.ts +43 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +532 -149
- package/dist/plugin.d.ts +15 -0
- package/dist/profile.d.ts +399 -0
- package/dist/project.d.ts +8 -0
- package/dist/prune.d.ts +6 -0
- package/dist/router.d.ts +3 -0
- package/dist/rules.d.ts +109 -0
- package/dist/safe.d.ts +7 -0
- package/dist/telemetry-store.d.ts +12 -0
- package/dist/telemetry.d.ts +3 -0
- package/docs/EXIT-CODES.md +71 -0
- package/docs/KNOWN-LIMITS.md +45 -21
- package/docs/RESEARCH.md +9 -4
- package/docs/ROUTING.md +53 -8
- package/eval/RESULTS.md +443 -0
- package/eval/ab.mjs +743 -0
- package/eval/attribute.mjs +170 -0
- package/eval/flags.mjs +88 -0
- package/eval/measure-install.mjs +323 -0
- package/eval/measure-prune.mjs +339 -0
- package/eval/metrics.js +79 -0
- package/eval/premises.mjs +200 -0
- package/eval/prepare.mjs +233 -0
- package/eval/run.mjs +376 -0
- package/eval/tasks-self.json +81 -0
- package/eval/tasks-ts.json +50 -0
- package/eval/tasks.json +83 -0
- package/eval/thresholds.mjs +134 -0
- package/eval/verify.mjs +287 -0
- package/package.json +44 -17
- package/profiles/fleet-all.json +6 -1
- package/profiles/fleet-safe.json +7 -3
- package/profiles/schema.json +182 -51
- package/prompts/build.md +4 -2
- package/skills/space-build/SKILL.md +4 -2
- package/skills/space-finish/SKILL.md +6 -2
- package/skills/space-plan/reference/task-template.md +13 -5
- package/src/config.js +182 -32
- package/src/denylist.js +95 -0
- package/src/driver/run.mjs +81 -16
- package/src/effort.js +15 -4
- package/src/guard.ts +165 -13
- package/src/health.js +80 -0
- package/src/index.ts +15 -4
- package/src/lane-usage.js +72 -18
- package/src/plugin-type-check.ts +38 -0
- package/src/plugin.ts +39 -0
- package/src/profile.js +225 -26
- package/src/project.js +47 -0
- package/src/prune.ts +110 -31
- package/src/router.ts +32 -63
- package/src/rules.js +99 -7
- package/src/safe.ts +27 -0
- package/src/sqlite.js +144 -0
- package/src/tasks.js +112 -29
- package/src/telemetry-store.ts +62 -0
- package/src/telemetry.ts +60 -57
- package/src/trust.js +70 -9
package/src/config.js
CHANGED
|
@@ -30,7 +30,9 @@ import {
|
|
|
30
30
|
} from "./profile.js"
|
|
31
31
|
|
|
32
32
|
import { join } from "node:path"
|
|
33
|
+
import { readFileSync } from "node:fs"
|
|
33
34
|
import { homedir } from "node:os"
|
|
35
|
+
import { BASH_DENYLIST } from "./denylist.js"
|
|
34
36
|
|
|
35
37
|
/** @typedef {import("./profile.js").Profile} Profile */
|
|
36
38
|
|
|
@@ -61,6 +63,121 @@ export function globalInstructionsPath() {
|
|
|
61
63
|
*/
|
|
62
64
|
export const GLOBAL_INSTRUCTIONS = "~/.config/opencode/GLOBAL.md"
|
|
63
65
|
|
|
66
|
+
/**
|
|
67
|
+
* The kit's own version, read from its package.json.
|
|
68
|
+
*
|
|
69
|
+
* @returns {string}
|
|
70
|
+
*/
|
|
71
|
+
export function kitVersion() {
|
|
72
|
+
try {
|
|
73
|
+
const pkg = JSON.parse(
|
|
74
|
+
readFileSync(new URL("../package.json", import.meta.url), "utf8"),
|
|
75
|
+
)
|
|
76
|
+
return String(pkg.version ?? "unknown")
|
|
77
|
+
} catch {
|
|
78
|
+
return "unknown"
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The memory plugin this kit can turn on.
|
|
84
|
+
*
|
|
85
|
+
* The kit does not implement memory. A local vector store with embeddings, a
|
|
86
|
+
* web UI and auto-capture is a large thing to build badly, and there is one
|
|
87
|
+
* that is already good: `opencode-mem` (MIT). What the kit can do is make it
|
|
88
|
+
* one flag away, and keep the config honest about it.
|
|
89
|
+
*/
|
|
90
|
+
export const MEMORY_PLUGIN = "opencode-mem@latest"
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The config with the memory plugin turned on.
|
|
94
|
+
*
|
|
95
|
+
* The plugin list is preserved. The user's own plugins are not the kit's to
|
|
96
|
+
* remove, and the entry is added rather than replaced so a config that
|
|
97
|
+
* already has it does not grow a second copy.
|
|
98
|
+
*
|
|
99
|
+
* opencode v1 calls the key `plugin` and v2 calls it `plugins`; whichever
|
|
100
|
+
* the config already uses is the one that is extended.
|
|
101
|
+
*
|
|
102
|
+
* @param {any} base the config as it is on disk
|
|
103
|
+
* @param {string} [spec] the plugin id, so a pinned version or a mirror works
|
|
104
|
+
* @returns {any}
|
|
105
|
+
*/
|
|
106
|
+
export function withMemoryPlugin(base, spec = MEMORY_PLUGIN) {
|
|
107
|
+
const b = base ?? {}
|
|
108
|
+
const key = Array.isArray(b.plugins) ? "plugins" : "plugin"
|
|
109
|
+
const existing = b[key]
|
|
110
|
+
if (existing !== undefined && !Array.isArray(existing)) {
|
|
111
|
+
// The value is not a list. Appending to it would replace whatever the
|
|
112
|
+
// user had, so it is left alone.
|
|
113
|
+
return { ...b }
|
|
114
|
+
}
|
|
115
|
+
const list = Array.isArray(existing) ? [...existing] : []
|
|
116
|
+
if (!list.includes(spec)) list.push(spec)
|
|
117
|
+
return { ...b, [key]: list }
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Whether the memory plugin can be added to this config.
|
|
122
|
+
*
|
|
123
|
+
* A config whose plugin value is not a list is left alone: replacing a
|
|
124
|
+
* string with a list is data loss, and the promise is that the user's own
|
|
125
|
+
* plugins are never touched.
|
|
126
|
+
*
|
|
127
|
+
* @param {any} base
|
|
128
|
+
* @returns {boolean}
|
|
129
|
+
*/
|
|
130
|
+
export function canAddMemoryPlugin(base) {
|
|
131
|
+
const b = base ?? {}
|
|
132
|
+
const key = Array.isArray(b.plugins) ? "plugins" : "plugin"
|
|
133
|
+
const existing = b[key]
|
|
134
|
+
return existing === undefined || Array.isArray(existing)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The utility route a profile names, as a full model id.
|
|
139
|
+
*
|
|
140
|
+
* `small_model` is opencode's route for the short auxiliary calls. It used to
|
|
141
|
+
* be the session model, so every title and every rewritten question paid the
|
|
142
|
+
* session model's reasoning - and reasoning, not output, is what a short call
|
|
143
|
+
* costs.
|
|
144
|
+
*
|
|
145
|
+
* @param {any} p the profile
|
|
146
|
+
* @returns {string | null}
|
|
147
|
+
*/
|
|
148
|
+
export function utilityModel(p) {
|
|
149
|
+
const u = p?.utility
|
|
150
|
+
if (!u || typeof u.model !== "string" || !u.model) return null
|
|
151
|
+
return `${p.provider ?? "opencode"}/${u.model}`
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* The instruction list a generated config carries.
|
|
156
|
+
*
|
|
157
|
+
* The kit's own file is always in it. A config that already had an
|
|
158
|
+
* `instructions` array used to replace it wholesale, which unloaded the
|
|
159
|
+
* whole behavioural layer while everything still looked correct.
|
|
160
|
+
*
|
|
161
|
+
* The user's own files keep their order and come first; the kit's is
|
|
162
|
+
* appended. Duplicates are dropped, so re-running `use` does not grow the
|
|
163
|
+
* list.
|
|
164
|
+
*
|
|
165
|
+
* @param {string[] | string | undefined} base
|
|
166
|
+
* @param {string} kitPath
|
|
167
|
+
* @returns {string[]}
|
|
168
|
+
*/
|
|
169
|
+
export function mergeInstructions(base, kitPath) {
|
|
170
|
+
const list = Array.isArray(base) ? base.filter((x) => typeof x === "string") : []
|
|
171
|
+
/** @type {string[]} */
|
|
172
|
+
const out = []
|
|
173
|
+
for (const entry of [...list, kitPath]) {
|
|
174
|
+
if (!entry) continue
|
|
175
|
+
if (out.includes(entry)) continue
|
|
176
|
+
out.push(entry)
|
|
177
|
+
}
|
|
178
|
+
return out
|
|
179
|
+
}
|
|
180
|
+
|
|
64
181
|
/**
|
|
65
182
|
* Read-only and verification commands.
|
|
66
183
|
*
|
|
@@ -84,7 +201,7 @@ export const BASH_ALLOWLIST = [
|
|
|
84
201
|
"cd*", "realpath*", "basename*", "dirname*", "readlink*",
|
|
85
202
|
"for*", "set*", "export*", "unset*", "bash*", "sh*", "true*", "false*", "test*",
|
|
86
203
|
"read*", "#*", "xargs*", "seq*", "exit*", ":*",
|
|
87
|
-
"node*", "python3*", "python*", "
|
|
204
|
+
"node*", "python3*", "python*", "bun*", "uv*", "go*", "cargo*", "rustc*",
|
|
88
205
|
"opencode*", "sqlite3*", "jq*", "fd*",
|
|
89
206
|
"git status*", "git log*", "git diff*", "git show*", "git branch*",
|
|
90
207
|
"git rev-parse*", "git config --get*", "git remote*", "git blame*",
|
|
@@ -103,27 +220,20 @@ export const BASH_ALLOWLIST = [
|
|
|
103
220
|
"python3 -m pytest*", "python3 -m venv*", "python3 -m compileall*",
|
|
104
221
|
"python3 -c*", "python3 agent/*", "python3 src/*", "python3 -",
|
|
105
222
|
".venv/bin/python*", "pytest*", "ruff*", "mypy*", "black*",
|
|
106
|
-
"npm test*", "npm run *", "npm
|
|
107
|
-
"npx
|
|
223
|
+
"npm test*", "npm run *", "npm ls*", "npm view *", "npm outdated*",
|
|
224
|
+
"npx tsc*", "npx eslint*", "npx prettier*",
|
|
225
|
+
"uv run *", "uv pip *", "bun run *", "bunx*", "node -e*",
|
|
108
226
|
"opencode debug*", "opencode --version*", "opencode models*",
|
|
109
227
|
]
|
|
110
228
|
|
|
111
|
-
/**
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
"chmod 777 *",
|
|
120
|
-
"chmod 777*",
|
|
121
|
-
"gh pr create*",
|
|
122
|
-
"gh pr merge*",
|
|
123
|
-
"gh pr close*",
|
|
124
|
-
"gh release*",
|
|
125
|
-
"gh repo delete*",
|
|
126
|
-
]
|
|
229
|
+
/**
|
|
230
|
+
* Hard blocks.
|
|
231
|
+
*
|
|
232
|
+
* The list lives in ./denylist.js because it is enforced in TWO places: by
|
|
233
|
+
* opencode's permission map, and by the guard plugin. The two used to be
|
|
234
|
+
* different lists, and the comment here claimed they were the same.
|
|
235
|
+
*/
|
|
236
|
+
export { BASH_DENYLIST } from "./denylist.js"
|
|
127
237
|
|
|
128
238
|
/**
|
|
129
239
|
* @param {Profile} p
|
|
@@ -214,10 +324,21 @@ export function generateConfig(p, opts = {}) {
|
|
|
214
324
|
...base_,
|
|
215
325
|
|
|
216
326
|
model,
|
|
217
|
-
//
|
|
218
|
-
//
|
|
219
|
-
// small_model
|
|
220
|
-
|
|
327
|
+
// The utility route, when the profile names one.
|
|
328
|
+
//
|
|
329
|
+
// `small_model` is what opencode uses for the short auxiliary calls: a
|
|
330
|
+
// title, a one-line summary, a question rewritten. It used to be the same
|
|
331
|
+
// model as the session, which meant every one of those calls paid the
|
|
332
|
+
// session model's reasoning - and reasoning, not output, is what a short
|
|
333
|
+
// call costs. Measured on this tier, a title costs 49 reasoning tokens on
|
|
334
|
+
// deepseek and 332 on nemotron-3.5-super-vl, and deepseek answers in
|
|
335
|
+
// 7.2s against 22.7s for glm.
|
|
336
|
+
//
|
|
337
|
+
// A profile that does not name a utility route keeps the session model.
|
|
338
|
+
// A single-model profile SHOULD keep it: "one route, and it is the one
|
|
339
|
+
// you asked for" is a promise the five-lane profiles make, and a cheaper
|
|
340
|
+
// route on the side would break it.
|
|
341
|
+
small_model: utilityModel(p) ?? model,
|
|
221
342
|
// No top-level `variant`. The 1.18.35 Config schema sets
|
|
222
343
|
// `additionalProperties: false` and has no `variant` key, so OpenCode
|
|
223
344
|
// strips it from the resolved config - verified with `opencode debug
|
|
@@ -231,16 +352,26 @@ export function generateConfig(p, opts = {}) {
|
|
|
231
352
|
subagent_depth: base.subagent_depth ?? 2,
|
|
232
353
|
// Omitting `lsp` disables it entirely, so this must be set explicitly.
|
|
233
354
|
lsp: base.lsp ?? true,
|
|
234
|
-
|
|
355
|
+
// The kit's own instruction file is always in the list.
|
|
356
|
+
//
|
|
357
|
+
// It used to be `base.instructions ?? [instructionsPath]`, so a config
|
|
358
|
+
// that already had an `instructions` array never referenced GLOBAL.md at
|
|
359
|
+
// all: the whole behavioural layer - the lane table, the rules, the
|
|
360
|
+
// read-pressure guidance - was silently unloaded, and `verify` still
|
|
361
|
+
// passed because it only checked the paths the config did name. The
|
|
362
|
+
// user's own files keep their order and come first; the kit's is
|
|
363
|
+
// appended, because it is the layer that has to hold when the others
|
|
364
|
+
// do not.
|
|
365
|
+
instructions: mergeInstructions(base.instructions, instructionsPath),
|
|
235
366
|
|
|
236
367
|
compaction,
|
|
237
368
|
// OpenCode's own default (2000 lines / 51200 bytes), reached back from an
|
|
238
|
-
// earlier 200 / 8192. The tight setting truncated 415 of 2,
|
|
239
|
-
// `read` calls, and
|
|
369
|
+
// earlier 200 / 8192. The tight setting truncated 415 of 2,966 measured
|
|
370
|
+
// `read` calls, and 423 calls were issued with an `offset` to fetch the
|
|
240
371
|
// rest: the truncation causes the re-read, one for one. At the measured
|
|
241
372
|
// 17.3s median per request that round trip costs far more than the tokens
|
|
242
373
|
// it saves, and prune.ts already bounds old tool output deterministically.
|
|
243
|
-
// Only 14 of 2,
|
|
374
|
+
// Only 14 of 2,966 reads exceed 51,200 bytes, so this ceiling is not the
|
|
244
375
|
// thing that limits context. `oc-harness baseline` re-measures all four
|
|
245
376
|
// numbers if this ever needs arguing again.
|
|
246
377
|
tool_output: { max_lines: 2000, max_bytes: 51200 },
|
|
@@ -278,13 +409,17 @@ export function generateConfig(p, opts = {}) {
|
|
|
278
409
|
|
|
279
410
|
agent,
|
|
280
411
|
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
412
|
+
// How a profile stays identifiable after the config has been hand-edited.
|
|
413
|
+
// The fallback queue is NOT written here: the router reads it straight
|
|
414
|
+
// from the profile file, and a copy of the first id in a key nothing
|
|
415
|
+
// reads is a second source of truth that can only drift.
|
|
284
416
|
harness: {
|
|
285
417
|
profile: p.id,
|
|
286
|
-
...(p.fallback?.length ? { fallback: p.fallback[0] } : {}),
|
|
287
418
|
threshold: pruneThreshold(p, { unhealthy }),
|
|
419
|
+
// Which kit wrote this. Without it, a config from three releases ago
|
|
420
|
+
// and one from today are the same file to `verify`, and the only way
|
|
421
|
+
// to tell them apart is to remember.
|
|
422
|
+
kit: kitVersion(),
|
|
288
423
|
},
|
|
289
424
|
}
|
|
290
425
|
|
|
@@ -321,6 +456,11 @@ function render(cfg, p, opts) {
|
|
|
321
456
|
`${c}`,
|
|
322
457
|
`${c}profile ${p.id}`,
|
|
323
458
|
`${c}model ${p.provider}/${p.model}`,
|
|
459
|
+
...(p.utility
|
|
460
|
+
? [
|
|
461
|
+
`${c}utility ${p.provider}/${p.utility.model} (opencode's short calls: titles, summaries)`,
|
|
462
|
+
]
|
|
463
|
+
: []),
|
|
324
464
|
...(splits.length
|
|
325
465
|
? [
|
|
326
466
|
`${c}lanes ${splits.length} enabled, each on the first model its chain allows`,
|
|
@@ -345,7 +485,17 @@ function render(cfg, p, opts) {
|
|
|
345
485
|
`${c}Derived numbers, do not hand-edit:`,
|
|
346
486
|
`${c} compaction.reserved ${reservedTokens(p, { unhealthy }).toLocaleString("en-US")} (${Math.round((p.effort?.reserveShare ?? 0.12) * 100)}% of the narrowest input cap in play, held back for reasoning growth)`,
|
|
347
487
|
`${c} prune threshold ${pruneThreshold(p, { unhealthy }).toLocaleString("en-US")} (narrowest input cap minus reserve)`,
|
|
348
|
-
`${c} role effort ${Object.entries(roles)
|
|
488
|
+
`${c} role effort ${Object.entries(roles)
|
|
489
|
+
.map(([k, v]) => {
|
|
490
|
+
// A lane that asked for a tier and did not get one says so HERE,
|
|
491
|
+
// in the file the user reads. The reason it is in the file and not
|
|
492
|
+
// only in `oc-harness show` is that the file is what a human
|
|
493
|
+
// looks at when a lane turns out to be slow.
|
|
494
|
+
const want = p.roles?.[k]
|
|
495
|
+
const dropped = want && want !== "none" && v !== want
|
|
496
|
+
return `${k}=${v ?? "unset"}${dropped ? `(asked ${want})` : ""}`
|
|
497
|
+
})
|
|
498
|
+
.join(" ")}`,
|
|
349
499
|
...(laneRows.length
|
|
350
500
|
? [
|
|
351
501
|
`${c} lane models ${laneRows.map((r) => `${r.role}=${r.model}${r.limits ? ` (out ${r.limits.output.toLocaleString("en-US")})` : ""}`).join(" ")}`,
|
package/src/denylist.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// The commands that are never allowed, in one place.
|
|
2
|
+
//
|
|
3
|
+
// This list is the MECHANICAL half of the kit's safety policy, and it is
|
|
4
|
+
// enforced in two places: by opencode's permission map, and by the guard
|
|
5
|
+
// plugin. Before this module existed, the two were different lists - the
|
|
6
|
+
// config denied `gh pr create` and `npm publish`, and the guard blocked
|
|
7
|
+
// neither - so the config's own comment, "the guard plugin enforces the same
|
|
8
|
+
// set in code", was false.
|
|
9
|
+
//
|
|
10
|
+
// One list, two enforcers. If it is not in here, the guard does not block it.
|
|
11
|
+
//
|
|
12
|
+
// A pattern is a glob: `*` matches anything. The patterns are matched
|
|
13
|
+
// against the COMMAND, not against a subcommand, so `rm -rf *` means "a
|
|
14
|
+
// command that starts with `rm -rf `".
|
|
15
|
+
|
|
16
|
+
/** @type {string[]} */
|
|
17
|
+
export const BASH_DENYLIST = [
|
|
18
|
+
// `git push` is intentionally absent: it falls through to the `*` ask rule
|
|
19
|
+
// so opencode prompts for human approval before publishing.
|
|
20
|
+
"rm -rf *",
|
|
21
|
+
"rm -fr *",
|
|
22
|
+
"curl * | sh",
|
|
23
|
+
"curl * | bash",
|
|
24
|
+
"chmod 777 *",
|
|
25
|
+
"chmod 777*",
|
|
26
|
+
"gh pr create*",
|
|
27
|
+
"gh pr merge*",
|
|
28
|
+
"gh pr close*",
|
|
29
|
+
"gh release*",
|
|
30
|
+
"gh repo delete*",
|
|
31
|
+
// Publishing to a registry. The allowlist had a blanket `npm*`, so
|
|
32
|
+
// `npm publish` was ALLOWED - no prompt, no question - while GLOBAL.md says
|
|
33
|
+
// to ask before publishing. The mechanical layer contradicted the judgement
|
|
34
|
+
// layer, which is the exact failure this kit is built to catch.
|
|
35
|
+
//
|
|
36
|
+
// The registry is the one destination where a mistake cannot be undone by
|
|
37
|
+
// deleting a file.
|
|
38
|
+
"npm publish*",
|
|
39
|
+
"npm unpublish*",
|
|
40
|
+
"npm deprecate*",
|
|
41
|
+
"npm login*",
|
|
42
|
+
"npm adduser*",
|
|
43
|
+
"npm token *",
|
|
44
|
+
"npm owner *",
|
|
45
|
+
"yarn publish*",
|
|
46
|
+
"pnpm publish*",
|
|
47
|
+
"bun publish*",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Whether a glob pattern matches a command.
|
|
52
|
+
*
|
|
53
|
+
* `*` matches any run of characters, including none.
|
|
54
|
+
*
|
|
55
|
+
* @param {string} command
|
|
56
|
+
* @param {string} pattern
|
|
57
|
+
* @returns {boolean}
|
|
58
|
+
*/
|
|
59
|
+
export function matchesPattern(command, pattern) {
|
|
60
|
+
const re = new RegExp(
|
|
61
|
+
"^" +
|
|
62
|
+
pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[\\s\\S]*") +
|
|
63
|
+
"$",
|
|
64
|
+
)
|
|
65
|
+
return re.test(command)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The first denied pattern a command matches, or null.
|
|
70
|
+
*
|
|
71
|
+
* The command is matched whole, and it is matched by the CONFIG too - the
|
|
72
|
+
* permission map globs the raw command, because that is the only thing it
|
|
73
|
+
* can see. So this function is the flat, literal half.
|
|
74
|
+
*
|
|
75
|
+
* The guard does not stop here. It calls this on each SEGMENT, with the
|
|
76
|
+
* wrappers stripped and the payload of a `bash -c` followed, so
|
|
77
|
+
* `sudo bash -c "npm publish"` is caught as well. The two enforcers read the
|
|
78
|
+
* same list and one of them reads it more carefully; that is the price of
|
|
79
|
+
* having the list in two places.
|
|
80
|
+
*
|
|
81
|
+
* The two PIPE patterns are for the config's side only. `segments()` splits
|
|
82
|
+
* on `|`, so a segment never contains " | sh" and the guard catches a piped
|
|
83
|
+
* script through its own rule instead.
|
|
84
|
+
*
|
|
85
|
+
* @param {string} command
|
|
86
|
+
* @returns {string | null}
|
|
87
|
+
*/
|
|
88
|
+
export function deniedBy(command) {
|
|
89
|
+
const cmd = String(command ?? "").trim()
|
|
90
|
+
if (!cmd) return null
|
|
91
|
+
for (const pattern of BASH_DENYLIST) {
|
|
92
|
+
if (matchesPattern(cmd, pattern)) return pattern
|
|
93
|
+
}
|
|
94
|
+
return null
|
|
95
|
+
}
|
package/src/driver/run.mjs
CHANGED
|
@@ -43,7 +43,11 @@ const red = (s) => `\x1b[31m${s}\x1b[0m`
|
|
|
43
43
|
* Never throws: the caller has to be able to write the failure into the task
|
|
44
44
|
* file, which is the only record anyone gets of why a task did not land.
|
|
45
45
|
*/
|
|
46
|
-
export function sh(
|
|
46
|
+
export function sh(
|
|
47
|
+
command,
|
|
48
|
+
/** @type {{ cwd?: string, timeoutMs?: number }} */
|
|
49
|
+
{ cwd, timeoutMs = 900_000 } = {},
|
|
50
|
+
) {
|
|
47
51
|
const started = Date.now()
|
|
48
52
|
const res = spawnSync(command, {
|
|
49
53
|
shell: true,
|
|
@@ -75,14 +79,22 @@ export function tail(text, n = 12) {
|
|
|
75
79
|
* the agent actually did rather than being told about it.
|
|
76
80
|
*/
|
|
77
81
|
export function invokeAgent(role, prompt, { cwd, timeoutMs }) {
|
|
82
|
+
// `--dir` is not optional, and neither is PWD.
|
|
83
|
+
//
|
|
84
|
+
// `spawnSync`'s `cwd` sets the child's working directory but does NOT update
|
|
85
|
+
// the inherited `PWD`, and opencode resolves its project directory from
|
|
86
|
+
// `process.env.PWD ?? process.cwd()`. So the agent was handed the right
|
|
87
|
+
// directory and worked in the wrong repository - the one the shell happened
|
|
88
|
+
// to be in. `eval/ab.mjs` documents the same trap and fixes it the same way.
|
|
78
89
|
const res = spawnSync(
|
|
79
90
|
"opencode",
|
|
80
|
-
["run", "--agent", role, "--format", "json", prompt],
|
|
91
|
+
["run", "--agent", role, "--dir", cwd, "--format", "json", prompt],
|
|
81
92
|
{
|
|
82
93
|
cwd,
|
|
83
94
|
encoding: "utf8",
|
|
84
95
|
timeout: timeoutMs,
|
|
85
96
|
maxBuffer: 64 * 1024 * 1024,
|
|
97
|
+
env: { ...process.env, PWD: cwd },
|
|
86
98
|
},
|
|
87
99
|
)
|
|
88
100
|
return {
|
|
@@ -106,7 +118,7 @@ export function buildPrompt(task) {
|
|
|
106
118
|
``,
|
|
107
119
|
`Rules for this run:`,
|
|
108
120
|
` - Change only the files named above. If the task is wrong or blocked, say so and stop.`,
|
|
109
|
-
` - Never run git push, never create a PR, never publish.
|
|
121
|
+
` - Never run git push, never create a PR, never publish. This loop runs unattended, so there is nobody to ask - a human pushes.`,
|
|
110
122
|
` - Report the real output of the verify command. Never invent a passing result.`,
|
|
111
123
|
].join("\n")
|
|
112
124
|
}
|
|
@@ -117,6 +129,11 @@ export function buildPrompt(task) {
|
|
|
117
129
|
* Stops at the first of: nothing pending, nothing verifiable, a blocked task, a
|
|
118
130
|
* failing verify, the round cap, or the time budget. Whichever it was gets
|
|
119
131
|
* printed, so a run that stops early always says why.
|
|
132
|
+
*
|
|
133
|
+
* The budget is checked BETWEEN rounds. The agent and the verify each get the
|
|
134
|
+
* time that is left, with a floor of half a minute - so a run can overrun the
|
|
135
|
+
* budget by that floor plus the time between the check and the call, and not
|
|
136
|
+
* by the thirty minutes the verify used to be allowed.
|
|
120
137
|
*/
|
|
121
138
|
export function runLoop({
|
|
122
139
|
cwd = process.cwd(),
|
|
@@ -140,6 +157,31 @@ export function runLoop({
|
|
|
140
157
|
let text = readFileSync(join(cwd, TASKS_PATH), "utf8")
|
|
141
158
|
let rounds = 0
|
|
142
159
|
|
|
160
|
+
/**
|
|
161
|
+
* The task file, read fresh, and the task we were working on inside it.
|
|
162
|
+
*
|
|
163
|
+
* The file is read once at the start and every write used that copy, so
|
|
164
|
+
* anything the agent wrote - a new task, a corrected verify command, a note
|
|
165
|
+
* about a blocker - was silently erased by the loop's own write. A human
|
|
166
|
+
* editing the file while the loop ran lost the same way.
|
|
167
|
+
*
|
|
168
|
+
* So the file is read again before every write, and the change is applied to
|
|
169
|
+
* the CURRENT text. The task is matched by its title, because a line number
|
|
170
|
+
* from the old copy means nothing in the new one.
|
|
171
|
+
*/
|
|
172
|
+
const reload = (task) => {
|
|
173
|
+
let fresh = ""
|
|
174
|
+
try {
|
|
175
|
+
fresh = readFileSync(join(cwd, TASKS_PATH), "utf8")
|
|
176
|
+
} catch {
|
|
177
|
+
return { text, task }
|
|
178
|
+
}
|
|
179
|
+
const parsed = parseTasks(fresh)
|
|
180
|
+
const same = parsed.tasks.filter((t) => t.title === task.title)[task.ordinal ?? 0]
|
|
181
|
+
if (!same) return { text, task }
|
|
182
|
+
return { text: fresh, task: same }
|
|
183
|
+
}
|
|
184
|
+
|
|
143
185
|
for (;;) {
|
|
144
186
|
const parsed = parseTasks(text)
|
|
145
187
|
const task = nextPending(parsed)
|
|
@@ -153,15 +195,17 @@ export function runLoop({
|
|
|
153
195
|
: `every task is done (${s.done}/${s.total}).`,
|
|
154
196
|
code: s.blocked > 0 ? 2 : 0,
|
|
155
197
|
summary: s,
|
|
198
|
+
rounds,
|
|
156
199
|
})
|
|
157
200
|
}
|
|
158
201
|
|
|
159
202
|
if (isBlocked(task)) {
|
|
160
|
-
return stop(log, { reason: `task is blocked and needs a person: ${task.title}`, code: 2 })
|
|
203
|
+
return stop(log, { reason: `task is blocked and needs a person: ${task.title}`, code: 2, rounds })
|
|
161
204
|
}
|
|
162
205
|
|
|
163
206
|
if (!task.verify) {
|
|
164
|
-
|
|
207
|
+
const fresh = reload(task)
|
|
208
|
+
text = markTask(fresh.text, fresh.task, {
|
|
165
209
|
status: "blocked",
|
|
166
210
|
verifyOutput: "blocked by the loop: this task has no verify command, so nothing could prove it finished.",
|
|
167
211
|
})
|
|
@@ -169,29 +213,35 @@ export function runLoop({
|
|
|
169
213
|
return stop(log, {
|
|
170
214
|
reason: `"${task.title}" has no verify command. It cannot be checked, so it cannot be run unattended. Add one, or do it yourself.`,
|
|
171
215
|
code: 2,
|
|
216
|
+
rounds,
|
|
172
217
|
})
|
|
173
218
|
}
|
|
174
219
|
|
|
175
|
-
rounds
|
|
176
|
-
|
|
177
|
-
return stop(log, { reason: `stopped at the round cap (${maxRounds}). Raise it with --max-rounds, or run fewer tasks.`, code: 2 })
|
|
220
|
+
if (rounds >= maxRounds) {
|
|
221
|
+
return stop(log, { reason: `stopped at the round cap (${maxRounds}). Raise it with --max-rounds, or run fewer tasks.`, code: 2, rounds })
|
|
178
222
|
}
|
|
179
223
|
|
|
180
224
|
const elapsed = Date.now() - started
|
|
181
225
|
if (elapsed >= budgetMs) {
|
|
182
|
-
return stop(log, { reason: `stopped at the time budget (${budgetMinutes} min, ${Math.round(elapsed / 60000)} min used).`, code: 2 })
|
|
226
|
+
return stop(log, { reason: `stopped at the time budget (${budgetMinutes} min, ${Math.round(elapsed / 60000)} min used).`, code: 2, rounds })
|
|
183
227
|
}
|
|
184
228
|
|
|
229
|
+
// The time that is left, with a floor so a nearly-spent budget still gets
|
|
230
|
+
// a command that can actually finish.
|
|
231
|
+
const remaining = () => Math.max(30_000, budgetMs - (Date.now() - started))
|
|
232
|
+
|
|
233
|
+
rounds++
|
|
185
234
|
console.log(`\n ${bold(`[${rounds}]`)} ${task.title}`)
|
|
186
235
|
if (task.files) console.log(` ${stamp(task.files)}`)
|
|
187
236
|
|
|
188
237
|
const agent = runAgent(role, buildPrompt(task), {
|
|
189
238
|
cwd,
|
|
190
|
-
timeoutMs:
|
|
239
|
+
timeoutMs: remaining(),
|
|
191
240
|
})
|
|
192
241
|
if (!agent.ok) {
|
|
193
242
|
const why = agent.error || agent.output || `exit ${agent.status}`
|
|
194
|
-
|
|
243
|
+
const fresh = reload(task)
|
|
244
|
+
text = markTask(fresh.text, fresh.task, {
|
|
195
245
|
status: "blocked",
|
|
196
246
|
verifyOutput: `blocked by the loop: the ${role} agent failed.\n${tail(why)}`,
|
|
197
247
|
})
|
|
@@ -199,6 +249,7 @@ export function runLoop({
|
|
|
199
249
|
return stop(log, {
|
|
200
250
|
reason: `the ${role} agent failed on "${task.title}": ${tail(why, 3).split("\n")[0]}`,
|
|
201
251
|
code: 3,
|
|
252
|
+
rounds,
|
|
202
253
|
})
|
|
203
254
|
}
|
|
204
255
|
|
|
@@ -206,11 +257,12 @@ export function runLoop({
|
|
|
206
257
|
// loop that trusts the agent's own word about whether it passed is a loop
|
|
207
258
|
// that reports success it never checked.
|
|
208
259
|
console.log(` ${stamp(`verify: ${task.verify}`)}`)
|
|
209
|
-
const check = runVerify(task.verify, { cwd, timeoutMs:
|
|
260
|
+
const check = runVerify(task.verify, { cwd, timeoutMs: remaining() })
|
|
210
261
|
log.push({ round: rounds, task: task.title, ok: check.ok, seconds: check.seconds })
|
|
211
262
|
|
|
212
263
|
if (!check.ok) {
|
|
213
|
-
|
|
264
|
+
const fresh = reload(task)
|
|
265
|
+
text = markTask(fresh.text, fresh.task, {
|
|
214
266
|
status: "blocked",
|
|
215
267
|
verifyOutput: `verify failed (exit ${check.status}, ${check.seconds}s)\n${tail(check.output)}`,
|
|
216
268
|
})
|
|
@@ -218,10 +270,12 @@ export function runLoop({
|
|
|
218
270
|
return stop(log, {
|
|
219
271
|
reason: `verify failed on "${task.title}". Marked blocked -- the real output is in ${TASKS_PATH}.`,
|
|
220
272
|
code: 4,
|
|
273
|
+
rounds,
|
|
221
274
|
})
|
|
222
275
|
}
|
|
223
276
|
|
|
224
|
-
|
|
277
|
+
const fresh = reload(task)
|
|
278
|
+
text = markTask(fresh.text, fresh.task, {
|
|
225
279
|
status: "done",
|
|
226
280
|
verifyOutput: `exit ${check.status} in ${check.seconds}s\n${tail(check.output)}`,
|
|
227
281
|
})
|
|
@@ -230,8 +284,19 @@ export function runLoop({
|
|
|
230
284
|
}
|
|
231
285
|
}
|
|
232
286
|
|
|
233
|
-
|
|
234
|
-
|
|
287
|
+
/**
|
|
288
|
+
* The outcome of a run.
|
|
289
|
+
*
|
|
290
|
+
* @param {any[]} log
|
|
291
|
+
* @param {{ reason: string, code?: number, summary?: any, rounds?: number }} stop
|
|
292
|
+
* @returns {{ reason: string, code: number, rounds: number, log: any[], summary?: any }}
|
|
293
|
+
*/
|
|
294
|
+
function stop(log, { reason, code, summary, rounds }) {
|
|
295
|
+
// `rounds` is the number of rounds the loop RAN, not the number of
|
|
296
|
+
// entries in the log. The log only gets an entry after a verify that
|
|
297
|
+
// passed, so a run whose agent died on the first round reported zero
|
|
298
|
+
// rounds - which reads as "it did nothing" when it did one.
|
|
299
|
+
return { reason, code: code ?? 0, rounds: rounds ?? log.length, log, summary }
|
|
235
300
|
}
|
|
236
301
|
|
|
237
302
|
export { TASKS_PATH }
|
package/src/effort.js
CHANGED
|
@@ -16,7 +16,8 @@ import { loadProfile, resolveRoles, resolveDefaultVariant, resolveLaneModel, LAN
|
|
|
16
16
|
/**
|
|
17
17
|
* @typedef {import("./profile.js").Profile} Profile
|
|
18
18
|
* @typedef {import("./profile.js").EffortRequest} EffortRequest
|
|
19
|
-
* @typedef {
|
|
19
|
+
* @typedef {object} Config
|
|
20
|
+
* @property {Record<string, any>} [agent]
|
|
20
21
|
* @typedef {object} EffortResult
|
|
21
22
|
* @property {string} profileId
|
|
22
23
|
* @property {string} model
|
|
@@ -54,8 +55,19 @@ export function planEffort(profileId, opts = {}) {
|
|
|
54
55
|
)
|
|
55
56
|
}
|
|
56
57
|
for (const [role, want] of Object.entries(p.roles ?? {})) {
|
|
58
|
+
if (!want) continue
|
|
57
59
|
const got = roles[role]
|
|
58
|
-
if (
|
|
60
|
+
if (got === null) {
|
|
61
|
+
// The request was lost entirely: the lane's model publishes no such
|
|
62
|
+
// level, so no variant is emitted for it. This is the one adjustment
|
|
63
|
+
// worth reporting loudly, because it is the one where the lane is not
|
|
64
|
+
// running at the effort anybody asked for.
|
|
65
|
+
notes.push(
|
|
66
|
+
`lane ${role} asked for ${want}; its model does not publish that level, so no variant is set.`,
|
|
67
|
+
)
|
|
68
|
+
continue
|
|
69
|
+
}
|
|
70
|
+
if (want !== got) {
|
|
59
71
|
notes.push(`lane ${role} asked for ${want}; its model tops out at ${got}.`)
|
|
60
72
|
}
|
|
61
73
|
}
|
|
@@ -90,7 +102,7 @@ export function planEffort(profileId, opts = {}) {
|
|
|
90
102
|
*
|
|
91
103
|
* @param {Config} config
|
|
92
104
|
* @param {EffortResult} result
|
|
93
|
-
* @param {{
|
|
105
|
+
* @param {{ sessionModel?: string }} [opts]
|
|
94
106
|
* @returns {Config}
|
|
95
107
|
*/
|
|
96
108
|
export function applyEffort(config, result, opts = {}) {
|
|
@@ -128,7 +140,6 @@ export function applyEffort(config, result, opts = {}) {
|
|
|
128
140
|
cfg.harness = {
|
|
129
141
|
...(cfg.harness ?? {}),
|
|
130
142
|
profile: result.profileId,
|
|
131
|
-
...(opts.fallbackProfileId ? { fallback: opts.fallbackProfileId } : {}),
|
|
132
143
|
}
|
|
133
144
|
|
|
134
145
|
return config
|