opencode-harness-kit 0.2.1 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +303 -0
- package/GLOBAL.md +50 -13
- package/MIGRATION.md +140 -0
- package/README.md +233 -116
- package/bin/oc-harness.mjs +474 -96
- package/dist/index.js +378 -155
- package/docs/KNOWN-LIMITS.md +189 -4
- package/docs/RESEARCH.md +80 -5
- package/docs/ROUTING.md +254 -0
- package/lanes.json +136 -0
- package/lanes.schema.json +55 -0
- package/package.json +12 -8
- package/profiles/beam-free.json +51 -0
- package/profiles/big-pickle.json +45 -0
- package/profiles/deepseek-v4.1-flash-free.json +49 -0
- package/profiles/fleet-all.json +139 -0
- package/profiles/fleet-safe.json +139 -0
- package/profiles/glm-5.3-flash-free.json +49 -0
- package/profiles/index.json +120 -37
- package/profiles/longcat-2.5-preview-free.json +14 -10
- package/profiles/mimo-v2.6-flash-free.json +44 -0
- package/profiles/minimax-m3.1-flash-free.json +51 -0
- package/profiles/nemotron-3-ultra-free.json +44 -0
- package/profiles/nemotron-3.5-lightning-free.json +44 -0
- package/profiles/nemotron-3.5-super-vl-free.json +52 -0
- package/profiles/schema.json +30 -11
- package/{agents → prompts}/build.md +2 -35
- package/prompts/doc.md +42 -0
- package/{agents → prompts}/locate.md +1 -16
- package/prompts/longform.md +34 -0
- package/{agents → prompts}/plan.md +2 -25
- package/{agents → prompts}/reviewer.md +2 -22
- package/prompts/schema.md +30 -0
- package/prompts/sweep.md +29 -0
- package/{agents → prompts}/verify.md +6 -29
- package/prompts/vision.md +33 -0
- package/retention.yml +75 -57
- package/skills/space-build/SKILL.md +16 -0
- package/skills/space-review/SKILL.md +8 -0
- package/src/config.js +94 -69
- package/src/effort.js +59 -22
- package/src/index.ts +15 -2
- package/src/lane-usage.js +142 -0
- package/src/profile.js +502 -212
- package/src/router.ts +148 -39
- package/src/telemetry.ts +98 -0
- package/examples/muse-spark-1.3-contributor-free.json +0 -41
- package/profiles/deepseek-v4-flash-free.json +0 -41
- package/profiles/muse-split-free.json +0 -64
- package/profiles/space-bunny-free.json +0 -42
- package/profiles/x-preview-f-free.json +0 -41
package/src/config.js
CHANGED
|
@@ -3,8 +3,31 @@
|
|
|
3
3
|
// Config that does not depend on the model - permission allowlist, pruning
|
|
4
4
|
// defaults, experimental flags - is fixed here. Everything model-specific is
|
|
5
5
|
// derived from the profile, so switching models changes only the profile file.
|
|
6
|
+
//
|
|
7
|
+
// A fleet profile turns several models on at once, and there is one hard
|
|
8
|
+
// constraint that shapes the whole file:
|
|
9
|
+
//
|
|
10
|
+
// The model lives in the CONFIG, never in a markdown agent file.
|
|
11
|
+
//
|
|
12
|
+
// An agent file of the same name replaces the config entry wholesale, and a
|
|
13
|
+
// file without a `model:` line makes the agent inherit the caller's model
|
|
14
|
+
// instead. Both were verified against OpenCode 1.19: a conflicting file won,
|
|
15
|
+
// and a file with no model ran on the parent session's model. So the lanes
|
|
16
|
+
// live in `agent.<lane>` here, and their prompt bodies live in prompts/ and
|
|
17
|
+
// are referenced with a path opencode can resolve.
|
|
6
18
|
|
|
7
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
pruneThreshold,
|
|
21
|
+
reservedTokens,
|
|
22
|
+
inputCap,
|
|
23
|
+
effectiveRetention,
|
|
24
|
+
resolveRoleModel,
|
|
25
|
+
resolveRoles,
|
|
26
|
+
resolveDefaultVariant,
|
|
27
|
+
roleSplitModels,
|
|
28
|
+
promptRef,
|
|
29
|
+
loadLanes,
|
|
30
|
+
} from "./profile.js"
|
|
8
31
|
|
|
9
32
|
import { join } from "node:path"
|
|
10
33
|
import { homedir } from "node:os"
|
|
@@ -67,8 +90,8 @@ export const BASH_ALLOWLIST = [
|
|
|
67
90
|
"git rev-parse*", "git config --get*", "git remote*", "git blame*",
|
|
68
91
|
"git stash list*", "git ls-files*", "git describe*", "git shortlog*", "git --version*",
|
|
69
92
|
// `git -C <dir> <read-verb>` only. A bare `git -C *` would allow `git -C x push`.
|
|
70
|
-
"git -C * status*", "git -C * log*", "git -C * diff*",
|
|
71
|
-
"git -C * ls-files*", "git -C * show*",
|
|
93
|
+
"git -C * status*", "git -C * log*", "git -C * diff*",
|
|
94
|
+
"git -C * rev-parse*", "git -C * ls-files*", "git -C * show*",
|
|
72
95
|
// Staging is local and reversible; publishing is not and stays gated.
|
|
73
96
|
"git add*",
|
|
74
97
|
// `chmod 777` is denied below. `+x` is ordinary build work.
|
|
@@ -102,66 +125,62 @@ export const BASH_DENYLIST = [
|
|
|
102
125
|
"gh repo delete*",
|
|
103
126
|
]
|
|
104
127
|
|
|
105
|
-
/**
|
|
106
|
-
* Iteration cap per role, written as `agent.<role>.steps`.
|
|
107
|
-
*
|
|
108
|
-
* `steps` is a loop breaker rather than a quality knob. On exhaustion OpenCode
|
|
109
|
-
* forces a text-only response, so an agent that has stopped making progress
|
|
110
|
-
* reports what it has instead of silently looping until the context is gone.
|
|
111
|
-
* The caps are sized to the role's real work: build gets room because it runs
|
|
112
|
-
* the edit-verify-fix cycle, verify gets room because a full suite is many
|
|
113
|
-
* commands, and the read-only roles are capped tight because they have one job.
|
|
114
|
-
*/
|
|
115
|
-
export const AGENT_STEPS = /** @type {Record<string, number>} */ ({
|
|
116
|
-
build: 40,
|
|
117
|
-
plan: 25,
|
|
118
|
-
reviewer: 15,
|
|
119
|
-
locate: 12,
|
|
120
|
-
verify: 20,
|
|
121
|
-
})
|
|
122
|
-
|
|
123
|
-
/** Roles the kit writes, with the permissions that keep them honest. */
|
|
124
|
-
const AGENT_PERMISSIONS = /** @type {Record<string, Record<string, string>>} */ ({
|
|
125
|
-
plan: { edit: "deny" },
|
|
126
|
-
reviewer: { edit: "deny" },
|
|
127
|
-
locate: { edit: "deny", bash: "deny" },
|
|
128
|
-
verify: { edit: "deny" },
|
|
129
|
-
})
|
|
130
|
-
|
|
131
128
|
/**
|
|
132
129
|
* @param {Profile} p
|
|
133
|
-
* @param {{ base?: Record<string, any>, comments?: boolean }} [opts]
|
|
130
|
+
* @param {{ base?: Record<string, any>, comments?: boolean, unhealthy?: Set<string> }} [opts]
|
|
134
131
|
* @returns {string}
|
|
135
132
|
*/
|
|
136
133
|
export function generateConfig(p, opts = {}) {
|
|
137
134
|
const base = opts.base ?? {}
|
|
135
|
+
const unhealthy = opts.unhealthy
|
|
138
136
|
const model = `${p.provider}/${p.model}`
|
|
139
|
-
const roles = resolveRoles(p)
|
|
137
|
+
const roles = resolveRoles(p, { unhealthy })
|
|
140
138
|
const instructionsPath = globalInstructionsPath()
|
|
139
|
+
const lanes = loadLanes()
|
|
141
140
|
|
|
142
141
|
const agent = { ...(base.agent ?? {}) }
|
|
142
|
+
/** Lanes the primary agents are allowed to delegate to. */
|
|
143
|
+
const delegable = Object.keys(roles).filter(
|
|
144
|
+
(r) => lanes[r]?.mode !== "primary",
|
|
145
|
+
)
|
|
146
|
+
|
|
143
147
|
for (const [role, level] of Object.entries(roles)) {
|
|
144
148
|
if (level === undefined) continue
|
|
149
|
+
const lane = lanes[role]
|
|
145
150
|
const existing = { ...(agent[role] ?? {}) }
|
|
146
|
-
|
|
151
|
+
|
|
147
152
|
// The model is pinned, never inherited. Preserving a model already written
|
|
148
153
|
// on the agent is how `build` ended up on a `may-train` model while the rest
|
|
149
154
|
// of the file claimed the profile's zero-retention one: the compiler was
|
|
150
155
|
// asked to re-run over a hand-edited config and kept the hand-edit.
|
|
151
|
-
|
|
152
|
-
// `roleModels` is the one sanctioned way a role differs, because it is
|
|
153
|
-
// declared in the profile next to the model it points at and validated
|
|
154
|
-
// against retention.yml. Still an assignment, never a preserved value.
|
|
156
|
+
const rm = resolveRoleModel(p, role, { unhealthy })
|
|
155
157
|
existing.model = rm ? `${p.provider}/${rm.model}` : model
|
|
158
|
+
|
|
156
159
|
if (level === null) delete existing.variant
|
|
157
160
|
else existing.variant = level
|
|
158
|
-
|
|
161
|
+
|
|
162
|
+
if (lane) {
|
|
163
|
+
if (lane.description) existing.description = lane.description
|
|
164
|
+
if (lane.mode) existing.mode = lane.mode
|
|
165
|
+
if (lane.steps) existing.steps = lane.steps
|
|
166
|
+
existing.prompt = promptRef(role)
|
|
167
|
+
if (lane.permission) {
|
|
168
|
+
existing.permission = {
|
|
169
|
+
...(existing.permission ?? {}),
|
|
170
|
+
...lane.permission,
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// A primary agent may hand work to a lane. The rule is per-agent, and it
|
|
176
|
+
// is last-match-wins, so the allow list has to come after the deny.
|
|
177
|
+
if (lane?.mode === "primary" && delegable.length) {
|
|
159
178
|
existing.permission = {
|
|
160
179
|
...(existing.permission ?? {}),
|
|
161
|
-
...
|
|
180
|
+
task: { "*": "deny", ...Object.fromEntries(delegable.map((r) => [r, "allow"])) },
|
|
162
181
|
}
|
|
163
182
|
}
|
|
164
|
-
|
|
183
|
+
|
|
165
184
|
agent[role] = existing
|
|
166
185
|
}
|
|
167
186
|
|
|
@@ -179,7 +198,7 @@ export function generateConfig(p, opts = {}) {
|
|
|
179
198
|
delete compaction.tail_turns
|
|
180
199
|
compaction.auto = true
|
|
181
200
|
compaction.prune = true
|
|
182
|
-
compaction.reserved = reservedTokens(p)
|
|
201
|
+
compaction.reserved = reservedTokens(p, { unhealthy })
|
|
183
202
|
|
|
184
203
|
const base_ = { ...base }
|
|
185
204
|
// Two keys are dropped from the inherited config outright rather than merged.
|
|
@@ -202,7 +221,7 @@ export function generateConfig(p, opts = {}) {
|
|
|
202
221
|
// No top-level `variant`. The 1.18.35 Config schema sets
|
|
203
222
|
// `additionalProperties: false` and has no `variant` key, so OpenCode
|
|
204
223
|
// strips it from the resolved config - verified with `opencode debug
|
|
205
|
-
// config`. The per-
|
|
224
|
+
// config`. The per-lane tiering is carried by `agent.<lane>.variant`,
|
|
206
225
|
// which is a real schema key, and `effort.js` sets the session default at
|
|
207
226
|
// the `config` hook where the runtime does still see it.
|
|
208
227
|
default_agent: base.default_agent ?? "build",
|
|
@@ -240,11 +259,11 @@ export function generateConfig(p, opts = {}) {
|
|
|
240
259
|
// `allow` here is load-bearing in a way that is easy to get backwards, and
|
|
241
260
|
// it is pinned rather than inherited for the same reason the model is: an
|
|
242
261
|
// inherited `ask` survives regeneration and re-breaks every read-only
|
|
243
|
-
//
|
|
262
|
+
// lane.
|
|
244
263
|
//
|
|
245
264
|
// OpenCode appends the global rule AFTER the agent's own, and last match
|
|
246
265
|
// wins, so a global `edit: "ask"` silently overrides every agent's
|
|
247
|
-
// `edit: "deny"` and the read-only
|
|
266
|
+
// `edit: "deny"` and the read-only lanes can edit files after a prompt.
|
|
248
267
|
// `allow` is the implicit default and emits no rule at all, which leaves
|
|
249
268
|
// the per-agent `deny` intact. Writes outside the project are still
|
|
250
269
|
// gated by `external_directory`, which defaults to `ask`.
|
|
@@ -265,33 +284,37 @@ export function generateConfig(p, opts = {}) {
|
|
|
265
284
|
harness: {
|
|
266
285
|
profile: p.id,
|
|
267
286
|
...(p.fallback?.length ? { fallback: p.fallback[0] } : {}),
|
|
268
|
-
threshold: pruneThreshold(p),
|
|
287
|
+
threshold: pruneThreshold(p, { unhealthy }),
|
|
269
288
|
},
|
|
270
289
|
}
|
|
271
290
|
|
|
272
|
-
return render(out, p, { comments: opts.comments ?? true, instructionsPath })
|
|
291
|
+
return render(out, p, { comments: opts.comments ?? true, instructionsPath, unhealthy })
|
|
273
292
|
}
|
|
274
293
|
|
|
275
294
|
/**
|
|
276
295
|
* @param {Record<string, any>} cfg
|
|
277
296
|
* @param {Profile} p
|
|
278
|
-
* @param {{ comments: boolean, instructionsPath?: string }} opts
|
|
297
|
+
* @param {{ comments: boolean, instructionsPath?: string, unhealthy?: Set<string> }} opts
|
|
279
298
|
* @returns {string}
|
|
280
299
|
*/
|
|
281
300
|
function render(cfg, p, opts) {
|
|
282
301
|
const c = opts.comments ? "// " : ""
|
|
283
302
|
const instructionsPath = opts.instructionsPath ?? globalInstructionsPath()
|
|
284
|
-
const
|
|
303
|
+
const unhealthy = opts.unhealthy
|
|
304
|
+
const lanes = loadLanes()
|
|
285
305
|
const levels = p.effort?.levels ?? []
|
|
286
|
-
const roles = resolveRoles(p)
|
|
287
|
-
|
|
288
|
-
const effective = effectiveRetention(p)
|
|
289
|
-
// The input cap that actually governs context management, which on a
|
|
290
|
-
// profile is the
|
|
291
|
-
// it, or `reserved` and `prune` read as arithmetic against a number that
|
|
292
|
-
// not the one they used.
|
|
293
|
-
const effectiveInput = inputCap(p)
|
|
306
|
+
const roles = resolveRoles(p, { unhealthy })
|
|
307
|
+
|
|
308
|
+
const effective = effectiveRetention(p, { unhealthy })
|
|
309
|
+
// The input cap that actually governs context management, which on a fleet
|
|
310
|
+
// profile is the NARROWEST lane model rather than the base. The header must
|
|
311
|
+
// name it, or `reserved` and `prune` read as arithmetic against a number that
|
|
312
|
+
// is not the one they used.
|
|
313
|
+
const effectiveInput = inputCap(p, { unhealthy })
|
|
294
314
|
const splits = Object.keys(p.roleModels ?? {})
|
|
315
|
+
const laneRows = roleSplitModels(p, { unhealthy })
|
|
316
|
+
const isFleet = splits.length > 0
|
|
317
|
+
|
|
295
318
|
const head = [
|
|
296
319
|
`${c}Generated by opencode-harness-kit. Edit profiles/${p.id}.json instead`,
|
|
297
320
|
`${c}of this file, then re-run: oc-harness use ${p.id}`,
|
|
@@ -300,35 +323,37 @@ function render(cfg, p, opts) {
|
|
|
300
323
|
`${c}model ${p.provider}/${p.model}`,
|
|
301
324
|
...(splits.length
|
|
302
325
|
? [
|
|
303
|
-
`${c}
|
|
304
|
-
`${c} (roles not listed above run on the base model above)`,
|
|
326
|
+
`${c}lanes ${splits.length} enabled, each on the first model its chain allows`,
|
|
305
327
|
]
|
|
306
328
|
: []),
|
|
307
|
-
`${c}context ${p.limits.context.toLocaleString("en-US")} tokens`,
|
|
308
|
-
`${c}input cap ${
|
|
329
|
+
`${c}context ${p.limits.context.toLocaleString("en-US")} tokens${isFleet ? " (base model)" : ""}`,
|
|
330
|
+
`${c}input cap ${effectiveInput.toLocaleString("en-US")} tokens${effectiveInput !== (p.limits.input ?? p.limits.context) ? " (narrowest lane; governs the shared reserve)" : ""}`,
|
|
309
331
|
`${c}output cap ${p.limits.output.toLocaleString("en-US")} tokens`,
|
|
310
332
|
`${c}reasoning ${levels.length ? levels.join(" < ") : "(not tunable)"}`,
|
|
311
|
-
// The header must name the retention that actually applies, which with a
|
|
312
|
-
// split is the least private model in the file, not the base one. Printing
|
|
313
|
-
// `p.retention` here is the same class of bug as the inherited `build`
|
|
314
|
-
// model: the file claims a privacy property the agents do not have.
|
|
315
333
|
`${c}retention ${effective.retention}${effective.retention === "may-train" ? " <-- prompts may train a model" : ""}`,
|
|
316
334
|
...(effective.retention !== p.retention
|
|
317
335
|
? [
|
|
318
336
|
`${c} base ${p.provider}/${p.model} is ${p.retention}, but ${effective.worst} makes the whole file ${effective.retention}`,
|
|
319
337
|
]
|
|
320
338
|
: []),
|
|
339
|
+
...(p.maxRetention && p.maxRetention !== p.retention
|
|
340
|
+
? [
|
|
341
|
+
`${c}ceiling ${p.maxRetention} - a lane may only resolve to a model at or below this`,
|
|
342
|
+
]
|
|
343
|
+
: []),
|
|
321
344
|
`${c}`,
|
|
322
345
|
`${c}Derived numbers, do not hand-edit:`,
|
|
323
|
-
`${c} compaction.reserved ${reservedTokens(p).toLocaleString("en-US")} (${Math.round((p.effort?.reserveShare ?? 0.12) * 100)}% of the
|
|
324
|
-
`${c} prune threshold ${pruneThreshold(p).toLocaleString("en-US")} (
|
|
346
|
+
`${c} compaction.reserved ${reservedTokens(p, { unhealthy }).toLocaleString("en-US")} (${Math.round((p.effort?.reserveShare ?? 0.12) * 100)}% of the narrowest input cap in play, held back for reasoning growth)`,
|
|
347
|
+
`${c} prune threshold ${pruneThreshold(p, { unhealthy }).toLocaleString("en-US")} (narrowest input cap minus reserve)`,
|
|
325
348
|
`${c} role effort ${Object.entries(roles).map(([k, v]) => `${k}=${v ?? "unset"}`).join(" ")}`,
|
|
326
|
-
...(
|
|
349
|
+
...(laneRows.length
|
|
327
350
|
? [
|
|
328
|
-
`${c}
|
|
351
|
+
`${c} lane models ${laneRows.map((r) => `${r.role}=${r.model}${r.limits ? ` (out ${r.limits.output.toLocaleString("en-US")})` : ""}`).join(" ")}`,
|
|
329
352
|
]
|
|
330
353
|
: []),
|
|
331
|
-
`${c} agent steps ${Object.entries(
|
|
354
|
+
`${c} agent steps ${Object.entries(roles)
|
|
355
|
+
.map(([k]) => `${k}=${lanes[k]?.steps ?? 0}`)
|
|
356
|
+
.join(" ")}`,
|
|
332
357
|
]
|
|
333
358
|
|
|
334
359
|
if (!levels.length) {
|
|
@@ -336,7 +361,7 @@ function render(cfg, p, opts) {
|
|
|
336
361
|
} else {
|
|
337
362
|
head.push(
|
|
338
363
|
`${c}`,
|
|
339
|
-
`${c}Effort is per-
|
|
364
|
+
`${c}Effort is per-lane above. There is no top-level "variant" key: the 1.18.35`,
|
|
340
365
|
`${c}Config schema rejects it, and the router sets the session default at runtime.`,
|
|
341
366
|
)
|
|
342
367
|
}
|
package/src/effort.js
CHANGED
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
// against 7.8s for `xhigh` on a different model, while 77% of responses ended
|
|
8
8
|
// on a tool call rather than prose.
|
|
9
9
|
//
|
|
10
|
-
// A level is never emitted raw. It is resolved against the
|
|
11
|
-
//
|
|
12
|
-
// the nearest level that actually exists.
|
|
10
|
+
// A level is never emitted raw. It is resolved against the levels of the model
|
|
11
|
+
// the lane actually runs on, so a lane asking for `max` on a model without
|
|
12
|
+
// `max` gets the nearest level that actually exists.
|
|
13
13
|
|
|
14
|
-
import { loadProfile, resolveRoles, resolveDefaultVariant } from "./profile.js"
|
|
14
|
+
import { loadProfile, resolveRoles, resolveDefaultVariant, resolveLaneModel, LANE_NAMES } from "./profile.js"
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* @typedef {import("./profile.js").Profile} Profile
|
|
@@ -22,39 +22,56 @@ import { loadProfile, resolveRoles, resolveDefaultVariant } from "./profile.js"
|
|
|
22
22
|
* @property {string} model
|
|
23
23
|
* @property {string|null} defaultVariant
|
|
24
24
|
* @property {Record<string, string|null>} roles
|
|
25
|
+
* @property {Record<string, string>} laneModels
|
|
25
26
|
* @property {string[]} notes
|
|
26
27
|
*/
|
|
27
28
|
|
|
28
|
-
/**
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The lanes a profile manages. A lane a profile does not mention is left alone.
|
|
31
|
+
*
|
|
32
|
+
* This used to be a hand-written list of nine lanes, and it drifted: the
|
|
33
|
+
* fleet has ten, and `longform` was missing. A second list of lane names is a
|
|
34
|
+
* second source of truth, and the lane table in `lanes.json` is the first one.
|
|
35
|
+
*
|
|
36
|
+
* @type {string[]}
|
|
37
|
+
*/
|
|
38
|
+
export const MANAGED_ROLES = LANE_NAMES
|
|
30
39
|
|
|
31
40
|
/**
|
|
32
41
|
* @param {string} profileId
|
|
42
|
+
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
33
43
|
* @returns {EffortResult}
|
|
34
44
|
*/
|
|
35
|
-
export function planEffort(profileId) {
|
|
45
|
+
export function planEffort(profileId, opts = {}) {
|
|
36
46
|
const p = loadProfile(profileId)
|
|
37
47
|
const levels = p.effort?.levels ?? []
|
|
38
|
-
const roles = resolveRoles(p)
|
|
48
|
+
const roles = resolveRoles(p, opts)
|
|
39
49
|
const notes = []
|
|
40
50
|
|
|
41
51
|
if (levels.length === 0) {
|
|
42
52
|
notes.push(
|
|
43
|
-
`${p.model} publishes no tunable reasoning effort; variant will be omitted for every
|
|
53
|
+
`${p.model} publishes no tunable reasoning effort; variant will be omitted for every lane.`,
|
|
44
54
|
)
|
|
45
55
|
}
|
|
46
56
|
for (const [role, want] of Object.entries(p.roles ?? {})) {
|
|
47
57
|
const got = roles[role]
|
|
48
58
|
if (want && got && want !== got) {
|
|
49
|
-
notes.push(`
|
|
59
|
+
notes.push(`lane ${role} asked for ${want}; its model tops out at ${got}.`)
|
|
50
60
|
}
|
|
51
61
|
}
|
|
52
62
|
|
|
63
|
+
/** @type {Record<string, string>} */
|
|
64
|
+
const laneModels = {}
|
|
65
|
+
for (const role of Object.keys(p.roles ?? {})) {
|
|
66
|
+
laneModels[role] = resolveLaneModel(p, role, opts).model
|
|
67
|
+
}
|
|
68
|
+
|
|
53
69
|
return {
|
|
54
70
|
profileId,
|
|
55
71
|
model: `${p.provider}/${p.model}`,
|
|
56
72
|
defaultVariant: resolveDefaultVariant(p),
|
|
57
73
|
roles,
|
|
74
|
+
laneModels,
|
|
58
75
|
notes,
|
|
59
76
|
}
|
|
60
77
|
}
|
|
@@ -65,34 +82,54 @@ export function planEffort(profileId) {
|
|
|
65
82
|
*
|
|
66
83
|
* `AgentConfig.variant` only applies to the agent's own configured model, so
|
|
67
84
|
* the model is written alongside the variant instead of relying on the global
|
|
68
|
-
* default. That is also what makes per-
|
|
85
|
+
* default. That is also what makes per-lane effort work at all.
|
|
86
|
+
*
|
|
87
|
+
* The model is only filled in when the agent does not have one. A fleet puts
|
|
88
|
+
* ten different models behind ten agents, and overwriting them with the
|
|
89
|
+
* profile's base model here would collapse the whole fleet onto one route.
|
|
69
90
|
*
|
|
70
91
|
* @param {Config} config
|
|
71
92
|
* @param {EffortResult} result
|
|
72
|
-
* @param {{ fallbackProfileId?: string }} [opts]
|
|
93
|
+
* @param {{ fallbackProfileId?: string, sessionModel?: string }} [opts]
|
|
73
94
|
* @returns {Config}
|
|
74
95
|
*/
|
|
75
96
|
export function applyEffort(config, result, opts = {}) {
|
|
76
97
|
const cfg = /** @type {Record<string, any>} */ (config)
|
|
77
98
|
|
|
78
|
-
//
|
|
79
|
-
// and
|
|
80
|
-
|
|
99
|
+
// The session model is the profile's base, unless the router moved it. A
|
|
100
|
+
// fleet whose base route is spent keeps its lanes and moves the session
|
|
101
|
+
// onto a fallback; pinning the base back here would undo that failover and
|
|
102
|
+
// send the session back to the dead route.
|
|
103
|
+
const sessionModel =
|
|
104
|
+
typeof opts.sessionModel === "string" && opts.sessionModel
|
|
105
|
+
? opts.sessionModel
|
|
106
|
+
: result.model
|
|
107
|
+
// Always pin the session model. When the model has no tunable effort we
|
|
108
|
+
// still set it and simply omit `variant` below.
|
|
109
|
+
cfg.model = sessionModel
|
|
81
110
|
|
|
82
111
|
cfg.agent = cfg.agent ?? {}
|
|
83
|
-
for (const role of
|
|
84
|
-
|
|
85
|
-
if (level === undefined) continue // role not described by this profile
|
|
112
|
+
for (const [role, level] of Object.entries(result.roles)) {
|
|
113
|
+
if (level === undefined) continue
|
|
86
114
|
const existing = cfg.agent[role] ?? {}
|
|
87
|
-
if (!existing.model)
|
|
115
|
+
if (!existing.model) {
|
|
116
|
+
existing.model = result.laneModels?.[role]
|
|
117
|
+
? `${result.model.split("/")[0]}/${result.laneModels[role]}`
|
|
118
|
+
: result.model
|
|
119
|
+
}
|
|
88
120
|
if (level === null) delete existing.variant
|
|
89
121
|
else existing.variant = level
|
|
90
122
|
cfg.agent[role] = existing
|
|
91
123
|
}
|
|
92
124
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
125
|
+
// The router publishes `sessionModel` and `threshold` on the same object.
|
|
126
|
+
// Replacing the whole thing here dropped them, which is how a fleet that
|
|
127
|
+
// had failed over ended up with a session model nobody chose.
|
|
128
|
+
cfg.harness = {
|
|
129
|
+
...(cfg.harness ?? {}),
|
|
130
|
+
profile: result.profileId,
|
|
131
|
+
...(opts.fallbackProfileId ? { fallback: opts.fallbackProfileId } : {}),
|
|
132
|
+
}
|
|
96
133
|
|
|
97
134
|
return config
|
|
98
135
|
}
|
package/src/index.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { join } from "node:path"
|
|
|
15
15
|
import type { Plugin } from "@opencode-ai/plugin"
|
|
16
16
|
import { GuardPlugin } from "./guard.js"
|
|
17
17
|
import { PrunePlugin } from "./prune.js"
|
|
18
|
-
import { RouterPlugin } from "./router.js"
|
|
18
|
+
import { RouterPlugin, unhealthyModels } from "./router.js"
|
|
19
19
|
import { TelemetryPlugin } from "./telemetry.js"
|
|
20
20
|
import { applyEffort, planEffort } from "./effort.js"
|
|
21
21
|
import { profileIdFromConfig } from "./profile.js"
|
|
@@ -63,7 +63,20 @@ export const HarnessPlugin: Plugin = async (ctx, options?: HarnessOptions) => {
|
|
|
63
63
|
const id = opts.profile ?? config?.harness?.profile ?? profileIdFromConfig()
|
|
64
64
|
if (!id) return
|
|
65
65
|
try {
|
|
66
|
-
|
|
66
|
+
// The same set of spent routes the router used. A lane that failed over
|
|
67
|
+
// to a model with fewer effort levels must not keep the variant the
|
|
68
|
+
// previous model had.
|
|
69
|
+
//
|
|
70
|
+
// The session model is read before the effort plan is applied, because
|
|
71
|
+
// the router may have moved it onto a fallback while the base route was
|
|
72
|
+
// spent. Pinning the base back here would undo that failover.
|
|
73
|
+
const sessionModel =
|
|
74
|
+
typeof config?.harness?.sessionModel === "string"
|
|
75
|
+
? config.harness.sessionModel
|
|
76
|
+
: undefined
|
|
77
|
+
applyEffort(config, planEffort(id, { unhealthy: unhealthyModels(ctx.worktree) }), {
|
|
78
|
+
sessionModel,
|
|
79
|
+
})
|
|
67
80
|
// Record it where the prune hook looks, so pruning picks up this
|
|
68
81
|
// profile's thresholds without access to the resolved config.
|
|
69
82
|
const dir = join(ctx.worktree, ".space")
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// Reading the lane telemetry back.
|
|
2
|
+
//
|
|
3
|
+
// The plugin writes it; this reads it. It is the other half of the "which
|
|
4
|
+
// lanes earn their place" question, and it is separated from the CLI so it
|
|
5
|
+
// can be tested without spawning a process.
|
|
6
|
+
//
|
|
7
|
+
// Two rules make the numbers mean something:
|
|
8
|
+
//
|
|
9
|
+
// 1. A CALL is one assistant message, not one event. opencode emits
|
|
10
|
+
// `message.updated` several times per message, and a counter that read
|
|
11
|
+
// every event reported three calls for every call made.
|
|
12
|
+
//
|
|
13
|
+
// 2. The file is found by walking UP from the working directory. The plugin
|
|
14
|
+
// writes to the project root, so reading `process.cwd()` alone reported
|
|
15
|
+
// "never called" for every lane whenever the command was run from a
|
|
16
|
+
// subdirectory - which is the normal way to run it.
|
|
17
|
+
//
|
|
18
|
+
// Plain JS with JSDoc types, like the rest of src/: the plugin loads .ts but
|
|
19
|
+
// the CLI has to run under plain node.
|
|
20
|
+
|
|
21
|
+
import { existsSync, readFileSync } from "node:fs"
|
|
22
|
+
import { dirname, join, resolve } from "node:path"
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @typedef {object} LaneUsage
|
|
26
|
+
* @property {number} requests distinct assistant messages that used the lane
|
|
27
|
+
* @property {number[]} ms durations of the requests that completed
|
|
28
|
+
* @property {Set<string>} models the models the lane was seen on
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The telemetry file for the project containing `startDir`.
|
|
33
|
+
*
|
|
34
|
+
* @param {string} startDir
|
|
35
|
+
* @returns {string | null} the path, or null when the project has no telemetry
|
|
36
|
+
*/
|
|
37
|
+
export function findTelemetryFile(startDir) {
|
|
38
|
+
let dir = resolve(startDir)
|
|
39
|
+
for (let i = 0; i < 6; i++) {
|
|
40
|
+
const file = join(dir, ".space", "telemetry.jsonl")
|
|
41
|
+
if (existsSync(file)) return file
|
|
42
|
+
const parent = dirname(dir)
|
|
43
|
+
if (parent === dir) break
|
|
44
|
+
dir = parent
|
|
45
|
+
}
|
|
46
|
+
return null
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The lane usage recorded in a telemetry file.
|
|
51
|
+
*
|
|
52
|
+
* Records are merged by message id: a lane record is written when the message
|
|
53
|
+
* appears and again when it completes, and only the second one carries a
|
|
54
|
+
* duration. A message is one request, and it contributes one duration.
|
|
55
|
+
*
|
|
56
|
+
* A REQUEST is one assistant message, which is one round trip to the
|
|
57
|
+
* provider. A lane that makes three tool calls makes three requests, so this
|
|
58
|
+
* is a count of provider traffic rather than of delegations - which is the
|
|
59
|
+
* number that decides whether a free route's quota can carry the fleet.
|
|
60
|
+
*
|
|
61
|
+
* Records without a message id are ignored. They come from kit versions
|
|
62
|
+
* before the counter was fixed, and their number is inflated by a factor of
|
|
63
|
+
* three, so counting them would put a known-wrong figure next a known-right
|
|
64
|
+
* one.
|
|
65
|
+
*
|
|
66
|
+
* @param {string | null} file
|
|
67
|
+
* @returns {{ lanes: Record<string, LaneUsage>, records: number, ignored: number }}
|
|
68
|
+
*/
|
|
69
|
+
export function readLaneUsage(file) {
|
|
70
|
+
/** @type {Record<string, LaneUsage>} */
|
|
71
|
+
const lanes = {}
|
|
72
|
+
/** @type {Map<string, { lane: string, ms?: number, model?: string }>} */
|
|
73
|
+
const byId = new Map()
|
|
74
|
+
let records = 0
|
|
75
|
+
let ignored = 0
|
|
76
|
+
|
|
77
|
+
if (!file || !existsSync(file)) return { lanes, records, ignored }
|
|
78
|
+
|
|
79
|
+
let text = ""
|
|
80
|
+
try {
|
|
81
|
+
text = readFileSync(file, "utf8")
|
|
82
|
+
} catch {
|
|
83
|
+
return { lanes, records, ignored }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (const line of text.split("\n")) {
|
|
87
|
+
if (!line.trim()) continue
|
|
88
|
+
let rec
|
|
89
|
+
try {
|
|
90
|
+
rec = JSON.parse(line)
|
|
91
|
+
} catch {
|
|
92
|
+
ignored++
|
|
93
|
+
continue
|
|
94
|
+
}
|
|
95
|
+
if (rec?.kind === "lane-call") {
|
|
96
|
+
// A record from the kit version whose counter was inflated threefold. It
|
|
97
|
+
// names a lane, but it cannot be correlated with a message, so counting
|
|
98
|
+
// it would double-count every call it shares with a `lane` record.
|
|
99
|
+
ignored++
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
if (rec?.kind !== "lane") continue
|
|
103
|
+
const lane = rec.agent
|
|
104
|
+
const id = rec.id
|
|
105
|
+
if (typeof lane !== "string" || !lane) continue
|
|
106
|
+
if (typeof id !== "string" || !id) {
|
|
107
|
+
ignored++
|
|
108
|
+
continue
|
|
109
|
+
}
|
|
110
|
+
records++
|
|
111
|
+
const prev = byId.get(id)
|
|
112
|
+
// A message is one call. The completed record carries the duration, so it
|
|
113
|
+
// wins over the open one; the open one is kept when the call never
|
|
114
|
+
// finished, because "was it called" is answered by the call existing.
|
|
115
|
+
if (prev && prev.ms !== undefined) continue
|
|
116
|
+
byId.set(id, {
|
|
117
|
+
lane,
|
|
118
|
+
ms: typeof rec.ms === "number" ? rec.ms : prev?.ms,
|
|
119
|
+
model: typeof rec.model === "string" ? rec.model : prev?.model,
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
for (const rec of byId.values()) {
|
|
124
|
+
const usage =
|
|
125
|
+
lanes[rec.lane] ?? (lanes[rec.lane] = { requests: 0, ms: [], models: new Set() })
|
|
126
|
+
usage.requests++
|
|
127
|
+
if (typeof rec.ms === "number") usage.ms.push(rec.ms)
|
|
128
|
+
if (rec.model) usage.models.add(rec.model)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return { lanes, records, ignored }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* The mean of a set of durations, rounded.
|
|
136
|
+
* @param {number[]} ms
|
|
137
|
+
* @returns {number | null}
|
|
138
|
+
*/
|
|
139
|
+
export function averageMs(ms) {
|
|
140
|
+
if (!ms.length) return null
|
|
141
|
+
return Math.round(ms.reduce((a, b) => a + b, 0) / ms.length)
|
|
142
|
+
}
|