opencode-harness-kit 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/README.md +217 -21
- package/bin/oc-harness.mjs +177 -10
- package/dist/index.js +54 -6
- package/dist/profile.d.ts +2 -1
- package/docs/KNOWN-LIMITS.md +22 -2
- package/package.json +1 -1
- package/profiles/beam-free.json +2 -3
- package/profiles/deepseek-v4.1-flash-free.json +1 -2
- package/profiles/fleet-all.json +6 -14
- package/profiles/fleet-safe.json +8 -19
- package/profiles/glm-5.3-flash-free.json +1 -2
- package/profiles/index.json +4 -11
- package/profiles/mimo-v2.6-flash-free.json +1 -2
- package/profiles/minimax-m3.1-flash-free.json +1 -2
- package/profiles/nemotron-3.5-lightning-free.json +2 -3
- package/profiles/nemotron-3.5-super-vl-free.json +1 -2
- package/profiles/schema.json +8 -0
- package/retention.yml +0 -9
- package/src/profile.js +14 -1
- package/src/prune.ts +90 -5
- package/src/research.js +462 -0
- package/profiles/longcat-2.5-preview-free.json +0 -45
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "glm-5.3-flash-free",
|
|
7
7
|
"family": "glm-flash",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"notes": "Zhipu. The only free model that publishes pdf input, and the only one with structured output and a zero-retention policy at the same time. The pdf does not work: the provider's endpoint rejects a `file` content part, and OpenCode will not read a binary file, so documents are extracted locally first. It measured 4.4-5.8s on the work a lane actually does, and it sits on the thorough lane because it is the careful one rather than the quick one.",
|
|
11
11
|
"limits": {
|
|
@@ -43,7 +43,6 @@
|
|
|
43
43
|
},
|
|
44
44
|
"fallback": [
|
|
45
45
|
"deepseek-v4.1-flash-free",
|
|
46
|
-
"longcat-2.5-preview-free",
|
|
47
46
|
"beam-free"
|
|
48
47
|
]
|
|
49
48
|
}
|
package/profiles/index.json
CHANGED
|
@@ -68,17 +68,6 @@
|
|
|
68
68
|
"default": false,
|
|
69
69
|
"note": "Structured output on a zero-retention provider. Careful rather than fast: 4.4-5.8s on the work a lane does."
|
|
70
70
|
},
|
|
71
|
-
{
|
|
72
|
-
"id": "longcat-2.5-preview-free",
|
|
73
|
-
"file": "profiles/longcat-2.5-preview-free.json",
|
|
74
|
-
"title": "LongCat 2.5 Preview Free",
|
|
75
|
-
"retention": "zero-retention",
|
|
76
|
-
"free": true,
|
|
77
|
-
"topEffort": null,
|
|
78
|
-
"context": 1000000,
|
|
79
|
-
"default": false,
|
|
80
|
-
"note": "1M context with image input and a zero-retention provider. A preview endpoint: it can be withdrawn without a rename."
|
|
81
|
-
},
|
|
82
71
|
{
|
|
83
72
|
"id": "nemotron-3.5-super-vl-free",
|
|
84
73
|
"file": "profiles/nemotron-3.5-super-vl-free.json",
|
|
@@ -156,6 +145,10 @@
|
|
|
156
145
|
{
|
|
157
146
|
"id": "mimo-v2.5-free",
|
|
158
147
|
"reason": "Withdrawn from the provider. Observed alive for one day. Its replacement on the tier is mimo-v2.6-flash-free."
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"id": "longcat-2.5-preview-free",
|
|
151
|
+
"reason": "Withdrawn from the provider. A preview endpoint, so it could be pulled without a rename - and it was: models.dev still carries the record and marks it deprecated. It was the longform lane's primary in both fleets, so every lane fell back to its second rung, and the profile named only this model had nothing left to fall back to and was retired."
|
|
159
152
|
}
|
|
160
153
|
]
|
|
161
154
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "mimo-v2.6-flash-free",
|
|
7
7
|
"family": "mimo",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"notes": "Xiaomi. The only free model that publishes audio input, and it read an image correctly in a live call, at 19.2s average - the slowest lane on the tier, and slower than minimax on the same picture. OpenCode 1.19 refuses to read a binary file at all, so the audio path is not reachable through the harness today; the vision path is. No effort levels, so no variant is written.",
|
|
11
11
|
"limits": {
|
|
@@ -38,7 +38,6 @@
|
|
|
38
38
|
"keepUserMessages": true
|
|
39
39
|
},
|
|
40
40
|
"fallback": [
|
|
41
|
-
"longcat-2.5-preview-free",
|
|
42
41
|
"minimax-m3.1-flash-free"
|
|
43
42
|
]
|
|
44
43
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "minimax-m3.1-flash-free",
|
|
7
7
|
"family": "minimax",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"notes": "Flagship coding and agent model on the free tier: 512k output tokens, the widest of any free route, and the fastest measured image reader at 7.7s. Tool calls are stable. It does not support temperature, and models.dev publishes no structured_output flag for it, so JSON output is requested in the prompt rather than negotiated with the API. Measured at 3.1s for a one-word reply.",
|
|
11
11
|
"limits": {
|
|
@@ -45,7 +45,6 @@
|
|
|
45
45
|
},
|
|
46
46
|
"fallback": [
|
|
47
47
|
"deepseek-v4.1-flash-free",
|
|
48
|
-
"longcat-2.5-preview-free",
|
|
49
48
|
"glm-5.3-flash-free"
|
|
50
49
|
]
|
|
51
50
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "nemotron-3.5-lightning-free",
|
|
7
7
|
"family": "nemotron-free",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"notes": "Its output cap equals its whole context window, so it is the lane for emitting a file larger than 128k tokens. It publishes no effort level, so it spends nothing on reasoning. It is usually the quickest lane on the tier - 3.7s average over four requests - but one measured call took 114s, so it is not on a latency-sensitive path.",
|
|
11
11
|
"limits": {
|
|
@@ -38,7 +38,6 @@
|
|
|
38
38
|
"keepUserMessages": true
|
|
39
39
|
},
|
|
40
40
|
"fallback": [
|
|
41
|
-
"big-pickle"
|
|
42
|
-
"longcat-2.5-preview-free"
|
|
41
|
+
"big-pickle"
|
|
43
42
|
]
|
|
44
43
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "nemotron-3.5-super-vl-free",
|
|
7
7
|
"family": "nemotron-free",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"notes": "NVIDIA 120B-A12B. The only free model that publishes a `none` effort level, which is what makes it the cheap no-reasoning sweep lane. The catalog says it takes images, but a live call with an image attachment hangs and never returns: it is used here for text and for reasoning off, never for pictures. The hang is specific to an attachment - a tool read of the same file answers in 4.2s - so the rule is 'never hand it a file', not 'never use it'.",
|
|
11
11
|
"limits": {
|
|
@@ -46,7 +46,6 @@
|
|
|
46
46
|
"keepUserMessages": true
|
|
47
47
|
},
|
|
48
48
|
"fallback": [
|
|
49
|
-
"longcat-2.5-preview-free",
|
|
50
49
|
"beam-free"
|
|
51
50
|
]
|
|
52
51
|
}
|
package/profiles/schema.json
CHANGED
|
@@ -267,6 +267,14 @@
|
|
|
267
267
|
"type": "boolean",
|
|
268
268
|
"default": true,
|
|
269
269
|
"description": "Keep the newest copy of each DISTINCT tool output, however old it is. The recent-calls window is a window on recency, and a long session walks out of it: after six more tool calls the only copy of a file the agent read is gone, and the agent reads it again. The long-task A/B measured seventeen re-reads and five times the input tokens for the same answer."
|
|
270
|
+
},
|
|
271
|
+
"keepDistinctBudgetChars": {
|
|
272
|
+
"type": [
|
|
273
|
+
"integer",
|
|
274
|
+
"null"
|
|
275
|
+
],
|
|
276
|
+
"minimum": 0,
|
|
277
|
+
"description": "The character budget the distinct outputs have to fit in, newest first. The ones past the budget are stubbed like everything else. Null or unset means the profile's own context budget (pruneThreshold), which is what the pruner is measured against: without a ceiling the rule keeps every output the session has ever produced, and on a real 137-message session that was 4.74 MB of a 5.26 MB request against a 176,000 character budget. 0 keeps no distinct output at all."
|
|
270
278
|
}
|
|
271
279
|
}
|
|
272
280
|
},
|
package/retention.yml
CHANGED
|
@@ -51,15 +51,6 @@ models:
|
|
|
51
51
|
verified: 2026-10-19
|
|
52
52
|
source: https://opencode.ai/docs/zen
|
|
53
53
|
|
|
54
|
-
opencode/longcat-2.5-preview-free:
|
|
55
|
-
retention: zero-retention
|
|
56
|
-
note: >-
|
|
57
|
-
"Its provider follows a zero-retention policy and does not use your data
|
|
58
|
-
for model training." Free for a limited time. A preview model: the
|
|
59
|
-
endpoint can be withdrawn without a rename.
|
|
60
|
-
verified: 2026-10-19
|
|
61
|
-
source: https://opencode.ai/docs/zen
|
|
62
|
-
|
|
63
54
|
# --- Retained: no training, but the provider keeps the data for a stated
|
|
64
55
|
# window. Not the same promise as zero-retention, so it gets its own value. ---
|
|
65
56
|
|
package/src/profile.js
CHANGED
|
@@ -77,7 +77,7 @@ export const ROOT = findRoot(resolve(HERE, ".."))
|
|
|
77
77
|
* @property {Record<string, RoleModel>} [roleModels]
|
|
78
78
|
* @property {{ enabled?: boolean, keepRecentToolCalls?: number,
|
|
79
79
|
* keepRecentTurns?: number, keepUserMessages?: boolean,
|
|
80
|
-
* keepDistinctOutputs?: boolean }} [prune]
|
|
80
|
+
* keepDistinctOutputs?: boolean, keepDistinctBudgetChars?: number | null }} [prune]
|
|
81
81
|
* @property {string[]} [fallback]
|
|
82
82
|
* @typedef {{ retention: Retention, note?: string, verified?: string, source?: string }} RetentionRecord
|
|
83
83
|
*/
|
|
@@ -214,6 +214,9 @@ export function normalize(p) {
|
|
|
214
214
|
keepRecentTurns: p.prune?.keepRecentTurns ?? 3,
|
|
215
215
|
keepUserMessages: p.prune?.keepUserMessages ?? true,
|
|
216
216
|
keepDistinctOutputs: p.prune?.keepDistinctOutputs ?? true,
|
|
217
|
+
// null means "the pruner's own budget decides" - the profile's context
|
|
218
|
+
// budget. A number is a ceiling in characters.
|
|
219
|
+
keepDistinctBudgetChars: p.prune?.keepDistinctBudgetChars ?? null,
|
|
217
220
|
},
|
|
218
221
|
fallback: p.fallback ?? [],
|
|
219
222
|
}
|
|
@@ -738,6 +741,16 @@ export function validate(p, path = "<memory>") {
|
|
|
738
741
|
) {
|
|
739
742
|
throw new ProfileError(`${where}: prune.keepDistinctOutputs must be a boolean`)
|
|
740
743
|
}
|
|
744
|
+
if (prune.keepDistinctBudgetChars !== undefined && prune.keepDistinctBudgetChars !== null) {
|
|
745
|
+
if (
|
|
746
|
+
!Number.isInteger(prune.keepDistinctBudgetChars) ||
|
|
747
|
+
prune.keepDistinctBudgetChars < 0
|
|
748
|
+
) {
|
|
749
|
+
throw new ProfileError(
|
|
750
|
+
`${where}: prune.keepDistinctBudgetChars must be an integer of at least 0`,
|
|
751
|
+
)
|
|
752
|
+
}
|
|
753
|
+
}
|
|
741
754
|
if (prune.enabled !== undefined && typeof prune.enabled !== "boolean") {
|
|
742
755
|
throw new ProfileError(`${where}: prune.enabled must be a boolean`)
|
|
743
756
|
}
|
package/src/prune.ts
CHANGED
|
@@ -174,6 +174,8 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
174
174
|
// Whether the newest copy of each DISTINCT tool output survives, even
|
|
175
175
|
// when it is older than the recent-calls window.
|
|
176
176
|
keepDistinct: boolean
|
|
177
|
+
// The character budget the distinct outputs have to fit in.
|
|
178
|
+
distinctBudget: number
|
|
177
179
|
threshold: number
|
|
178
180
|
}
|
|
179
181
|
const DISABLED: Resolved = {
|
|
@@ -182,12 +184,17 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
182
184
|
keepTurns: 0,
|
|
183
185
|
keepUsers: true,
|
|
184
186
|
keepDistinct: false,
|
|
187
|
+
distinctBudget: 0,
|
|
185
188
|
threshold: 0,
|
|
186
189
|
}
|
|
187
190
|
// Only a SUCCESSFUL lookup is cached. A lookup that found nothing is
|
|
188
191
|
// retried on the next request, so a project whose marker appears later
|
|
189
192
|
// still gets pruning.
|
|
190
193
|
let cached: Resolved | null = null
|
|
194
|
+
// A broken profile is reported once per session, not once per request:
|
|
195
|
+
// only a SUCCESSFUL lookup is cached, so a failing one is retried on every
|
|
196
|
+
// request and would otherwise fill the console with the same line.
|
|
197
|
+
let reportedFailure = false
|
|
191
198
|
|
|
192
199
|
const resolve = (): Resolved => {
|
|
193
200
|
if (cached) return cached
|
|
@@ -195,17 +202,50 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
195
202
|
try {
|
|
196
203
|
const id = activeProfileId(project)
|
|
197
204
|
if (id) profile = loadProfile(id)
|
|
198
|
-
} catch {
|
|
205
|
+
} catch (err) {
|
|
199
206
|
profile = null
|
|
207
|
+
if (!reportedFailure) {
|
|
208
|
+
reportedFailure = true
|
|
209
|
+
// A profile that cannot be loaded is not a reason to prune silently.
|
|
210
|
+
//
|
|
211
|
+
// The catch around the transform below logs when a prune FAILS, and
|
|
212
|
+
// this one did not: a marker pointing at a renamed or missing
|
|
213
|
+
// profile turned the pruner off with no line in the log, nothing on
|
|
214
|
+
// the console, and nothing to show the user why the context kept
|
|
215
|
+
// growing. A project in that state cannot tell the difference
|
|
216
|
+
// between "the pruner has nothing to do" and "the pruner is broken".
|
|
217
|
+
console.error(
|
|
218
|
+
`[opencode-harness-kit] prune disabled: ${
|
|
219
|
+
err instanceof Error ? err.message : String(err)
|
|
220
|
+
}`,
|
|
221
|
+
)
|
|
222
|
+
}
|
|
200
223
|
}
|
|
201
224
|
if (!profile?.prune?.enabled) return DISABLED
|
|
225
|
+
const threshold = pruneThreshold(profile)
|
|
202
226
|
cached = {
|
|
203
227
|
profile,
|
|
204
228
|
keepCalls: profile.prune.keepRecentToolCalls ?? 6,
|
|
205
229
|
keepTurns: profile.prune.keepRecentTurns ?? 3,
|
|
206
230
|
keepUsers: profile.prune.keepUserMessages ?? true,
|
|
207
231
|
keepDistinct: profile.prune.keepDistinctOutputs ?? true,
|
|
208
|
-
|
|
232
|
+
// The budget the distinct outputs have to fit in.
|
|
233
|
+
//
|
|
234
|
+
// It defaults to the profile's own context budget - the number the
|
|
235
|
+
// pruner is measured against - because a rule with no ceiling is not a
|
|
236
|
+
// trade-off, it is a leak. Measured on a real 137-message session: the
|
|
237
|
+
// request was 5.26 MB and the pruner removed 0.04 MB of it (0.8%),
|
|
238
|
+
// because EVERY distinct output the session had ever produced was kept
|
|
239
|
+
// (4.74 MB, 92% of the request) "however old it is". The threshold
|
|
240
|
+
// that session was supposed to live under was 176,000 characters and
|
|
241
|
+
// the request was 5,140,000 - twenty-nine times over.
|
|
242
|
+
//
|
|
243
|
+
// The ceiling keeps the property the rule was added for (the newest
|
|
244
|
+
// copy of a file the agent read survives, so it does not read it
|
|
245
|
+
// again) and drops the part that was never wanted (the copy of every
|
|
246
|
+
// file the session has EVER touched, for ever).
|
|
247
|
+
distinctBudget: profile.prune.keepDistinctBudgetChars ?? threshold,
|
|
248
|
+
threshold,
|
|
209
249
|
}
|
|
210
250
|
return cached
|
|
211
251
|
}
|
|
@@ -213,7 +253,7 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
213
253
|
return {
|
|
214
254
|
"experimental.chat.messages.transform": async (_input: any, output: any) => {
|
|
215
255
|
try {
|
|
216
|
-
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, threshold } = resolve()
|
|
256
|
+
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, distinctBudget, threshold } = resolve()
|
|
217
257
|
if (!profile) return
|
|
218
258
|
const messages = output.messages
|
|
219
259
|
if (!Array.isArray(messages) || messages.length === 0) return
|
|
@@ -257,17 +297,62 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
257
297
|
// same input have the same output.
|
|
258
298
|
if (keepDistinct) {
|
|
259
299
|
const seenSignatures = new Set<string>()
|
|
300
|
+
// What the distinct outputs kept so far cost the request.
|
|
301
|
+
let kept = 0
|
|
260
302
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
261
303
|
const msg = messages[i]
|
|
262
304
|
if (!msg?.parts) continue
|
|
305
|
+
// The size of the NEW distinct outputs in this message. A message
|
|
306
|
+
// that repeats an output it already has adds nothing, so it is
|
|
307
|
+
// not charged twice.
|
|
308
|
+
let added = 0
|
|
263
309
|
for (const part of msg.parts) {
|
|
264
310
|
if (part?.type !== "tool") continue
|
|
265
311
|
if (part.state?.status !== "completed") continue
|
|
266
312
|
const sig = toolSignature(part)
|
|
267
|
-
if (!sig
|
|
313
|
+
if (!sig) continue
|
|
314
|
+
const seen = seenSignatures.has(sig)
|
|
268
315
|
seenSignatures.add(sig)
|
|
269
|
-
|
|
316
|
+
// A REPEAT is never kept by this rule - the newest copy is.
|
|
317
|
+
if (seen) continue
|
|
318
|
+
// Outside the prunable floor the part survives anyway, and it
|
|
319
|
+
// is the newest copy of its output, so it owns that output: an
|
|
320
|
+
// older copy of the same thing is stubbed and this one costs
|
|
321
|
+
// the budget nothing.
|
|
322
|
+
if (i >= floor) continue
|
|
323
|
+
added += partSize(part)
|
|
270
324
|
}
|
|
325
|
+
if (added === 0) continue
|
|
326
|
+
// The ceiling has to charge what the rule actually KEEPS, and the
|
|
327
|
+
// rule keeps the WHOLE message - every part in it, verbatim
|
|
328
|
+
// (see the keepToolMessageIds branch in the main loop).
|
|
329
|
+
//
|
|
330
|
+
// Charging only the tool parts left the rest of the message free:
|
|
331
|
+
// a message with a 300 character tool output and a 4,000
|
|
332
|
+
// character reasoning part cost the budget 300 and kept 4,300.
|
|
333
|
+
// With a 5,000 character ceiling, twenty such messages kept
|
|
334
|
+
// 40,000 characters of reasoning the ceiling had already ruled
|
|
335
|
+
// out - the rule kept eight times its own budget, and the
|
|
336
|
+
// reasoning it carried along was exactly what the pruner exists
|
|
337
|
+
// to stub. There is no floor under which a reasoning part is
|
|
338
|
+
// left alone, and the ceiling is not one either.
|
|
339
|
+
added = (msg.parts ?? []).reduce(
|
|
340
|
+
(n: number, p: any) => n + partSize(p),
|
|
341
|
+
0,
|
|
342
|
+
)
|
|
343
|
+
// The ceiling. The walk is newest-first, so what the agent is
|
|
344
|
+
// working with survives and what the session touched at the start
|
|
345
|
+
// is stubbed like everything else.
|
|
346
|
+
//
|
|
347
|
+
// Without it the rule is not a trade-off but a leak: measured on a
|
|
348
|
+
// real 137-message session, the request was 5.26 MB, the pruner
|
|
349
|
+
// removed 0.04 MB of it, and 4.74 MB of it - 92% - was the
|
|
350
|
+
// distinct outputs of the whole session, kept "however old it
|
|
351
|
+
// is". The profile's own budget for that request was 176,000
|
|
352
|
+
// characters.
|
|
353
|
+
if (kept + added > distinctBudget) break
|
|
354
|
+
kept += added
|
|
355
|
+
keepToolMessageIds.add(msg.info?.id ?? String(i))
|
|
271
356
|
}
|
|
272
357
|
}
|
|
273
358
|
|