opencode-harness-kit 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +0 -2
- package/README.md +10 -6
- package/bin/oc-harness.mjs +176 -9
- package/dist/health.d.ts +15 -0
- package/dist/index.js +52 -16
- package/dist/router.d.ts +16 -0
- package/eval/ab.mjs +1 -1
- package/package.json +1 -1
- package/profiles/fleet-all.json +10 -10
- package/profiles/fleet-safe.json +16 -16
- package/profiles/flint-40b-free.json +50 -0
- package/profiles/index.json +39 -23
- package/profiles/mimo-v2.6-flash-free.json +2 -4
- package/retention.yml +17 -8
- package/src/health.js +51 -4
- package/src/router.ts +72 -15
- package/profiles/minimax-m3.1-flash-free.json +0 -50
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -107,7 +107,7 @@ about your prompts.
|
|
|
107
107
|
|
|
108
108
|
| lane | job | model | measured |
|
|
109
109
|
|---|---|---|---|
|
|
110
|
-
| `build` | write and verify the change |
|
|
110
|
+
| `build` | write and verify the change | Flint 40B Free | 6.9 s |
|
|
111
111
|
| `plan` | turn a request into a plan | Beam | **6.1 s** |
|
|
112
112
|
| `reviewer` | read a change in two passes | Nemotron 3 Ultra | 7.8 s (34 s cold) |
|
|
113
113
|
| `locate` | find where something lives | Nemotron 3.5 Lightning | 6.9 s |
|
|
@@ -116,13 +116,14 @@ about your prompts.
|
|
|
116
116
|
| `doc` | read a long document or a spec | GLM 5.3 Flash | 7.4 s |
|
|
117
117
|
| `schema` | produce JSON or a JSON schema | DeepSeek V4.1 Flash | 10.7 s |
|
|
118
118
|
| `sweep` | sweep a lot of code mechanically | Nemotron 3.5 Super VL | 8.4 s |
|
|
119
|
-
| `longform` | read a megabyte, or write alternatives |
|
|
119
|
+
| `longform` | read a megabyte, or write alternatives | Flint 40B Free | 6.9 s |
|
|
120
120
|
|
|
121
121
|
`longform` used to run on `longcat-2.5-preview-free`, and the provider
|
|
122
122
|
withdrew it. A withdrawn model is not a lane, it is a hole in a lane: the
|
|
123
|
-
route falls to the next rung of the same chain,
|
|
124
|
-
|
|
125
|
-
|
|
123
|
+
route falls to the next rung of the same chain, where MiniMax then flint
|
|
124
|
+
carried the lane - flint is the model that replaced MiniMax on the tier, and
|
|
125
|
+
the only one left with the context the lane exists for. The `measured`
|
|
126
|
+
column for that lane is therefore flint's, not either withdrawn model's.
|
|
126
127
|
|
|
127
128
|
The `measured` column is the median of five one-line requests to that model
|
|
128
129
|
on 2026-10-21 - the same prompt for every model, from the process start, so
|
|
@@ -243,7 +244,7 @@ it. One file covers ten jobs either way.
|
|
|
243
244
|
|---|---|---|---|---|
|
|
244
245
|
| `fleet-safe` (default) | 10 | 200,000 | `max` | zero-retention |
|
|
245
246
|
| `fleet-all` | 10 | 160,000 | `max` | **may train** |
|
|
246
|
-
| `
|
|
247
|
+
| `flint-40b-free` | 5 | 262,144 | `max` | **may train** |
|
|
247
248
|
| `beam-free` | 5 | 1,048,576 | `max` | retained |
|
|
248
249
|
| `deepseek-v4.1-flash-free` | 5 | 200,000 | `max` | zero-retention |
|
|
249
250
|
| `glm-5.3-flash-free` | 5 | 200,000 | `max` | zero-retention |
|
|
@@ -252,6 +253,7 @@ it. One file covers ten jobs either way.
|
|
|
252
253
|
| `nemotron-3.5-lightning-free` | 5 | 262,144 | — | **may train** |
|
|
253
254
|
| `mimo-v2.6-flash-free` | 5 | 200,000 | — | **may train** |
|
|
254
255
|
| `big-pickle` | 5 | 160,000 | — | **may train** |
|
|
256
|
+
| `flint-40b-free` | 5 | 262,144 | `max` | **may train** |
|
|
255
257
|
|
|
256
258
|
The five-lane profiles run everything on one model, which is the right shape
|
|
257
259
|
when you want one route and want to know exactly which one.
|
|
@@ -282,6 +284,8 @@ withdrawn, and one of them was the **default** at the time:
|
|
|
282
284
|
| `deepseek-v4-flash-free` | Withdrawn within a month. Replaced by `deepseek-v4.1-flash-free`. |
|
|
283
285
|
| `muse-spark-1.3-contributor-free` | A Meta contributor route whose provider was permitted to train on prompts. The paid route is still in the catalogue. |
|
|
284
286
|
| `mimo-v2.5-free` | Observed alive for one day. Replaced by `mimo-v2.6-flash-free`. |
|
|
287
|
+
| `longcat-2.5-preview-free` | Was the `longform` lane's primary in both fleets, so when it was pulled every lane fell back a rung, and the one profile built only for that model was retired. |
|
|
288
|
+
| `minimax-m3.1-flash-free` | Deprecated by the provider and replaced on the tier by `flint-40b-free`, which is the build and `longform` lane today. |
|
|
285
289
|
|
|
286
290
|
The full list, with a reason for each, is in
|
|
287
291
|
[`profiles/index.json`](profiles/index.json).
|
package/bin/oc-harness.mjs
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// The point of the kit: switching models is a data edit, not a code change.
|
|
13
13
|
// Everything model-specific lives in profiles/<id>.json.
|
|
14
14
|
|
|
15
|
-
import { chmodSync, copyFileSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, symlinkSync, readdirSync, readlinkSync, realpathSync, lstatSync, appendFileSync } from "node:fs"
|
|
15
|
+
import { chmodSync, copyFileSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, symlinkSync, readdirSync, readlinkSync, realpathSync, lstatSync, appendFileSync, renameSync, unlinkSync } from "node:fs"
|
|
16
16
|
import { homedir } from "node:os"
|
|
17
17
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"
|
|
18
18
|
import { execFileSync } from "node:child_process"
|
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
} from "../src/profile.js"
|
|
46
46
|
import { generateConfig, globalInstructionsPath, utilityModel, withMemoryPlugin, canAddMemoryPlugin, kitVersion, MEMORY_PLUGIN } from "../src/config.js"
|
|
47
47
|
import { ruleNames, BASH_RULES, WRITE_RULES } from "../src/rules.js"
|
|
48
|
+
import { readHealth, unhealthyRoutes, writeHealth } from "../src/health.js"
|
|
48
49
|
import { planEffort } from "../src/effort.js"
|
|
49
50
|
import { findTelemetryFiles, readLaneUsage, averageMs } from "../src/lane-usage.js"
|
|
50
51
|
import { researchModels, isWithdrawn, pruneWithdrawnChains } from "../src/research.js"
|
|
@@ -489,6 +490,46 @@ function projectRoot() {
|
|
|
489
490
|
* one in the plural one, and a checker that only looks at `agents/` misses
|
|
490
491
|
* half of them.
|
|
491
492
|
*/
|
|
493
|
+
/**
|
|
494
|
+
* The health records whose model the catalogue lists again.
|
|
495
|
+
*
|
|
496
|
+
* Issue #9: a route recorded as withdrawn carried the quota's day-long
|
|
497
|
+
* window, so a pulled model was retried once a day, forever. The withdrawn
|
|
498
|
+
* window fixed the expiry half; this is the other half. A record whose
|
|
499
|
+
* model has COME BACK to the catalogue - back on the tier, or no longer
|
|
500
|
+
* flagged as withdrawn in it - describes a route that works again right
|
|
501
|
+
* now, and every hour the record still stands, the profile's config hook
|
|
502
|
+
* benches a live route and sends the lane to a rung down its chain.
|
|
503
|
+
*
|
|
504
|
+
* The clear is conservative in the way the health file itself is: a model
|
|
505
|
+
* that is IN the catalogue and not flagged withdrawn in it is trusted as
|
|
506
|
+
* returned. A model the fetch never saw stays recorded - a network error
|
|
507
|
+
* must not un-bench a route.
|
|
508
|
+
*
|
|
509
|
+
* @param {string} dir the project root the health file lives under
|
|
510
|
+
* @param {any} catalog the live model catalogue, or null
|
|
511
|
+
* @returns {string[]} the route keys that were cleared
|
|
512
|
+
*/
|
|
513
|
+
function clearRecoveredRoutes(dir, catalog) {
|
|
514
|
+
if (!catalog || typeof catalog !== "object") return []
|
|
515
|
+
const routes = unhealthyRoutes(dir)
|
|
516
|
+
const recovered = []
|
|
517
|
+
for (const key of Object.keys(routes)) {
|
|
518
|
+
const slash = key.indexOf("/")
|
|
519
|
+
// A key without its provider half was never written by the router.
|
|
520
|
+
if (slash < 1) continue
|
|
521
|
+
const provider = key.slice(0, slash)
|
|
522
|
+
const model = key.slice(slash + 1)
|
|
523
|
+
const record = catalog?.[provider]?.models?.[model]
|
|
524
|
+
if (record && !isWithdrawn(record)) recovered.push(key)
|
|
525
|
+
}
|
|
526
|
+
if (!recovered.length) return []
|
|
527
|
+
const health = readHealth(dir)
|
|
528
|
+
for (const key of recovered) delete health.unhealthy?.[key]
|
|
529
|
+
writeHealth(dir, health)
|
|
530
|
+
return recovered
|
|
531
|
+
}
|
|
532
|
+
|
|
492
533
|
function agentDirs() {
|
|
493
534
|
const project = projectRoot()
|
|
494
535
|
return [
|
|
@@ -876,6 +917,9 @@ async function cmdAuto() {
|
|
|
876
917
|
try {
|
|
877
918
|
// 1. The catalogue. A model that was withdrawn has to be caught before
|
|
878
919
|
// it is used, and the profile files are the only place that lives.
|
|
920
|
+
// The same fetch feeds the health cleanup and the use step below;
|
|
921
|
+
// offline, it stays null and both of those stand down with it.
|
|
922
|
+
let refreshed = { changed: [], gone: [], same: [], skipped: [], catalog: null }
|
|
879
923
|
if (flag("--offline")) {
|
|
880
924
|
steps.push({ step: "refresh", status: "skipped", detail: "--offline" })
|
|
881
925
|
} else {
|
|
@@ -884,7 +928,7 @@ async function cmdAuto() {
|
|
|
884
928
|
// profile that could not be read used to vanish from the report - and
|
|
885
929
|
// the report said "ok": true while a profile was never checked against
|
|
886
930
|
// the live catalogue.
|
|
887
|
-
|
|
931
|
+
refreshed = await cmdRefreshAll()
|
|
888
932
|
for (const g of refreshed.gone) {
|
|
889
933
|
problems.push(`profile ${g} is no longer in the live catalogue`)
|
|
890
934
|
}
|
|
@@ -901,14 +945,35 @@ async function cmdAuto() {
|
|
|
901
945
|
})
|
|
902
946
|
}
|
|
903
947
|
|
|
948
|
+
// 1.5. The health file. A record whose model the catalogue lists again
|
|
949
|
+
// describes a route that works, and the profile's config hook benches
|
|
950
|
+
// a live route while it stands. This step runs with the same fetch the
|
|
951
|
+
// refresh made, so it costs nothing extra. A `--offline` doctor skips
|
|
952
|
+
// it with the refresh it belongs to: a catalogue that was not read
|
|
953
|
+
// must not be trusted to un-bench anything.
|
|
954
|
+
const recovered = clearRecoveredRoutes(projectRoot(), refreshed.catalog)
|
|
955
|
+
if (recovered.length) {
|
|
956
|
+
for (const key of recovered) console.log(` ${green("+")} ${key} is in the live catalogue again; the spent record was cleared`)
|
|
957
|
+
steps.push({
|
|
958
|
+
step: "health",
|
|
959
|
+
status: "ok",
|
|
960
|
+
detail: `${recovered.length} spent record(s) cleared`,
|
|
961
|
+
})
|
|
962
|
+
}
|
|
963
|
+
|
|
904
964
|
// 2. The behavioural layer: the plugin link, the prompts, the skills, the
|
|
905
965
|
// global rules. Without them the config means nothing.
|
|
906
966
|
await cmdInstall()
|
|
907
967
|
steps.push({ step: "install", status: "ok" })
|
|
908
968
|
|
|
909
|
-
// 3. The config, recompiled from the profile
|
|
910
|
-
|
|
911
|
-
|
|
969
|
+
// 3. The config, recompiled from the profile - against the SAME catalogue
|
|
970
|
+
// fetch, so the step does not pull models.dev a second time.
|
|
971
|
+
const used = await cmdUse(id, refreshed.catalog, { wantJson, report: { problems } })
|
|
972
|
+
if (used === "refused") {
|
|
973
|
+
steps.push({ step: "use", status: "failed", detail: id })
|
|
974
|
+
} else {
|
|
975
|
+
steps.push({ step: "use", status: "ok", detail: id })
|
|
976
|
+
}
|
|
912
977
|
|
|
913
978
|
// 4. The check that says whether all of that worked.
|
|
914
979
|
const verifyProblems = cmdVerify()
|
|
@@ -1452,7 +1517,28 @@ function memorySpec() {
|
|
|
1452
1517
|
return MEMORY_PLUGIN
|
|
1453
1518
|
}
|
|
1454
1519
|
|
|
1455
|
-
|
|
1520
|
+
/**
|
|
1521
|
+
* Install a profile's config.
|
|
1522
|
+
*
|
|
1523
|
+
* @param {string} id
|
|
1524
|
+
* @param {any} [catalog] a catalogue an earlier step already fetched
|
|
1525
|
+
* @param {{ wantJson?: boolean, report?: { problems: string[] } | null }} [sink]
|
|
1526
|
+
* where a refusal goes instead of `fail()`. The one caller that runs with
|
|
1527
|
+
* an calmed stdout is `auto --json`: a script reads its report, the report
|
|
1528
|
+
* names the refusal as a problem, and the run still gets to print the
|
|
1529
|
+
* report. Channels an exit here and the report is never printed, which is
|
|
1530
|
+
* exactly the bug the JSON report exists to prevent.
|
|
1531
|
+
*/
|
|
1532
|
+
async function cmdUse(id, catalog = null, sink = {}) {
|
|
1533
|
+
const wantJson = sink.wantJson === true
|
|
1534
|
+
/** Refuse: issue the message the way the caller's stdout wants it said. */
|
|
1535
|
+
const refuse = (msg) => {
|
|
1536
|
+
if (sink.report) {
|
|
1537
|
+
sink.report.problems.push(msg)
|
|
1538
|
+
return false
|
|
1539
|
+
}
|
|
1540
|
+
fail(msg, wantJson)
|
|
1541
|
+
}
|
|
1456
1542
|
if (!id) fail("usage: oc-harness use <profile> [--print] [--yes]")
|
|
1457
1543
|
let p = null
|
|
1458
1544
|
try {
|
|
@@ -1507,6 +1593,76 @@ async function cmdUse(id) {
|
|
|
1507
1593
|
console.error(msg + " Continuing: there is no terminal to ask.\n")
|
|
1508
1594
|
}
|
|
1509
1595
|
|
|
1596
|
+
// The live catalogue, checked at install time because the profile that gets
|
|
1597
|
+
// installed is a promise about what will serve tomorrow.
|
|
1598
|
+
//
|
|
1599
|
+
// Issue #11: `use` used to install a profile without reading the catalogue
|
|
1600
|
+
// at all, so the first run of the harness was the first thing that knew a
|
|
1601
|
+
// lane's model was gone. The profile's file was written, the header said
|
|
1602
|
+
// every lane was healthy, and the harness said so by failing.
|
|
1603
|
+
//
|
|
1604
|
+
// - A dead BASE model ends the install: the profile has nothing to fall
|
|
1605
|
+
// back to, and writing a config that fails on the session's first token
|
|
1606
|
+
// is the bug, not the fix.
|
|
1607
|
+
// - Dead rungs are pruned IN MEMORY before the config is compiled, so the
|
|
1608
|
+
// written config runs on what the catalogue still serves. The profile
|
|
1609
|
+
// file on disk keeps the rung until `oc-harness refresh` makes the fix
|
|
1610
|
+
// durable - an install must not rewrite the package's own data.
|
|
1611
|
+
// - A lane with no rung left fails the install: a lane with no model is
|
|
1612
|
+
// not a lane, and the profile's author has to name the replacement.
|
|
1613
|
+
//
|
|
1614
|
+
// Fail-open: a catalogue that cannot be read is a warning, not a refusal.
|
|
1615
|
+
// Refusing installs on a flaky network takes the tool away from the
|
|
1616
|
+
// machine that needs to build offline, and `--offline` says that out loud.
|
|
1617
|
+
if (!catalog && !flag("--offline")) {
|
|
1618
|
+
try {
|
|
1619
|
+
catalog = await fetchCatalog()
|
|
1620
|
+
} catch (err) {
|
|
1621
|
+
console.error(
|
|
1622
|
+
yellow(
|
|
1623
|
+
` could not check the live catalogue (${err instanceof Error ? err.message : String(err)}). Install continues with the profile as it ships.`,
|
|
1624
|
+
),
|
|
1625
|
+
)
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
if (catalog) {
|
|
1629
|
+
const models = catalog?.[p.provider]?.models ?? {}
|
|
1630
|
+
const record = models[p.model]
|
|
1631
|
+
if (!record || isWithdrawn(record)) {
|
|
1632
|
+
if (!refuse(
|
|
1633
|
+
`profile ${id}'s base model ${p.provider}/${p.model} is no longer in the live catalogue.` +
|
|
1634
|
+
` Pick another profile: oc-harness list <no-args>`,
|
|
1635
|
+
)) return "refused"
|
|
1636
|
+
}
|
|
1637
|
+
// The prune is silent until it has something to say. Its costs are the
|
|
1638
|
+
// output below: the rungs it dropped and the one route that moved, and
|
|
1639
|
+
// the way to make both durable.
|
|
1640
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
1641
|
+
if (prune.emptied.length) {
|
|
1642
|
+
const lanes = prune.emptied.map((lane) => {
|
|
1643
|
+
const rm = prune.lanes.find((l) => l.lane === lane)
|
|
1644
|
+
return `${lane} (its models were: ${rm?.removed.join(", ")})`
|
|
1645
|
+
})
|
|
1646
|
+
if (!refuse(
|
|
1647
|
+
`profile ${id}: every rung of the ${prune.emptied.join(", ")} lane is no longer in the live catalogue.` +
|
|
1648
|
+
` No model can serve that lane, so the profile needs a replacement before it installs.`,
|
|
1649
|
+
)) return "refused"
|
|
1650
|
+
}
|
|
1651
|
+
if (prune.pruned.length || prune.utilityMoved) {
|
|
1652
|
+
const drops = []
|
|
1653
|
+
for (const lane of prune.lanes) drops.push(`lane ${lane.lane}: ${lane.from.join(" -> ")} becomes ${lane.to.join(" -> ")}`)
|
|
1654
|
+
if (prune.fallback) drops.push(`fallback: ${prune.fallback.removed.join(", ")} dropped`)
|
|
1655
|
+
if (prune.utilityMoved) drops.push(`the utility route moved to ${p.utility.model}`)
|
|
1656
|
+
console.error(
|
|
1657
|
+
yellow(
|
|
1658
|
+
` the live catalogue no longer serves ${prune.pruned.join(", ")}: the config this run compiled drops them.` +
|
|
1659
|
+
` Run oc-harness refresh to make the fix durable in the profile file itself.`,
|
|
1660
|
+
) +
|
|
1661
|
+
`\n ${drops.join("\n ")}`,
|
|
1662
|
+
)
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1510
1666
|
const base = readConfigJson()
|
|
1511
1667
|
// `--memory` turns on the memory plugin. The kit does not implement memory
|
|
1512
1668
|
// - a local vector store with embeddings, a web UI and auto-capture is a
|
|
@@ -1539,7 +1695,7 @@ async function cmdUse(id) {
|
|
|
1539
1695
|
|
|
1540
1696
|
mkdirSync(dirname(CONFIG_FILE), { recursive: true })
|
|
1541
1697
|
if (existsSync(CONFIG_FILE)) {
|
|
1542
|
-
// The config is the user's file once written. The backup is unconditional:
|
|
1698
|
+
// The config is the user's file once written. The backup is unconditional:
|
|
1543
1699
|
// `--yes` skips the question, it does not skip the safety net. Gating the
|
|
1544
1700
|
// backup on `--yes` meant the one path that overwrites without asking was
|
|
1545
1701
|
// also the one path with nothing to fall back to.
|
|
@@ -1560,7 +1716,15 @@ async function cmdUse(id) {
|
|
|
1560
1716
|
stamp(` existing config backed up to ${join(BACKUP_DIR, kept)}`),
|
|
1561
1717
|
)
|
|
1562
1718
|
}
|
|
1563
|
-
|
|
1719
|
+
// Write atomically to avoid truncating/corrupting the config on crash.
|
|
1720
|
+
const tmp = `${CONFIG_FILE}.${process.pid}.${Date.now()}.tmp`
|
|
1721
|
+
try {
|
|
1722
|
+
writeFileSync(tmp, out)
|
|
1723
|
+
renameSync(tmp, CONFIG_FILE)
|
|
1724
|
+
} catch (err) {
|
|
1725
|
+
try { unlinkSync(tmp) } catch {}
|
|
1726
|
+
throw err
|
|
1727
|
+
}
|
|
1564
1728
|
console.log(`\n ${green("✓")} wrote ${CONFIG_FILE}`)
|
|
1565
1729
|
summary(p)
|
|
1566
1730
|
console.log(`\n ${stamp("verify with: opencode debug config")}\n`)
|
|
@@ -1825,7 +1989,10 @@ async function cmdRefreshAll() {
|
|
|
1825
1989
|
// dropped, so a caller that silences the human output - `auto --json` -
|
|
1826
1990
|
// never saw them: a withdrawn model or a profile that was never checked
|
|
1827
1991
|
// against the catalogue went into a report that said everything was fine.
|
|
1828
|
-
|
|
1992
|
+
// The catalogue itself comes back with the report: `doctor` runs the
|
|
1993
|
+
// refresh as its opening step, and the steps that follow it check against
|
|
1994
|
+
// the same fetch instead of pulling models.dev twice.
|
|
1995
|
+
return { changed, gone, same, skipped, catalog }
|
|
1829
1996
|
}
|
|
1830
1997
|
|
|
1831
1998
|
/**
|
package/dist/health.d.ts
CHANGED
|
@@ -7,6 +7,13 @@ export type SpentEntry = {
|
|
|
7
7
|
* the pattern that matched
|
|
8
8
|
*/
|
|
9
9
|
reason: string;
|
|
10
|
+
/**
|
|
11
|
+
* why the route was written off.
|
|
12
|
+
* `withdrawn` entries expire on the catalogue's clock, not the quota's:
|
|
13
|
+
* a rate limit resets at 00:00 UTC, a pulled model does not. Entries
|
|
14
|
+
* written before the field existed behave as `spent`.
|
|
15
|
+
*/
|
|
16
|
+
class?: ("spent" | "withdrawn");
|
|
10
17
|
};
|
|
11
18
|
export type Health = {
|
|
12
19
|
unhealthy: Record<string, SpentEntry>;
|
|
@@ -21,6 +28,14 @@ export type Health = {
|
|
|
21
28
|
* @returns {number}
|
|
22
29
|
*/
|
|
23
30
|
export declare function spentWindowMs(): number;
|
|
31
|
+
/**
|
|
32
|
+
* The window, in milliseconds, a route recorded as withdrawn is honoured for.
|
|
33
|
+
*
|
|
34
|
+
* Read at call time, like `spentWindowMs`.
|
|
35
|
+
*
|
|
36
|
+
* @returns {number}
|
|
37
|
+
*/
|
|
38
|
+
export declare function withdrawnWindowMs(): number;
|
|
24
39
|
/**
|
|
25
40
|
* Whether a recorded route is still spent.
|
|
26
41
|
*
|
package/dist/index.js
CHANGED
|
@@ -1932,9 +1932,10 @@ function utilityModel(p) {
|
|
|
1932
1932
|
}
|
|
1933
1933
|
|
|
1934
1934
|
// src/health.js
|
|
1935
|
-
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync } from "node:fs";
|
|
1935
|
+
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync3, renameSync as renameSync2, unlinkSync, writeFileSync } from "node:fs";
|
|
1936
1936
|
import { join as join7 } from "node:path";
|
|
1937
1937
|
var DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1e3;
|
|
1938
|
+
var DEFAULT_WITHDRAWN_WINDOW_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
1938
1939
|
function spentWindowMs() {
|
|
1939
1940
|
const raw = process.env.HARNESS_SPENT_WINDOW_HOURS;
|
|
1940
1941
|
if (!raw) return DEFAULT_WINDOW_MS;
|
|
@@ -1942,6 +1943,13 @@ function spentWindowMs() {
|
|
|
1942
1943
|
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WINDOW_MS;
|
|
1943
1944
|
return hours * 60 * 60 * 1e3;
|
|
1944
1945
|
}
|
|
1946
|
+
function withdrawnWindowMs() {
|
|
1947
|
+
const raw = process.env.HARNESS_WITHDRAWN_WINDOW_HOURS;
|
|
1948
|
+
if (!raw) return DEFAULT_WITHDRAWN_WINDOW_MS;
|
|
1949
|
+
const hours = Number(raw);
|
|
1950
|
+
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WITHDRAWN_WINDOW_MS;
|
|
1951
|
+
return hours * 60 * 60 * 1e3;
|
|
1952
|
+
}
|
|
1945
1953
|
function isSpent(entry, now = Date.now()) {
|
|
1946
1954
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
1947
1955
|
const at = (
|
|
@@ -1951,7 +1959,11 @@ function isSpent(entry, now = Date.now()) {
|
|
|
1951
1959
|
if (typeof at !== "string") return true;
|
|
1952
1960
|
const when = Date.parse(at);
|
|
1953
1961
|
if (Number.isNaN(when)) return true;
|
|
1954
|
-
|
|
1962
|
+
const kind = (
|
|
1963
|
+
/** @type {any} */
|
|
1964
|
+
entry.class
|
|
1965
|
+
);
|
|
1966
|
+
return now - when <= (kind === "withdrawn" ? withdrawnWindowMs() : spentWindowMs());
|
|
1955
1967
|
}
|
|
1956
1968
|
function liveUnhealthy(health, now = Date.now()) {
|
|
1957
1969
|
const out = {};
|
|
@@ -1983,9 +1995,21 @@ function readHealth(dir) {
|
|
|
1983
1995
|
}
|
|
1984
1996
|
function writeHealth(dir, health) {
|
|
1985
1997
|
try {
|
|
1986
|
-
|
|
1998
|
+
const spaceDir = join7(dir, ".space");
|
|
1999
|
+
if (!existsSync5(spaceDir)) mkdirSync2(spaceDir, { recursive: true });
|
|
1987
2000
|
const pruned = { ...health ?? {}, unhealthy: liveUnhealthy(health) };
|
|
1988
|
-
|
|
2001
|
+
const file = healthPath(dir);
|
|
2002
|
+
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
|
2003
|
+
try {
|
|
2004
|
+
writeFileSync(tmp, JSON.stringify(pruned, null, 2));
|
|
2005
|
+
renameSync2(tmp, file);
|
|
2006
|
+
} catch (err) {
|
|
2007
|
+
try {
|
|
2008
|
+
unlinkSync(tmp);
|
|
2009
|
+
} catch {
|
|
2010
|
+
}
|
|
2011
|
+
throw err;
|
|
2012
|
+
}
|
|
1989
2013
|
} catch {
|
|
1990
2014
|
}
|
|
1991
2015
|
}
|
|
@@ -2006,18 +2030,18 @@ var SPENT = [
|
|
|
2006
2030
|
/rate[_ -]?limit/i,
|
|
2007
2031
|
/\b429\b/,
|
|
2008
2032
|
/quota (?:exceeded|exhausted)/i,
|
|
2009
|
-
/overloaded_error/i
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
// "ModelNotFound" - because the text this runs against is the provider's
|
|
2013
|
-
// own error and that can be a whole response body. The old pattern ran
|
|
2014
|
-
// over the entire event, where a body that mentioned a model in one place
|
|
2015
|
-
// and "not found" in another read as a withdrawn model: a live route
|
|
2016
|
-
// written off by a coincidence.
|
|
2033
|
+
/overloaded_error/i
|
|
2034
|
+
];
|
|
2035
|
+
var WITHDRAWN = [
|
|
2017
2036
|
/\bmodel[\W_]*not[\W_]*found\b/i,
|
|
2018
2037
|
/\bmodel\b[\w.:/-]*(?:\s+[\w.:/-]+){0,2}\s+not found\b/i,
|
|
2019
2038
|
/no available provider/i
|
|
2020
2039
|
];
|
|
2040
|
+
var TRANSIENT = [
|
|
2041
|
+
/\b502\b|\b503\b|\b529\b/,
|
|
2042
|
+
/bad gateway/i,
|
|
2043
|
+
/\bupstream\b/
|
|
2044
|
+
];
|
|
2021
2045
|
function errorText(err) {
|
|
2022
2046
|
if (err == null) return "";
|
|
2023
2047
|
if (typeof err === "string") return err;
|
|
@@ -2047,7 +2071,12 @@ function errorText(err) {
|
|
|
2047
2071
|
function classifyError(err) {
|
|
2048
2072
|
const text = errorText(err);
|
|
2049
2073
|
if (!text) return null;
|
|
2050
|
-
for (const p of SPENT)
|
|
2074
|
+
for (const p of SPENT)
|
|
2075
|
+
if (p.test(text)) return { kind: "spent", reason: p.source };
|
|
2076
|
+
for (const p of WITHDRAWN)
|
|
2077
|
+
if (p.test(text)) return { kind: "withdrawn", reason: p.source };
|
|
2078
|
+
for (const p of TRANSIENT)
|
|
2079
|
+
if (p.test(text)) return { kind: "transient", reason: p.source };
|
|
2051
2080
|
return null;
|
|
2052
2081
|
}
|
|
2053
2082
|
function errorOf(e) {
|
|
@@ -2163,14 +2192,21 @@ var RouterPlugin = async ({ worktree, directory }) => {
|
|
|
2163
2192
|
}
|
|
2164
2193
|
const err = errorOf(e);
|
|
2165
2194
|
if (!err) return;
|
|
2166
|
-
const
|
|
2167
|
-
if (!
|
|
2195
|
+
const verdict = classifyError(err);
|
|
2196
|
+
if (!verdict) return;
|
|
2168
2197
|
const sessionID = sessionIdOf(e);
|
|
2169
2198
|
const key = seen ?? (sessionID ? sessionRoutes.get(sessionID) : void 0) ?? configuredRoute ?? lastRoute;
|
|
2170
2199
|
if (!key) return;
|
|
2200
|
+
if (verdict.kind === "transient") return;
|
|
2171
2201
|
const health = readHealth(project);
|
|
2172
2202
|
if (unhealthyRoutes(project)[key]) return;
|
|
2173
|
-
health.unhealthy[key] = {
|
|
2203
|
+
health.unhealthy[key] = {
|
|
2204
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2205
|
+
reason: verdict.reason,
|
|
2206
|
+
// The class decides the window the entry expires under. A model the
|
|
2207
|
+
// provider pulled is not back at 00:00 UTC, whichever the quota says.
|
|
2208
|
+
class: verdict.kind
|
|
2209
|
+
};
|
|
2174
2210
|
writeHealth(project, health);
|
|
2175
2211
|
failedSessions++;
|
|
2176
2212
|
},
|
package/dist/router.d.ts
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
1
|
import type { Plugin } from "./plugin.js";
|
|
2
|
+
/**
|
|
3
|
+
* Which of the three classes a provider error belongs to.
|
|
4
|
+
*
|
|
5
|
+
* The classes decide what happens to the health file: `spent` is recorded
|
|
6
|
+
* under the quota's window, `withdrawn` under the catalogue's, and
|
|
7
|
+
* `transient` is not recorded at all. Exported so a test can feed an error
|
|
8
|
+
* straight through the classifier instead of inventing an event to carry
|
|
9
|
+
* it.
|
|
10
|
+
*
|
|
11
|
+
* @param {any} err
|
|
12
|
+
* @returns {{ kind: "spent" | "withdrawn" | "transient", reason: string } | null}
|
|
13
|
+
*/
|
|
14
|
+
export declare function classifyError(err: any): {
|
|
15
|
+
kind: "spent" | "withdrawn" | "transient";
|
|
16
|
+
reason: string;
|
|
17
|
+
} | null;
|
|
2
18
|
export declare const RouterPlugin: Plugin;
|
|
3
19
|
export default RouterPlugin;
|
package/eval/ab.mjs
CHANGED
|
@@ -750,7 +750,7 @@ console.log("")
|
|
|
750
750
|
console.log(` ${"metric (median)".padEnd(16)} ${"kit on".padStart(10)} ${"kit off".padStart(10)}`)
|
|
751
751
|
console.log(` ${"-".repeat(16)} ${"-".repeat(10)} ${"-".repeat(10)}`)
|
|
752
752
|
for (const [label, a, b] of rows) {
|
|
753
|
-
console.log(` ${label.padEnd(16)} ${String(a).padStart(10)} ${String(b).padStart(10)}`)
|
|
753
|
+
console.log(` ${String(label).padEnd(16)} ${String(a).padStart(10)} ${String(b).padStart(10)}`)
|
|
754
754
|
}
|
|
755
755
|
|
|
756
756
|
if (jobs.length > 1) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-harness-kit",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",
|
package/profiles/fleet-all.json
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
"id": "fleet-all",
|
|
4
4
|
"title": "Ten jobs, every free model, one of them may train",
|
|
5
5
|
"provider": "opencode",
|
|
6
|
-
"model": "
|
|
7
|
-
"family":
|
|
6
|
+
"model": "flint-40b-free",
|
|
7
|
+
"family": "flint",
|
|
8
8
|
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"maxRetention": "may-train",
|
|
11
11
|
"notes": "Every lane gets the model built for it: beam plans, Nemotron 3 Ultra reviews, Lightning sweeps, MiMo sees, GLM audits, Super VL reasons with the effort turned off. Five of the ten providers state that collected data may be used to improve the model, or log the session, so the profile as a whole is may-train and the generated config says so in its header. Install it for public or throwaway repositories.",
|
|
12
12
|
"limits": {
|
|
13
|
-
"context":
|
|
14
|
-
"output":
|
|
13
|
+
"context": 262144,
|
|
14
|
+
"output": 65536
|
|
15
15
|
},
|
|
16
16
|
"supports": {
|
|
17
17
|
"reasoning": true,
|
|
@@ -49,9 +49,9 @@
|
|
|
49
49
|
},
|
|
50
50
|
"roleModels": {
|
|
51
51
|
"build": {
|
|
52
|
-
"model": "
|
|
52
|
+
"model": "flint-40b-free",
|
|
53
53
|
"chain": [
|
|
54
|
-
"
|
|
54
|
+
"flint-40b-free",
|
|
55
55
|
"deepseek-v4.1-flash-free"
|
|
56
56
|
]
|
|
57
57
|
},
|
|
@@ -90,7 +90,7 @@
|
|
|
90
90
|
"model": "mimo-v2.6-flash-free",
|
|
91
91
|
"chain": [
|
|
92
92
|
"mimo-v2.6-flash-free",
|
|
93
|
-
"
|
|
93
|
+
"deepseek-v4.1-flash-free"
|
|
94
94
|
]
|
|
95
95
|
},
|
|
96
96
|
"doc": {
|
|
@@ -116,9 +116,9 @@
|
|
|
116
116
|
]
|
|
117
117
|
},
|
|
118
118
|
"longform": {
|
|
119
|
-
"model": "
|
|
119
|
+
"model": "flint-40b-free",
|
|
120
120
|
"chain": [
|
|
121
|
-
"
|
|
121
|
+
"flint-40b-free",
|
|
122
122
|
"beam-free"
|
|
123
123
|
]
|
|
124
124
|
}
|
|
@@ -130,7 +130,7 @@
|
|
|
130
130
|
"keepUserMessages": true
|
|
131
131
|
},
|
|
132
132
|
"fallback": [
|
|
133
|
-
"
|
|
133
|
+
"flint-40b-free",
|
|
134
134
|
"deepseek-v4.1-flash-free"
|
|
135
135
|
]
|
|
136
136
|
}
|
package/profiles/fleet-safe.json
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
"id": "fleet-safe",
|
|
4
4
|
"title": "Ten jobs, zero-retention models only",
|
|
5
5
|
"provider": "opencode",
|
|
6
|
-
"model": "
|
|
7
|
-
"family":
|
|
6
|
+
"model": "deepseek-v4.1-flash-free",
|
|
7
|
+
"family": "deepseek",
|
|
8
8
|
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"maxRetention": "zero-retention",
|
|
11
|
-
"notes": "The default. Every lane resolves to a model whose provider states a zero-retention policy: no training, nothing kept. The cost is that six of the ten jobs run on a second-choice model - the planner loses beam (30-day retention) and the reviewer loses Nemotron 3 Ultra - so this is the floor, not the ceiling. The chains are ordered within the ceiling rather than copied from `fleet-all`: `locate` takes deepseek because it publishes a `low` effort level that longcat does not, and `vision` takes
|
|
11
|
+
"notes": "The default. Every lane resolves to a model whose provider states a zero-retention policy: no training, nothing kept. The cost is that six of the ten jobs run on a second-choice model - the planner loses beam (30-day retention) and the reviewer loses Nemotron 3 Ultra - so this is the floor, not the ceiling. The chains are ordered within the ceiling rather than copied from `fleet-all`: `locate` takes deepseek because it publishes a `low` effort level that longcat does not, and `vision` takes deepseek because it is the zero-retention route the provider kept for image reads. `fleet-all` gives every lane the model built for it.",
|
|
12
12
|
"limits": {
|
|
13
|
-
"context":
|
|
14
|
-
"output":
|
|
13
|
+
"context": 200000,
|
|
14
|
+
"output": 32000
|
|
15
15
|
},
|
|
16
16
|
"supports": {
|
|
17
17
|
"reasoning": true,
|
|
@@ -49,10 +49,10 @@
|
|
|
49
49
|
},
|
|
50
50
|
"roleModels": {
|
|
51
51
|
"build": {
|
|
52
|
-
"model": "
|
|
52
|
+
"model": "deepseek-v4.1-flash-free",
|
|
53
53
|
"chain": [
|
|
54
|
-
"
|
|
55
|
-
"
|
|
54
|
+
"deepseek-v4.1-flash-free",
|
|
55
|
+
"glm-5.3-flash-free"
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"plan": {
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
"model": "deepseek-v4.1-flash-free",
|
|
74
74
|
"chain": [
|
|
75
75
|
"deepseek-v4.1-flash-free",
|
|
76
|
-
"
|
|
76
|
+
"glm-5.3-flash-free"
|
|
77
77
|
]
|
|
78
78
|
},
|
|
79
79
|
"verify": {
|
|
@@ -84,10 +84,10 @@
|
|
|
84
84
|
]
|
|
85
85
|
},
|
|
86
86
|
"vision": {
|
|
87
|
-
"model": "
|
|
87
|
+
"model": "deepseek-v4.1-flash-free",
|
|
88
88
|
"chain": [
|
|
89
|
-
"
|
|
90
|
-
"
|
|
89
|
+
"deepseek-v4.1-flash-free",
|
|
90
|
+
"glm-5.3-flash-free"
|
|
91
91
|
]
|
|
92
92
|
},
|
|
93
93
|
"doc": {
|
|
@@ -108,14 +108,14 @@
|
|
|
108
108
|
"model": "deepseek-v4.1-flash-free",
|
|
109
109
|
"chain": [
|
|
110
110
|
"deepseek-v4.1-flash-free",
|
|
111
|
-
"
|
|
111
|
+
"glm-5.3-flash-free"
|
|
112
112
|
]
|
|
113
113
|
},
|
|
114
114
|
"longform": {
|
|
115
|
-
"model": "
|
|
115
|
+
"model": "deepseek-v4.1-flash-free",
|
|
116
116
|
"chain": [
|
|
117
|
-
"
|
|
118
|
-
"
|
|
117
|
+
"deepseek-v4.1-flash-free",
|
|
118
|
+
"glm-5.3-flash-free"
|
|
119
119
|
]
|
|
120
120
|
}
|
|
121
121
|
},
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "Metadata read from models.dev on 2026-10-23 and confirmed by a live call the same day. No latency figure is claimed: one call that answered is not a measurement. Regenerate the capability fields with `oc-harness refresh flint-40b-free`.",
|
|
3
|
+
"id": "flint-40b-free",
|
|
4
|
+
"title": "Flint 40B Free",
|
|
5
|
+
"provider": "opencode",
|
|
6
|
+
"model": "flint-40b-free",
|
|
7
|
+
"family": "flint",
|
|
8
|
+
"verified": "2026-10-23",
|
|
9
|
+
"retention": "may-train",
|
|
10
|
+
"notes": "An open-weight mixture of experts, 40B total with about 2.5B active, built for coding agents. Small enough to serve locally, and the weights are published, so the same model can be pointed at a local runtime instead of the free tier. 262k context and a 64k output cap, the full effort ladder, tool calls and structured output; it does not read images. Zen keeps prompts for 30 days and may train on them during the free period.",
|
|
11
|
+
"limits": {
|
|
12
|
+
"context": 262144,
|
|
13
|
+
"output": 65536
|
|
14
|
+
},
|
|
15
|
+
"supports": {
|
|
16
|
+
"reasoning": true,
|
|
17
|
+
"temperature": true,
|
|
18
|
+
"toolCall": true,
|
|
19
|
+
"structuredOutput": true,
|
|
20
|
+
"attachment": false
|
|
21
|
+
},
|
|
22
|
+
"effort": {
|
|
23
|
+
"levels": [
|
|
24
|
+
"low",
|
|
25
|
+
"medium",
|
|
26
|
+
"high",
|
|
27
|
+
"xhigh",
|
|
28
|
+
"max"
|
|
29
|
+
],
|
|
30
|
+
"default": "max",
|
|
31
|
+
"reserveShare": 0.12
|
|
32
|
+
},
|
|
33
|
+
"roles": {
|
|
34
|
+
"plan": "max",
|
|
35
|
+
"build": "max",
|
|
36
|
+
"reviewer": "max",
|
|
37
|
+
"locate": "low",
|
|
38
|
+
"verify": "low"
|
|
39
|
+
},
|
|
40
|
+
"prune": {
|
|
41
|
+
"enabled": true,
|
|
42
|
+
"keepRecentToolCalls": 6,
|
|
43
|
+
"keepRecentTurns": 3,
|
|
44
|
+
"keepUserMessages": true
|
|
45
|
+
},
|
|
46
|
+
"fallback": [
|
|
47
|
+
"deepseek-v4.1-flash-free",
|
|
48
|
+
"nemotron-3.5-lightning-free"
|
|
49
|
+
]
|
|
50
|
+
}
|
package/profiles/index.json
CHANGED
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
"$comment": "Profiles verified against live provider metadata. `retention` must come from retention.yml; a model missing there cannot enter this list. The default is a fleet, because a fleet is the point: ten jobs, ten models, one session. `context` is the input cap the config can actually use: the narrowest lane model's on a fleet, the model's own on a single-model profile. `oc-harness list` recomputes it from the profile rather than trusting this copy.",
|
|
3
3
|
"default": "fleet-safe",
|
|
4
4
|
"profiles": [
|
|
5
|
+
{
|
|
6
|
+
"id": "beam-free",
|
|
7
|
+
"file": "profiles/beam-free.json",
|
|
8
|
+
"title": "Beam Free",
|
|
9
|
+
"retention": "retained",
|
|
10
|
+
"free": true,
|
|
11
|
+
"topEffort": "max",
|
|
12
|
+
"context": 1048576,
|
|
13
|
+
"default": false,
|
|
14
|
+
"note": "Text only, freshest knowledge on the tier, fastest measured route. Keeps prompts for 30 days, so it is not zero-retention."
|
|
15
|
+
},
|
|
5
16
|
{
|
|
6
17
|
"id": "fleet-safe",
|
|
7
18
|
"file": "profiles/fleet-safe.json",
|
|
@@ -24,28 +35,6 @@
|
|
|
24
35
|
"default": false,
|
|
25
36
|
"note": "Every lane gets the model built for it. Five of the ten providers may use the data to improve the model, so the whole profile is may-train."
|
|
26
37
|
},
|
|
27
|
-
{
|
|
28
|
-
"id": "minimax-m3.1-flash-free",
|
|
29
|
-
"file": "profiles/minimax-m3.1-flash-free.json",
|
|
30
|
-
"title": "MiniMax M3.1 Flash Free",
|
|
31
|
-
"retention": "zero-retention",
|
|
32
|
-
"free": true,
|
|
33
|
-
"topEffort": "max",
|
|
34
|
-
"context": 1000000,
|
|
35
|
-
"default": false,
|
|
36
|
-
"note": "512k output tokens, the widest of any free route, and the fastest measured image reader at 7.7s."
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"id": "beam-free",
|
|
40
|
-
"file": "profiles/beam-free.json",
|
|
41
|
-
"title": "Beam Free",
|
|
42
|
-
"retention": "retained",
|
|
43
|
-
"free": true,
|
|
44
|
-
"topEffort": "max",
|
|
45
|
-
"context": 1048576,
|
|
46
|
-
"default": false,
|
|
47
|
-
"note": "Text only, freshest knowledge on the tier, fastest measured route. Keeps prompts for 30 days, so it is not zero-retention."
|
|
48
|
-
},
|
|
49
38
|
{
|
|
50
39
|
"id": "deepseek-v4.1-flash-free",
|
|
51
40
|
"file": "profiles/deepseek-v4.1-flash-free.json",
|
|
@@ -112,6 +101,17 @@
|
|
|
112
101
|
"default": false,
|
|
113
102
|
"note": "Reads images correctly, at 19.2s - the slowest lane on the tier. Publishes audio input too, but OpenCode will not read a binary file, so that path is unreachable today."
|
|
114
103
|
},
|
|
104
|
+
{
|
|
105
|
+
"id": "flint-40b-free",
|
|
106
|
+
"file": "profiles/flint-40b-free.json",
|
|
107
|
+
"title": "Flint 40B Free",
|
|
108
|
+
"retention": "may-train",
|
|
109
|
+
"free": true,
|
|
110
|
+
"topEffort": "max",
|
|
111
|
+
"context": 262144,
|
|
112
|
+
"default": false,
|
|
113
|
+
"note": "Open-weight MoE with about 2.5B active parameters for local coding agents. Knowledge cutoff 2026-06, no image input. May train on prompts during its free period."
|
|
114
|
+
},
|
|
115
115
|
{
|
|
116
116
|
"id": "big-pickle",
|
|
117
117
|
"file": "profiles/big-pickle.json",
|
|
@@ -122,6 +122,17 @@
|
|
|
122
122
|
"context": 160000,
|
|
123
123
|
"default": false,
|
|
124
124
|
"note": "Stealth model, no public identity, no effort levels, no image input. The provider may train on it during its free period."
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
"id": "flint-40b-free",
|
|
128
|
+
"file": "profiles/flint-40b-free.json",
|
|
129
|
+
"title": "Flint 40B Free",
|
|
130
|
+
"retention": "may-train",
|
|
131
|
+
"free": true,
|
|
132
|
+
"topEffort": "max",
|
|
133
|
+
"context": 262144,
|
|
134
|
+
"default": false,
|
|
135
|
+
"note": "Open-weight MoE, 40B total with about 2.5B active, built for coding agents and small enough to serve locally. Full effort ladder, 262k context, 64k output, no image input."
|
|
125
136
|
}
|
|
126
137
|
],
|
|
127
138
|
"excluded": [],
|
|
@@ -149,6 +160,11 @@
|
|
|
149
160
|
{
|
|
150
161
|
"id": "longcat-2.5-preview-free",
|
|
151
162
|
"reason": "Withdrawn from the provider. A preview endpoint, so it could be pulled without a rename - and it was: models.dev still carries the record and marks it deprecated. It was the longform lane's primary in both fleets, so every lane fell back to its second rung, and the profile named only this model had nothing left to fall back to and was retired."
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "minimax-m3.1-flash-free",
|
|
166
|
+
"reason": "Deprecated by provider. Replaced on the tier by flint-40b-free for the lanes that ran it."
|
|
152
167
|
}
|
|
153
|
-
]
|
|
168
|
+
],
|
|
169
|
+
"active": []
|
|
154
170
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "mimo-v2.6-flash-free",
|
|
7
7
|
"family": "mimo",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-23",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"notes": "Xiaomi. The only free model that publishes audio input, and it read an image correctly in a live call, at 19.2s average - the slowest lane on the tier, and slower than minimax on the same picture. OpenCode 1.19 refuses to read a binary file at all, so the audio path is not reachable through the harness today; the vision path is. No effort levels, so no variant is written.",
|
|
11
11
|
"limits": {
|
|
@@ -37,7 +37,5 @@
|
|
|
37
37
|
"keepRecentTurns": 3,
|
|
38
38
|
"keepUserMessages": true
|
|
39
39
|
},
|
|
40
|
-
"fallback": [
|
|
41
|
-
"minimax-m3.1-flash-free"
|
|
42
|
-
]
|
|
40
|
+
"fallback": []
|
|
43
41
|
}
|
package/retention.yml
CHANGED
|
@@ -27,14 +27,6 @@ models:
|
|
|
27
27
|
# --- Zero-retention: the provider states a zero-retention policy and does
|
|
28
28
|
# not train on your data. ---
|
|
29
29
|
|
|
30
|
-
opencode/minimax-m3.1-flash-free:
|
|
31
|
-
retention: zero-retention
|
|
32
|
-
note: >-
|
|
33
|
-
"Its provider follows a zero-retention policy and does not use your data
|
|
34
|
-
for model training." Free for a limited time.
|
|
35
|
-
verified: 2026-10-19
|
|
36
|
-
source: https://opencode.ai/docs/zen
|
|
37
|
-
|
|
38
30
|
opencode/glm-5.3-flash-free:
|
|
39
31
|
retention: zero-retention
|
|
40
32
|
note: >-
|
|
@@ -74,6 +66,15 @@ models:
|
|
|
74
66
|
verified: 2026-10-19
|
|
75
67
|
source: https://opencode.ai/docs/zen
|
|
76
68
|
|
|
69
|
+
opencode/flint-40b-free:
|
|
70
|
+
retention: may-train
|
|
71
|
+
note: >-
|
|
72
|
+
"During its free period, collected data may be used to improve the
|
|
73
|
+
model. Data is retained for 30 days." The weights are open, so the same
|
|
74
|
+
model can be served locally instead of through the free tier.
|
|
75
|
+
verified: 2026-10-23
|
|
76
|
+
source: https://opencode.ai/docs/zen
|
|
77
|
+
|
|
77
78
|
opencode/mimo-v2.6-flash-free:
|
|
78
79
|
retention: may-train
|
|
79
80
|
note: >-
|
|
@@ -109,6 +110,14 @@ models:
|
|
|
109
110
|
verified: 2026-10-19
|
|
110
111
|
source: https://opencode.ai/docs/zen
|
|
111
112
|
|
|
113
|
+
opencode/flint-40b-free:
|
|
114
|
+
retention: may-train
|
|
115
|
+
note: >-
|
|
116
|
+
Free trial endpoint on opencode zen; treated as may-train until terms
|
|
117
|
+
state otherwise.
|
|
118
|
+
verified: 2026-10-22
|
|
119
|
+
source: https://opencode.ai/docs/zen
|
|
120
|
+
|
|
112
121
|
# --- Paid routes, kept because the catalog still serves them and a profile
|
|
113
122
|
# may be pointed at one. ---
|
|
114
123
|
|
package/src/health.js
CHANGED
|
@@ -12,13 +12,17 @@
|
|
|
12
12
|
// Plain JS with JSDoc types, like the rest of src/: the CLI and the tests run
|
|
13
13
|
// under plain node, and the plugin loads .ts.
|
|
14
14
|
|
|
15
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
15
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"
|
|
16
16
|
import { join } from "node:path"
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* @typedef {object} SpentEntry
|
|
20
20
|
* @property {string} at when the route was recorded as spent
|
|
21
21
|
* @property {string} reason the pattern that matched
|
|
22
|
+
* @property {("spent"|"withdrawn")} [class] why the route was written off.
|
|
23
|
+
* `withdrawn` entries expire on the catalogue's clock, not the quota's:
|
|
24
|
+
* a rate limit resets at 00:00 UTC, a pulled model does not. Entries
|
|
25
|
+
* written before the field existed behave as `spent`.
|
|
22
26
|
*/
|
|
23
27
|
|
|
24
28
|
/**
|
|
@@ -42,6 +46,20 @@ import { join } from "node:path"
|
|
|
42
46
|
*/
|
|
43
47
|
const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000
|
|
44
48
|
|
|
49
|
+
/**
|
|
50
|
+
* How long a route recorded as WITHDRAWN is honoured for, by default.
|
|
51
|
+
*
|
|
52
|
+
* A rate limit resets at 00:00 UTC. A withdrawal does not: the provider
|
|
53
|
+
* pulled the model, and no number of days' sleep brings it back. Treating
|
|
54
|
+
* both with the one window meant a model that had left the catalogue was
|
|
55
|
+
* retried once a day, forever, and every lane that named it first paid a
|
|
56
|
+
* failed request before falling through.
|
|
57
|
+
*
|
|
58
|
+
* Thirty days is the scale the toolkit's catalogue turns on - free tiers
|
|
59
|
+
* rotate monthly - and HARNESS_WITHDRAWN_WINDOW_HOURS overrides it.
|
|
60
|
+
*/
|
|
61
|
+
const DEFAULT_WITHDRAWN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000
|
|
62
|
+
|
|
45
63
|
/**
|
|
46
64
|
* The window, in milliseconds, a recorded spent route is honoured for.
|
|
47
65
|
*
|
|
@@ -61,6 +79,21 @@ export function spentWindowMs() {
|
|
|
61
79
|
return hours * 60 * 60 * 1000
|
|
62
80
|
}
|
|
63
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The window, in milliseconds, a route recorded as withdrawn is honoured for.
|
|
84
|
+
*
|
|
85
|
+
* Read at call time, like `spentWindowMs`.
|
|
86
|
+
*
|
|
87
|
+
* @returns {number}
|
|
88
|
+
*/
|
|
89
|
+
export function withdrawnWindowMs() {
|
|
90
|
+
const raw = process.env.HARNESS_WITHDRAWN_WINDOW_HOURS
|
|
91
|
+
if (!raw) return DEFAULT_WITHDRAWN_WINDOW_MS
|
|
92
|
+
const hours = Number(raw)
|
|
93
|
+
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WITHDRAWN_WINDOW_MS
|
|
94
|
+
return hours * 60 * 60 * 1000
|
|
95
|
+
}
|
|
96
|
+
|
|
64
97
|
/**
|
|
65
98
|
* Whether a recorded route is still spent.
|
|
66
99
|
*
|
|
@@ -79,7 +112,12 @@ export function isSpent(entry, now = Date.now()) {
|
|
|
79
112
|
if (typeof at !== "string") return true
|
|
80
113
|
const when = Date.parse(at)
|
|
81
114
|
if (Number.isNaN(when)) return true
|
|
82
|
-
|
|
115
|
+
// The window the entry was written under. An entry that carries no class
|
|
116
|
+
// was written by an older version of the kit, or by a hand, and gets the
|
|
117
|
+
// quota's window: the quota resets, which is the assumption every entry
|
|
118
|
+
// used to carry.
|
|
119
|
+
const kind = /** @type {any} */ (entry).class
|
|
120
|
+
return now - when <= (kind === "withdrawn" ? withdrawnWindowMs() : spentWindowMs())
|
|
83
121
|
}
|
|
84
122
|
|
|
85
123
|
/**
|
|
@@ -158,9 +196,18 @@ export function readHealth(dir) {
|
|
|
158
196
|
*/
|
|
159
197
|
export function writeHealth(dir, health) {
|
|
160
198
|
try {
|
|
161
|
-
|
|
199
|
+
const spaceDir = join(dir, ".space")
|
|
200
|
+
if (!existsSync(spaceDir)) mkdirSync(spaceDir, { recursive: true })
|
|
162
201
|
const pruned = { ...(health ?? {}), unhealthy: liveUnhealthy(health) }
|
|
163
|
-
|
|
202
|
+
const file = healthPath(dir)
|
|
203
|
+
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`
|
|
204
|
+
try {
|
|
205
|
+
writeFileSync(tmp, JSON.stringify(pruned, null, 2))
|
|
206
|
+
renameSync(tmp, file)
|
|
207
|
+
} catch (err) {
|
|
208
|
+
try { unlinkSync(tmp) } catch {}
|
|
209
|
+
throw err
|
|
210
|
+
}
|
|
164
211
|
} catch {
|
|
165
212
|
// Best effort. A health file that cannot be written means no failover,
|
|
166
213
|
// and failing to write it must not break the session.
|
package/src/router.ts
CHANGED
|
@@ -36,25 +36,53 @@ import {
|
|
|
36
36
|
resolveRoles,
|
|
37
37
|
} from "./profile.js"
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Patterns that mean "this route is spent", not "this request was bad": the
|
|
41
|
+
* provider's quota for the free tier ran out. The quota resets, which is why
|
|
42
|
+
* these expire on the day's window.
|
|
43
|
+
*/
|
|
40
44
|
const SPENT = [
|
|
41
45
|
/FreeUsageLimitError/i,
|
|
42
46
|
/rate[_ -]?limit/i,
|
|
43
47
|
/\b429\b/,
|
|
44
48
|
/quota (?:exceeded|exhausted)/i,
|
|
45
49
|
/overloaded_error/i,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Patterns that mean "this model is gone": the provider pulled it.
|
|
54
|
+
*
|
|
55
|
+
* The words have to sit next to each other - "model minimax-m3.1-flash-free
|
|
56
|
+
* not found", "model_not_found", "ModelNotFound" - because the text this
|
|
57
|
+
* runs against is the provider's own error and that can be a whole response
|
|
58
|
+
* body. The old pattern ran over the entire event, where a body that
|
|
59
|
+
* mentioned a model in one place and "not found" in another read as a
|
|
60
|
+
* withdrawn model: a live route written off by a coincidence.
|
|
61
|
+
*
|
|
62
|
+
* These get their own class and their own window: no number of days' sleep
|
|
63
|
+
* returns a pulled model, so benching one for 24h retried a death once a
|
|
64
|
+
* day, forever.
|
|
65
|
+
*/
|
|
66
|
+
const WITHDRAWN = [
|
|
53
67
|
/\bmodel[\W_]*not[\W_]*found\b/i,
|
|
54
68
|
/\bmodel\b[\w.:/-]*(?:\s+[\w.:/-]+){0,2}\s+not found\b/i,
|
|
55
69
|
/no available provider/i,
|
|
56
70
|
]
|
|
57
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Patterns that mean "this request could not be answered" - the host was at
|
|
74
|
+
* capacity, a gateway hiccuped - and nothing about the ROUTE is worse.
|
|
75
|
+
*
|
|
76
|
+
* Benching a route for one of these benches a model that answered minutes
|
|
77
|
+
* later. They are not spent, not withdrawn, and they record nothing: the
|
|
78
|
+
* next request tries the same route again.
|
|
79
|
+
*/
|
|
80
|
+
const TRANSIENT = [
|
|
81
|
+
/\b502\b|\b503\b|\b529\b/,
|
|
82
|
+
/bad gateway/i,
|
|
83
|
+
/\bupstream\b/,
|
|
84
|
+
]
|
|
85
|
+
|
|
58
86
|
/**
|
|
59
87
|
* The text of a provider error, and nothing else.
|
|
60
88
|
*
|
|
@@ -99,15 +127,26 @@ function errorText(err: any): string {
|
|
|
99
127
|
}
|
|
100
128
|
|
|
101
129
|
/**
|
|
102
|
-
*
|
|
130
|
+
* Which of the three classes a provider error belongs to.
|
|
131
|
+
*
|
|
132
|
+
* The classes decide what happens to the health file: `spent` is recorded
|
|
133
|
+
* under the quota's window, `withdrawn` under the catalogue's, and
|
|
134
|
+
* `transient` is not recorded at all. Exported so a test can feed an error
|
|
135
|
+
* straight through the classifier instead of inventing an event to carry
|
|
136
|
+
* it.
|
|
103
137
|
*
|
|
104
138
|
* @param {any} err
|
|
105
|
-
* @returns {
|
|
139
|
+
* @returns {{ kind: "spent" | "withdrawn" | "transient", reason: string } | null}
|
|
106
140
|
*/
|
|
107
|
-
function classifyError(err: any): string | null {
|
|
141
|
+
export function classifyError(err: any): { kind: "spent" | "withdrawn" | "transient"; reason: string } | null {
|
|
108
142
|
const text = errorText(err)
|
|
109
143
|
if (!text) return null
|
|
110
|
-
for (const p of SPENT)
|
|
144
|
+
for (const p of SPENT)
|
|
145
|
+
if (p.test(text)) return { kind: "spent", reason: p.source }
|
|
146
|
+
for (const p of WITHDRAWN)
|
|
147
|
+
if (p.test(text)) return { kind: "withdrawn", reason: p.source }
|
|
148
|
+
for (const p of TRANSIENT)
|
|
149
|
+
if (p.test(text)) return { kind: "transient", reason: p.source }
|
|
111
150
|
return null
|
|
112
151
|
}
|
|
113
152
|
|
|
@@ -352,8 +391,8 @@ export const RouterPlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
352
391
|
// anything about the route.
|
|
353
392
|
const err = errorOf(e)
|
|
354
393
|
if (!err) return
|
|
355
|
-
const
|
|
356
|
-
if (!
|
|
394
|
+
const verdict = classifyError(err)
|
|
395
|
+
if (!verdict) return
|
|
357
396
|
|
|
358
397
|
// The route the failure belongs to: the event's own, else the route
|
|
359
398
|
// this session was last seen on, else the route it was configured
|
|
@@ -367,9 +406,27 @@ export const RouterPlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
367
406
|
lastRoute
|
|
368
407
|
if (!key) return
|
|
369
408
|
|
|
409
|
+
// A transient failure is the host failing to answer one request, not
|
|
410
|
+
// the route becoming unusable. Nothing is written, so the next request
|
|
411
|
+
// tries the same rung again - benching the route under the quota's
|
|
412
|
+
// window is how a capacity blip cost a model a day.
|
|
413
|
+
//
|
|
414
|
+
// ("Advances to the next rung inside the same request" is not on the
|
|
415
|
+
// table: `chat.params` carries only temperature and sampling, the model
|
|
416
|
+
// on `chat.message` is read-only, and `config` has already run. What a
|
|
417
|
+
// transient verdict buys - and what this buys - is a route that is
|
|
418
|
+
// still live when the next request arrives.)
|
|
419
|
+
if (verdict.kind === "transient") return
|
|
420
|
+
|
|
370
421
|
const health = readHealth(project)
|
|
371
422
|
if (unhealthyRoutes(project)[key]) return // already recorded, do not spam
|
|
372
|
-
health.unhealthy[key] = {
|
|
423
|
+
health.unhealthy[key] = {
|
|
424
|
+
at: new Date().toISOString(),
|
|
425
|
+
reason: verdict.reason,
|
|
426
|
+
// The class decides the window the entry expires under. A model the
|
|
427
|
+
// provider pulled is not back at 00:00 UTC, whichever the quota says.
|
|
428
|
+
class: verdict.kind,
|
|
429
|
+
}
|
|
373
430
|
writeHealth(project, health)
|
|
374
431
|
failedSessions++
|
|
375
432
|
},
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$comment": "Metadata verified against models.dev on 2026-10-19 and confirmed by a live call the same day. Regenerate the capability fields with `oc-harness refresh minimax-m3.1-flash-free`.",
|
|
3
|
-
"id": "minimax-m3.1-flash-free",
|
|
4
|
-
"title": "MiniMax M3.1 Flash Free",
|
|
5
|
-
"provider": "opencode",
|
|
6
|
-
"model": "minimax-m3.1-flash-free",
|
|
7
|
-
"family": "minimax",
|
|
8
|
-
"verified": "2026-10-22",
|
|
9
|
-
"retention": "zero-retention",
|
|
10
|
-
"notes": "Flagship coding and agent model on the free tier: 512k output tokens, the widest of any free route, and the fastest measured image reader at 7.7s. Tool calls are stable. It does not support temperature, and models.dev publishes no structured_output flag for it, so JSON output is requested in the prompt rather than negotiated with the API. Measured at 3.1s for a one-word reply.",
|
|
11
|
-
"limits": {
|
|
12
|
-
"context": 1000000,
|
|
13
|
-
"output": 512000
|
|
14
|
-
},
|
|
15
|
-
"supports": {
|
|
16
|
-
"reasoning": true,
|
|
17
|
-
"temperature": true,
|
|
18
|
-
"toolCall": true,
|
|
19
|
-
"structuredOutput": false,
|
|
20
|
-
"attachment": true
|
|
21
|
-
},
|
|
22
|
-
"effort": {
|
|
23
|
-
"levels": [
|
|
24
|
-
"low",
|
|
25
|
-
"medium",
|
|
26
|
-
"high",
|
|
27
|
-
"xhigh",
|
|
28
|
-
"max"
|
|
29
|
-
],
|
|
30
|
-
"default": "max",
|
|
31
|
-
"reserveShare": 0.12
|
|
32
|
-
},
|
|
33
|
-
"roles": {
|
|
34
|
-
"plan": "max",
|
|
35
|
-
"build": "max",
|
|
36
|
-
"reviewer": "max",
|
|
37
|
-
"locate": "low",
|
|
38
|
-
"verify": "low"
|
|
39
|
-
},
|
|
40
|
-
"prune": {
|
|
41
|
-
"enabled": true,
|
|
42
|
-
"keepRecentToolCalls": 6,
|
|
43
|
-
"keepRecentTurns": 3,
|
|
44
|
-
"keepUserMessages": true
|
|
45
|
-
},
|
|
46
|
-
"fallback": [
|
|
47
|
-
"deepseek-v4.1-flash-free",
|
|
48
|
-
"glm-5.3-flash-free"
|
|
49
|
-
]
|
|
50
|
-
}
|