opencode-harness-kit 0.5.8 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -0
- package/README.md +223 -23
- package/bin/oc-harness.mjs +188 -13
- package/dist/index.js +69 -9
- package/dist/profile.d.ts +2 -1
- package/docs/KNOWN-LIMITS.md +22 -2
- package/eval/ab.mjs +1 -1
- package/package.json +1 -1
- package/profiles/beam-free.json +2 -3
- package/profiles/deepseek-v4.1-flash-free.json +1 -2
- package/profiles/fleet-all.json +15 -23
- package/profiles/fleet-safe.json +21 -32
- package/profiles/flint-40b-free.json +50 -0
- package/profiles/glm-5.3-flash-free.json +1 -2
- package/profiles/index.json +43 -34
- package/profiles/mimo-v2.6-flash-free.json +2 -5
- package/profiles/nemotron-3.5-lightning-free.json +2 -3
- package/profiles/nemotron-3.5-super-vl-free.json +1 -2
- package/profiles/schema.json +8 -0
- package/retention.yml +17 -17
- package/src/health.js +12 -3
- package/src/profile.js +14 -1
- package/src/prune.ts +90 -5
- package/src/research.js +462 -0
- package/profiles/longcat-2.5-preview-free.json +0 -45
- package/profiles/minimax-m3.1-flash-free.json +0 -51
package/bin/oc-harness.mjs
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
// The point of the kit: switching models is a data edit, not a code change.
|
|
13
13
|
// Everything model-specific lives in profiles/<id>.json.
|
|
14
14
|
|
|
15
|
-
import { chmodSync, copyFileSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, symlinkSync, readdirSync, readlinkSync, realpathSync, lstatSync, appendFileSync } from "node:fs"
|
|
15
|
+
import { chmodSync, copyFileSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, symlinkSync, readdirSync, readlinkSync, realpathSync, lstatSync, appendFileSync, renameSync, unlinkSync } from "node:fs"
|
|
16
16
|
import { homedir } from "node:os"
|
|
17
17
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"
|
|
18
18
|
import { execFileSync } from "node:child_process"
|
|
@@ -47,6 +47,7 @@ import { generateConfig, globalInstructionsPath, utilityModel, withMemoryPlugin,
|
|
|
47
47
|
import { ruleNames, BASH_RULES, WRITE_RULES } from "../src/rules.js"
|
|
48
48
|
import { planEffort } from "../src/effort.js"
|
|
49
49
|
import { findTelemetryFiles, readLaneUsage, averageMs } from "../src/lane-usage.js"
|
|
50
|
+
import { researchModels, isWithdrawn, pruneWithdrawnChains } from "../src/research.js"
|
|
50
51
|
import { main as trustMain } from "../src/trust.js"
|
|
51
52
|
|
|
52
53
|
const CONFIG_DIR =
|
|
@@ -1538,7 +1539,7 @@ async function cmdUse(id) {
|
|
|
1538
1539
|
|
|
1539
1540
|
mkdirSync(dirname(CONFIG_FILE), { recursive: true })
|
|
1540
1541
|
if (existsSync(CONFIG_FILE)) {
|
|
1541
|
-
// The config is the user's file once written. The backup is unconditional:
|
|
1542
|
+
// The config is the user's file once written. The backup is unconditional:
|
|
1542
1543
|
// `--yes` skips the question, it does not skip the safety net. Gating the
|
|
1543
1544
|
// backup on `--yes` meant the one path that overwrites without asking was
|
|
1544
1545
|
// also the one path with nothing to fall back to.
|
|
@@ -1559,7 +1560,15 @@ async function cmdUse(id) {
|
|
|
1559
1560
|
stamp(` existing config backed up to ${join(BACKUP_DIR, kept)}`),
|
|
1560
1561
|
)
|
|
1561
1562
|
}
|
|
1562
|
-
|
|
1563
|
+
// Write atomically to avoid truncating/corrupting the config on crash.
|
|
1564
|
+
const tmp = `${CONFIG_FILE}.${process.pid}.${Date.now()}.tmp`
|
|
1565
|
+
try {
|
|
1566
|
+
writeFileSync(tmp, out)
|
|
1567
|
+
renameSync(tmp, CONFIG_FILE)
|
|
1568
|
+
} catch (err) {
|
|
1569
|
+
try { unlinkSync(tmp) } catch {}
|
|
1570
|
+
throw err
|
|
1571
|
+
}
|
|
1563
1572
|
console.log(`\n ${green("✓")} wrote ${CONFIG_FILE}`)
|
|
1564
1573
|
summary(p)
|
|
1565
1574
|
console.log(`\n ${stamp("verify with: opencode debug config")}\n`)
|
|
@@ -1702,6 +1711,11 @@ function diffProfile(p, live) {
|
|
|
1702
1711
|
* Only machine-derived fields are rewritten. `retention`, `notes`, `roles` and
|
|
1703
1712
|
* the fallback queue are human judgements and are never touched by a script:
|
|
1704
1713
|
* losing a privacy verdict to an automated rewrite would be worse than drift.
|
|
1714
|
+
*
|
|
1715
|
+
* The ONE exception is a chain rung the provider has withdrawn. A chain names
|
|
1716
|
+
* the models that could serve a lane, and a withdrawn model is not one of them
|
|
1717
|
+
* any more - it cannot run. Removing it is a derivation from live metadata
|
|
1718
|
+
* and it only ever removes; nothing is ever put INTO a lane.
|
|
1705
1719
|
*/
|
|
1706
1720
|
/**
|
|
1707
1721
|
* Refresh every shipped profile against the live catalogue.
|
|
@@ -1742,8 +1756,18 @@ async function cmdRefreshAll() {
|
|
|
1742
1756
|
gone.push(`${id} (${p.provider}/${p.model})`)
|
|
1743
1757
|
continue
|
|
1744
1758
|
}
|
|
1759
|
+
// A model the provider has withdrawn is still IN the catalogue, so the
|
|
1760
|
+
// check above passes it. A profile whose own model was withdrawn has no
|
|
1761
|
+
// chain behind it to fall back to, so there is nothing a refresh can
|
|
1762
|
+
// repair - it has to be retired, and that is a person's call.
|
|
1763
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
1764
|
+
if (prune.dead) {
|
|
1765
|
+
gone.push(`${id} (${p.provider}/${p.model} was withdrawn from the provider)`)
|
|
1766
|
+
continue
|
|
1767
|
+
}
|
|
1745
1768
|
const before = diffProfile(p, live)
|
|
1746
|
-
|
|
1769
|
+
const pruned = prune.pruned.length > 0
|
|
1770
|
+
if (!before.length && !pruned) {
|
|
1747
1771
|
same.push(id)
|
|
1748
1772
|
continue
|
|
1749
1773
|
}
|
|
@@ -1751,8 +1775,18 @@ async function cmdRefreshAll() {
|
|
|
1751
1775
|
// `verified` is the field that says when a profile was last checked
|
|
1752
1776
|
// against live metadata. Leaving it stale after a refresh makes it a lie.
|
|
1753
1777
|
p.verified = new Date().toISOString().slice(0, 10)
|
|
1778
|
+
const notes = [...before]
|
|
1779
|
+
if (pruned) {
|
|
1780
|
+
notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
|
|
1781
|
+
for (const lane of prune.lanes) {
|
|
1782
|
+
notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
|
|
1783
|
+
}
|
|
1784
|
+
if (prune.utilityMoved) {
|
|
1785
|
+
notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1754
1788
|
if (flag("--dryrun")) {
|
|
1755
|
-
changed.push(`${id}: ${
|
|
1789
|
+
changed.push(`${id}: ${notes.join("; ")} (dry run, nothing written)`)
|
|
1756
1790
|
continue
|
|
1757
1791
|
}
|
|
1758
1792
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
@@ -1765,7 +1799,7 @@ async function cmdRefreshAll() {
|
|
|
1765
1799
|
continue
|
|
1766
1800
|
}
|
|
1767
1801
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
1768
|
-
changed.push(`${id}: ${
|
|
1802
|
+
changed.push(`${id}: ${notes.join("; ")}`)
|
|
1769
1803
|
}
|
|
1770
1804
|
console.log(
|
|
1771
1805
|
`\n${bold("refreshed")} ${stamp(`${checked} of ${ids.length} profile(s) against the live catalogue`)}\n`,
|
|
@@ -2303,8 +2337,30 @@ async function cmdRefresh(id) {
|
|
|
2303
2337
|
)
|
|
2304
2338
|
}
|
|
2305
2339
|
|
|
2340
|
+
// A withdrawn model is still in the catalogue, so the check above passes
|
|
2341
|
+
// it. A profile whose own model was withdrawn has no chain behind it, so
|
|
2342
|
+
// there is nothing to fall back to and nothing a refresh can repair.
|
|
2343
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
2344
|
+
if (prune.dead) {
|
|
2345
|
+
fail(
|
|
2346
|
+
`${p.provider}/${p.model} was withdrawn from the provider.\n` +
|
|
2347
|
+
` This profile has no chain to fall back to, so it cannot be repaired.\n` +
|
|
2348
|
+
` Remove it, or point it at a model that is still live.`,
|
|
2349
|
+
)
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2306
2352
|
const before = diffProfile(p, live)
|
|
2307
|
-
|
|
2353
|
+
const notes = [...before]
|
|
2354
|
+
if (prune.pruned.length > 0) {
|
|
2355
|
+
notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
|
|
2356
|
+
for (const lane of prune.lanes) {
|
|
2357
|
+
notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
|
|
2358
|
+
}
|
|
2359
|
+
if (prune.utilityMoved) {
|
|
2360
|
+
notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
if (notes.length === 0) {
|
|
2308
2364
|
console.log(`\n ${green("✓")} ${id} already matches live metadata\n`)
|
|
2309
2365
|
return
|
|
2310
2366
|
}
|
|
@@ -2337,7 +2393,7 @@ async function cmdRefresh(id) {
|
|
|
2337
2393
|
|
|
2338
2394
|
if (flag("--dryrun")) {
|
|
2339
2395
|
console.log(`\n ${stamp("dry run, nothing written")}`)
|
|
2340
|
-
for (const line of
|
|
2396
|
+
for (const line of notes) console.log(` ${line}`)
|
|
2341
2397
|
return
|
|
2342
2398
|
}
|
|
2343
2399
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
@@ -2352,9 +2408,9 @@ async function cmdRefresh(id) {
|
|
|
2352
2408
|
}
|
|
2353
2409
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
2354
2410
|
console.log(`\n ${green("✓")} refreshed ${path}`)
|
|
2355
|
-
for (const line of
|
|
2411
|
+
for (const line of notes) console.log(` ${line}`)
|
|
2356
2412
|
console.log(
|
|
2357
|
-
`\n ${stamp("kept as-is: retention, roles, notes
|
|
2413
|
+
`\n ${stamp("kept as-is: retention, roles, notes (human judgement)")}\n`,
|
|
2358
2414
|
)
|
|
2359
2415
|
}
|
|
2360
2416
|
|
|
@@ -2594,6 +2650,111 @@ async function cmdDoctor() {
|
|
|
2594
2650
|
// second doctor in the same process does not inherit the first one's findings.
|
|
2595
2651
|
const doctorReport = { ok: true, problems: 0, drift: 0, strict: false, findings: [], json: false }
|
|
2596
2652
|
|
|
2653
|
+
/**
|
|
2654
|
+
* Research the live tier for models the kit could use.
|
|
2655
|
+
*
|
|
2656
|
+
* `doctor` says when a model has gone. This says what has come to replace
|
|
2657
|
+
* it, and what is still a person's decision.
|
|
2658
|
+
*
|
|
2659
|
+
* It writes nothing. The shortlist it prints is a list to argue with, not a
|
|
2660
|
+
* list to accept: `retention` is a judgement about a provider's terms, and
|
|
2661
|
+
* that is the one field a script must not fill in.
|
|
2662
|
+
*/
|
|
2663
|
+
async function cmdResearch() {
|
|
2664
|
+
const wantJson = flag("--json")
|
|
2665
|
+
let catalog
|
|
2666
|
+
try {
|
|
2667
|
+
catalog = await fetchCatalog()
|
|
2668
|
+
} catch (err) {
|
|
2669
|
+
fail(err instanceof Error ? err.message : String(err), wantJson)
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
// A profile that cannot be read is `doctor`'s finding, not this command's.
|
|
2673
|
+
// Research is about the tier, so a broken profile is left out of the
|
|
2674
|
+
// comparison rather than failing the whole report.
|
|
2675
|
+
const profiles = []
|
|
2676
|
+
for (const id of listProfileIds()) {
|
|
2677
|
+
try {
|
|
2678
|
+
profiles.push(loadProfile(id))
|
|
2679
|
+
} catch {
|
|
2680
|
+
// doctor reports this, with the profile's own name on it.
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
const report = researchModels({
|
|
2685
|
+
catalog,
|
|
2686
|
+
profiles,
|
|
2687
|
+
retentionOf: (model) => modelRetention(model),
|
|
2688
|
+
defaultProfileId: index().default ?? null,
|
|
2689
|
+
})
|
|
2690
|
+
|
|
2691
|
+
if (wantJson) {
|
|
2692
|
+
console.log(JSON.stringify({ ok: true, ...report }, null, 2))
|
|
2693
|
+
return
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
console.log(
|
|
2697
|
+
`\n${bold("research")} ${stamp(`tier ${report.tier.provider}, ${report.tier.live} live of ${report.tier.total} free`)}\n`,
|
|
2698
|
+
)
|
|
2699
|
+
|
|
2700
|
+
// The models the kit uses that the provider has pulled. This is the part
|
|
2701
|
+
// that has to be acted on, so it comes first.
|
|
2702
|
+
if (report.shipped.gone.length) {
|
|
2703
|
+
console.log(` ${red("gone")} ${report.shipped.gone.length} model(s) in use have been withdrawn\n`)
|
|
2704
|
+
for (const g of report.shipped.gone) {
|
|
2705
|
+
console.log(` ${red("✗")} ${g.id}`)
|
|
2706
|
+
console.log(` ${g.reason}`)
|
|
2707
|
+
if (g.asBase.length) {
|
|
2708
|
+
console.log(` ${stamp("base model of")} ${g.asBase.join(", ")}`)
|
|
2709
|
+
}
|
|
2710
|
+
for (const [profile, laneList] of Object.entries(g.inChains ?? {})) {
|
|
2711
|
+
console.log(` ${stamp("fallback rung in")} ${profile}: ${laneList.join(", ")}`)
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
console.log("")
|
|
2715
|
+
} else {
|
|
2716
|
+
console.log(` ${green("✓")} every model in use is still live\n`)
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
// Per lane: which rungs of its chain are alive. A lane whose whole chain
|
|
2720
|
+
// has gone is a lane with no fallback at all.
|
|
2721
|
+
if (report.lanes.length) {
|
|
2722
|
+
console.log(` ${bold("lanes")}`)
|
|
2723
|
+
for (const l of report.lanes) {
|
|
2724
|
+
const dead = l.dead.length
|
|
2725
|
+
? yellow(` ${l.dead.length} dead (${l.dead.join(", ")})`)
|
|
2726
|
+
: green(" 0 dead")
|
|
2727
|
+
console.log(` ${l.lane.padEnd(12)} ${String(l.live.length).padStart(2)} live${dead}`)
|
|
2728
|
+
}
|
|
2729
|
+
console.log("")
|
|
2730
|
+
}
|
|
2731
|
+
|
|
2732
|
+
// The shortlist. What each candidate can do, which lanes it could serve,
|
|
2733
|
+
// and what is still undecided about it.
|
|
2734
|
+
if (!report.candidates.length) {
|
|
2735
|
+
console.log(` ${stamp("no live free model outside the ones the kit already uses")}\n`)
|
|
2736
|
+
return
|
|
2737
|
+
}
|
|
2738
|
+
console.log(` ${bold("candidates")} ${stamp("live, free, and not in use yet")}\n`)
|
|
2739
|
+
for (const c of report.candidates) {
|
|
2740
|
+
const effort = c.effort.length ? c.effort.join(",") : "none published"
|
|
2741
|
+
console.log(
|
|
2742
|
+
` ${bold(c.id)} ${stamp(
|
|
2743
|
+
`ctx ${Number(c.context).toLocaleString("en-US")} out ${Number(c.output).toLocaleString("en-US")} effort ${effort}`,
|
|
2744
|
+
)}`,
|
|
2745
|
+
)
|
|
2746
|
+
console.log(` fits: ${c.fits.length ? c.fits.join(", ") : "no lane on its own terms"}`)
|
|
2747
|
+
console.log(
|
|
2748
|
+
` ${c.retention === "unknown" ? yellow("retention: unknown") : stamp(`retention: ${c.retention}`)}`,
|
|
2749
|
+
)
|
|
2750
|
+
for (const b of c.blockers) console.log(` ${yellow(b)}`)
|
|
2751
|
+
console.log("")
|
|
2752
|
+
}
|
|
2753
|
+
console.log(
|
|
2754
|
+
` ${stamp("nothing here is written anywhere. A retention verdict is a person reading a provider's terms.")}\n`,
|
|
2755
|
+
)
|
|
2756
|
+
}
|
|
2757
|
+
|
|
2597
2758
|
async function runDoctor(catalog) {
|
|
2598
2759
|
const retention = loadRetention().models
|
|
2599
2760
|
const ids = listProfileIds()
|
|
@@ -2641,7 +2802,16 @@ async function runDoctor(catalog) {
|
|
|
2641
2802
|
// with no model at all, which no amount of `refresh` on the base profile
|
|
2642
2803
|
// would reveal.
|
|
2643
2804
|
for (const [lane, chain] of Object.entries(chainOf(p))) {
|
|
2644
|
-
|
|
2805
|
+
// A withdrawn model is still IN the catalog: models.dev keeps the
|
|
2806
|
+
// record and marks it deprecated rather than deleting it. So "is it
|
|
2807
|
+
// still there" is not the same question as "can it still run", and
|
|
2808
|
+
// asking the first one reported a dead model as a healthy fallback.
|
|
2809
|
+
// The message below always said "withdrawn" - only the filter was
|
|
2810
|
+
// wrong, which is why the fleet's dead rung stayed invisible.
|
|
2811
|
+
const alive = chain.filter((rung) => {
|
|
2812
|
+
const record = catalog?.[p.provider]?.models?.[rung]
|
|
2813
|
+
return record && !isWithdrawn(record)
|
|
2814
|
+
})
|
|
2645
2815
|
if (!alive.length) {
|
|
2646
2816
|
issues.push(red(`lane ${lane}: every model in its chain has left the catalog`))
|
|
2647
2817
|
} else if (alive.length < chain.length) {
|
|
@@ -3142,6 +3312,9 @@ try {
|
|
|
3142
3312
|
if (flag("--watch") || flag("--check-models")) await cmdDoctorWatch()
|
|
3143
3313
|
else await cmdDoctor()
|
|
3144
3314
|
break
|
|
3315
|
+
case "research":
|
|
3316
|
+
await cmdResearch()
|
|
3317
|
+
break
|
|
3145
3318
|
case "baseline":
|
|
3146
3319
|
await import("./baseline.mjs")
|
|
3147
3320
|
break
|
|
@@ -3174,6 +3347,8 @@ try {
|
|
|
3174
3347
|
lanes report which lanes the fleet actually used
|
|
3175
3348
|
trust set up npm trusted publishing for this package
|
|
3176
3349
|
doctor re-check profiles against models.dev and the installed config
|
|
3350
|
+
research find the live free models the kit could use, and the ones in
|
|
3351
|
+
use that have been withdrawn
|
|
3177
3352
|
upgrade check for a newer kit, refresh the profiles, rewrite the config
|
|
3178
3353
|
backups the configs that were replaced, newest first
|
|
3179
3354
|
uninstall remove the plugin, prompts, skills and GLOBAL.md again
|
|
@@ -3191,8 +3366,8 @@ try {
|
|
|
3191
3366
|
--strict doctor: fail on perishable catalogue drift too
|
|
3192
3367
|
--watch doctor: keep watching the catalogue, report what disappeared
|
|
3193
3368
|
--check-models doctor: one run, non-zero when a shipped model is gone
|
|
3194
|
-
--json doctor, show, lanes, auto: print the
|
|
3195
|
-
and nothing else
|
|
3369
|
+
--json doctor, show, lanes, auto, research: print the
|
|
3370
|
+
machine-readable form, and nothing else
|
|
3196
3371
|
--offline auto: skip the refresh, for a machine that cannot reach
|
|
3197
3372
|
models.dev
|
|
3198
3373
|
--hook install: also install the pre-commit hook into this
|
package/dist/index.js
CHANGED
|
@@ -1103,7 +1103,10 @@ function normalize(p) {
|
|
|
1103
1103
|
keepRecentToolCalls: p.prune?.keepRecentToolCalls ?? 6,
|
|
1104
1104
|
keepRecentTurns: p.prune?.keepRecentTurns ?? 3,
|
|
1105
1105
|
keepUserMessages: p.prune?.keepUserMessages ?? true,
|
|
1106
|
-
keepDistinctOutputs: p.prune?.keepDistinctOutputs ?? true
|
|
1106
|
+
keepDistinctOutputs: p.prune?.keepDistinctOutputs ?? true,
|
|
1107
|
+
// null means "the pruner's own budget decides" - the profile's context
|
|
1108
|
+
// budget. A number is a ceiling in characters.
|
|
1109
|
+
keepDistinctBudgetChars: p.prune?.keepDistinctBudgetChars ?? null
|
|
1107
1110
|
},
|
|
1108
1111
|
fallback: p.fallback ?? []
|
|
1109
1112
|
};
|
|
@@ -1399,6 +1402,13 @@ function validate(p, path = "<memory>") {
|
|
|
1399
1402
|
if (prune.keepDistinctOutputs !== void 0 && typeof prune.keepDistinctOutputs !== "boolean") {
|
|
1400
1403
|
throw new ProfileError(`${where}: prune.keepDistinctOutputs must be a boolean`);
|
|
1401
1404
|
}
|
|
1405
|
+
if (prune.keepDistinctBudgetChars !== void 0 && prune.keepDistinctBudgetChars !== null) {
|
|
1406
|
+
if (!Number.isInteger(prune.keepDistinctBudgetChars) || prune.keepDistinctBudgetChars < 0) {
|
|
1407
|
+
throw new ProfileError(
|
|
1408
|
+
`${where}: prune.keepDistinctBudgetChars must be an integer of at least 0`
|
|
1409
|
+
);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1402
1412
|
if (prune.enabled !== void 0 && typeof prune.enabled !== "boolean") {
|
|
1403
1413
|
throw new ProfileError(`${where}: prune.enabled must be a boolean`);
|
|
1404
1414
|
}
|
|
@@ -1705,33 +1715,58 @@ var PrunePlugin = async ({ worktree, directory }) => {
|
|
|
1705
1715
|
keepTurns: 0,
|
|
1706
1716
|
keepUsers: true,
|
|
1707
1717
|
keepDistinct: false,
|
|
1718
|
+
distinctBudget: 0,
|
|
1708
1719
|
threshold: 0
|
|
1709
1720
|
};
|
|
1710
1721
|
let cached = null;
|
|
1722
|
+
let reportedFailure = false;
|
|
1711
1723
|
const resolve4 = () => {
|
|
1712
1724
|
if (cached) return cached;
|
|
1713
1725
|
let profile = null;
|
|
1714
1726
|
try {
|
|
1715
1727
|
const id = activeProfileId(project);
|
|
1716
1728
|
if (id) profile = loadProfile(id);
|
|
1717
|
-
} catch {
|
|
1729
|
+
} catch (err) {
|
|
1718
1730
|
profile = null;
|
|
1731
|
+
if (!reportedFailure) {
|
|
1732
|
+
reportedFailure = true;
|
|
1733
|
+
console.error(
|
|
1734
|
+
`[opencode-harness-kit] prune disabled: ${err instanceof Error ? err.message : String(err)}`
|
|
1735
|
+
);
|
|
1736
|
+
}
|
|
1719
1737
|
}
|
|
1720
1738
|
if (!profile?.prune?.enabled) return DISABLED;
|
|
1739
|
+
const threshold = pruneThreshold(profile);
|
|
1721
1740
|
cached = {
|
|
1722
1741
|
profile,
|
|
1723
1742
|
keepCalls: profile.prune.keepRecentToolCalls ?? 6,
|
|
1724
1743
|
keepTurns: profile.prune.keepRecentTurns ?? 3,
|
|
1725
1744
|
keepUsers: profile.prune.keepUserMessages ?? true,
|
|
1726
1745
|
keepDistinct: profile.prune.keepDistinctOutputs ?? true,
|
|
1727
|
-
|
|
1746
|
+
// The budget the distinct outputs have to fit in.
|
|
1747
|
+
//
|
|
1748
|
+
// It defaults to the profile's own context budget - the number the
|
|
1749
|
+
// pruner is measured against - because a rule with no ceiling is not a
|
|
1750
|
+
// trade-off, it is a leak. Measured on a real 137-message session: the
|
|
1751
|
+
// request was 5.26 MB and the pruner removed 0.04 MB of it (0.8%),
|
|
1752
|
+
// because EVERY distinct output the session had ever produced was kept
|
|
1753
|
+
// (4.74 MB, 92% of the request) "however old it is". The threshold
|
|
1754
|
+
// that session was supposed to live under was 176,000 characters and
|
|
1755
|
+
// the request was 5,140,000 - twenty-nine times over.
|
|
1756
|
+
//
|
|
1757
|
+
// The ceiling keeps the property the rule was added for (the newest
|
|
1758
|
+
// copy of a file the agent read survives, so it does not read it
|
|
1759
|
+
// again) and drops the part that was never wanted (the copy of every
|
|
1760
|
+
// file the session has EVER touched, for ever).
|
|
1761
|
+
distinctBudget: profile.prune.keepDistinctBudgetChars ?? threshold,
|
|
1762
|
+
threshold
|
|
1728
1763
|
};
|
|
1729
1764
|
return cached;
|
|
1730
1765
|
};
|
|
1731
1766
|
return {
|
|
1732
1767
|
"experimental.chat.messages.transform": async (_input, output) => {
|
|
1733
1768
|
try {
|
|
1734
|
-
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, threshold } = resolve4();
|
|
1769
|
+
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, distinctBudget, threshold } = resolve4();
|
|
1735
1770
|
if (!profile) return;
|
|
1736
1771
|
const messages = output.messages;
|
|
1737
1772
|
if (!Array.isArray(messages) || messages.length === 0) return;
|
|
@@ -1750,17 +1785,30 @@ var PrunePlugin = async ({ worktree, directory }) => {
|
|
|
1750
1785
|
}
|
|
1751
1786
|
if (keepDistinct) {
|
|
1752
1787
|
const seenSignatures = /* @__PURE__ */ new Set();
|
|
1788
|
+
let kept = 0;
|
|
1753
1789
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1754
1790
|
const msg = messages[i];
|
|
1755
1791
|
if (!msg?.parts) continue;
|
|
1792
|
+
let added = 0;
|
|
1756
1793
|
for (const part of msg.parts) {
|
|
1757
1794
|
if (part?.type !== "tool") continue;
|
|
1758
1795
|
if (part.state?.status !== "completed") continue;
|
|
1759
1796
|
const sig = toolSignature(part);
|
|
1760
|
-
if (!sig
|
|
1797
|
+
if (!sig) continue;
|
|
1798
|
+
const seen2 = seenSignatures.has(sig);
|
|
1761
1799
|
seenSignatures.add(sig);
|
|
1762
|
-
|
|
1800
|
+
if (seen2) continue;
|
|
1801
|
+
if (i >= floor) continue;
|
|
1802
|
+
added += partSize(part);
|
|
1763
1803
|
}
|
|
1804
|
+
if (added === 0) continue;
|
|
1805
|
+
added = (msg.parts ?? []).reduce(
|
|
1806
|
+
(n, p) => n + partSize(p),
|
|
1807
|
+
0
|
|
1808
|
+
);
|
|
1809
|
+
if (kept + added > distinctBudget) break;
|
|
1810
|
+
kept += added;
|
|
1811
|
+
keepToolMessageIds.add(msg.info?.id ?? String(i));
|
|
1764
1812
|
}
|
|
1765
1813
|
}
|
|
1766
1814
|
const stats = {
|
|
@@ -1884,7 +1932,7 @@ function utilityModel(p) {
|
|
|
1884
1932
|
}
|
|
1885
1933
|
|
|
1886
1934
|
// src/health.js
|
|
1887
|
-
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync } from "node:fs";
|
|
1935
|
+
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync3, renameSync as renameSync2, unlinkSync, writeFileSync } from "node:fs";
|
|
1888
1936
|
import { join as join7 } from "node:path";
|
|
1889
1937
|
var DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1e3;
|
|
1890
1938
|
function spentWindowMs() {
|
|
@@ -1935,9 +1983,21 @@ function readHealth(dir) {
|
|
|
1935
1983
|
}
|
|
1936
1984
|
function writeHealth(dir, health) {
|
|
1937
1985
|
try {
|
|
1938
|
-
|
|
1986
|
+
const spaceDir = join7(dir, ".space");
|
|
1987
|
+
if (!existsSync5(spaceDir)) mkdirSync2(spaceDir, { recursive: true });
|
|
1939
1988
|
const pruned = { ...health ?? {}, unhealthy: liveUnhealthy(health) };
|
|
1940
|
-
|
|
1989
|
+
const file = healthPath(dir);
|
|
1990
|
+
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
|
1991
|
+
try {
|
|
1992
|
+
writeFileSync(tmp, JSON.stringify(pruned, null, 2));
|
|
1993
|
+
renameSync2(tmp, file);
|
|
1994
|
+
} catch (err) {
|
|
1995
|
+
try {
|
|
1996
|
+
unlinkSync(tmp);
|
|
1997
|
+
} catch {
|
|
1998
|
+
}
|
|
1999
|
+
throw err;
|
|
2000
|
+
}
|
|
1941
2001
|
} catch {
|
|
1942
2002
|
}
|
|
1943
2003
|
}
|
package/dist/profile.d.ts
CHANGED
|
@@ -55,6 +55,7 @@ export type Profile = {
|
|
|
55
55
|
keepRecentTurns?: number;
|
|
56
56
|
keepUserMessages?: boolean;
|
|
57
57
|
keepDistinctOutputs?: boolean;
|
|
58
|
+
keepDistinctBudgetChars?: number | null;
|
|
58
59
|
};
|
|
59
60
|
fallback?: string[];
|
|
60
61
|
};
|
|
@@ -88,7 +89,7 @@ export type RetentionRecord = {
|
|
|
88
89
|
* @property {Record<string, RoleModel>} [roleModels]
|
|
89
90
|
* @property {{ enabled?: boolean, keepRecentToolCalls?: number,
|
|
90
91
|
* keepRecentTurns?: number, keepUserMessages?: boolean,
|
|
91
|
-
* keepDistinctOutputs?: boolean }} [prune]
|
|
92
|
+
* keepDistinctOutputs?: boolean, keepDistinctBudgetChars?: number | null }} [prune]
|
|
92
93
|
* @property {string[]} [fallback]
|
|
93
94
|
* @typedef {{ retention: Retention, note?: string, verified?: string, source?: string }} RetentionRecord
|
|
94
95
|
*/
|
package/docs/KNOWN-LIMITS.md
CHANGED
|
@@ -373,12 +373,32 @@ RECENT calls was a window on recency, and a long session walks out of it: after
|
|
|
373
373
|
six more tool calls, the only copy of a file the agent had read was gone, and
|
|
374
374
|
the agent read it again. The long-task A/B measured seventeen re-reads, five
|
|
375
375
|
times the input tokens and the same final context. The rule now keeps the
|
|
376
|
-
newest copy of each distinct output
|
|
377
|
-
REPEATS of it. The context grows by the size of the distinct outputs, and the
|
|
376
|
+
newest copy of each distinct output, and stubs only the REPEATS of it. The
|
|
378
377
|
re-reads stop: **-69% input tokens and -60% wall time** on that task, at the
|
|
379
378
|
cost of a bigger context. It is a setting - `prune.keepDistinctOutputs` - and
|
|
380
379
|
turning it off brings the old behaviour back.
|
|
381
380
|
|
|
381
|
+
**And the distinct outputs have a ceiling.** The rule as it first shipped had
|
|
382
|
+
no upper bound, which made it a leak rather than a trade-off: it kept every
|
|
383
|
+
output the session had ever produced, and a long session grew without limit.
|
|
384
|
+
Measured on a real 137-message session on 2026-10-22: the request was 5.26 MB,
|
|
385
|
+
the pruner removed 0.04 MB of it - 0.8% - and 4.74 MB of it, 92%, was the
|
|
386
|
+
distinct outputs of the whole session. The profile's own budget for that
|
|
387
|
+
request was 176,000 characters, so the request was twenty-nine times over it.
|
|
388
|
+
|
|
389
|
+
A request that size is not only expensive, it is what makes the server
|
|
390
|
+
slow: opencode persists a full copy of every updated part into its event
|
|
391
|
+
store, so every turn writes megabytes into a database that on that machine had
|
|
392
|
+
reached 4.2 GB, and the TUI - which is a client of that server - started
|
|
393
|
+
dropping keystrokes mid-word.
|
|
394
|
+
|
|
395
|
+
So the distinct outputs are now kept newest-first and only while they fit in
|
|
396
|
+
a budget: `prune.keepDistinctBudgetChars`, unset meaning the profile's own
|
|
397
|
+
context budget, `0` meaning none. The property the rule was added for still
|
|
398
|
+
holds - the copy of a recent output the agent is working with survives, so it
|
|
399
|
+
does not read the file again - and the part that was never wanted, the copy of
|
|
400
|
+
everything the session has ever touched for ever, is gone.
|
|
401
|
+
|
|
382
402
|
`savedChars` in `prune.jsonl` is the size of the request before the pass minus
|
|
383
403
|
the size after it, measured on the parts as they are serialised - not on the
|
|
384
404
|
text inside them. It used to measure the text only, and to ignore the state
|
package/eval/ab.mjs
CHANGED
|
@@ -750,7 +750,7 @@ console.log("")
|
|
|
750
750
|
console.log(` ${"metric (median)".padEnd(16)} ${"kit on".padStart(10)} ${"kit off".padStart(10)}`)
|
|
751
751
|
console.log(` ${"-".repeat(16)} ${"-".repeat(10)} ${"-".repeat(10)}`)
|
|
752
752
|
for (const [label, a, b] of rows) {
|
|
753
|
-
console.log(` ${label.padEnd(16)} ${String(a).padStart(10)} ${String(b).padStart(10)}`)
|
|
753
|
+
console.log(` ${String(label).padEnd(16)} ${String(a).padStart(10)} ${String(b).padStart(10)}`)
|
|
754
754
|
}
|
|
755
755
|
|
|
756
756
|
if (jobs.length > 1) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-harness-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",
|
package/profiles/beam-free.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "beam-free",
|
|
7
7
|
"family": "beam",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "retained",
|
|
10
10
|
"notes": "Reflection 501B-A23B, text only, and the freshest knowledge of the free set at 2026-06-30. Measured as the fastest route on the tier: 2.6s average over seven requests. Its provider does not train on prompts but retains them for 30 days, which is why it is `retained` rather than `zero-retention`.",
|
|
11
11
|
"limits": {
|
|
@@ -45,7 +45,6 @@
|
|
|
45
45
|
},
|
|
46
46
|
"fallback": [
|
|
47
47
|
"deepseek-v4.1-flash-free",
|
|
48
|
-
"glm-5.3-flash-free"
|
|
49
|
-
"longcat-2.5-preview-free"
|
|
48
|
+
"glm-5.3-flash-free"
|
|
50
49
|
]
|
|
51
50
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "deepseek-v4.1-flash-free",
|
|
7
7
|
"family": "deepseek-flash",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"notes": "Reasoning plus agentic coding plus structured output on a zero-retention provider. The natural schema lane. Its knowledge cutoff is 2025-05, the oldest of the tier, so it is not the lane for anything that depends on recent facts.",
|
|
11
11
|
"limits": {
|
|
@@ -43,7 +43,6 @@
|
|
|
43
43
|
},
|
|
44
44
|
"fallback": [
|
|
45
45
|
"glm-5.3-flash-free",
|
|
46
|
-
"longcat-2.5-preview-free",
|
|
47
46
|
"beam-free"
|
|
48
47
|
]
|
|
49
48
|
}
|
package/profiles/fleet-all.json
CHANGED
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
"id": "fleet-all",
|
|
4
4
|
"title": "Ten jobs, every free model, one of them may train",
|
|
5
5
|
"provider": "opencode",
|
|
6
|
-
"model": "
|
|
7
|
-
"family":
|
|
8
|
-
"verified": "2026-10-
|
|
6
|
+
"model": "flint-40b-free",
|
|
7
|
+
"family": "flint",
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"maxRetention": "may-train",
|
|
11
11
|
"notes": "Every lane gets the model built for it: beam plans, Nemotron 3 Ultra reviews, Lightning sweeps, MiMo sees, GLM audits, Super VL reasons with the effort turned off. Five of the ten providers state that collected data may be used to improve the model, or log the session, so the profile as a whole is may-train and the generated config says so in its header. Install it for public or throwaway repositories.",
|
|
12
12
|
"limits": {
|
|
13
|
-
"context":
|
|
14
|
-
"output":
|
|
13
|
+
"context": 262144,
|
|
14
|
+
"output": 65536
|
|
15
15
|
},
|
|
16
16
|
"supports": {
|
|
17
17
|
"reasoning": true,
|
|
@@ -49,19 +49,17 @@
|
|
|
49
49
|
},
|
|
50
50
|
"roleModels": {
|
|
51
51
|
"build": {
|
|
52
|
-
"model": "
|
|
52
|
+
"model": "flint-40b-free",
|
|
53
53
|
"chain": [
|
|
54
|
-
"
|
|
55
|
-
"deepseek-v4.1-flash-free"
|
|
56
|
-
"longcat-2.5-preview-free"
|
|
54
|
+
"flint-40b-free",
|
|
55
|
+
"deepseek-v4.1-flash-free"
|
|
57
56
|
]
|
|
58
57
|
},
|
|
59
58
|
"plan": {
|
|
60
59
|
"model": "beam-free",
|
|
61
60
|
"chain": [
|
|
62
61
|
"beam-free",
|
|
63
|
-
"deepseek-v4.1-flash-free"
|
|
64
|
-
"longcat-2.5-preview-free"
|
|
62
|
+
"deepseek-v4.1-flash-free"
|
|
65
63
|
]
|
|
66
64
|
},
|
|
67
65
|
"reviewer": {
|
|
@@ -77,7 +75,6 @@
|
|
|
77
75
|
"chain": [
|
|
78
76
|
"nemotron-3.5-lightning-free",
|
|
79
77
|
"big-pickle",
|
|
80
|
-
"longcat-2.5-preview-free",
|
|
81
78
|
"deepseek-v4.1-flash-free"
|
|
82
79
|
]
|
|
83
80
|
},
|
|
@@ -93,16 +90,14 @@
|
|
|
93
90
|
"model": "mimo-v2.6-flash-free",
|
|
94
91
|
"chain": [
|
|
95
92
|
"mimo-v2.6-flash-free",
|
|
96
|
-
"
|
|
97
|
-
"minimax-m3.1-flash-free"
|
|
93
|
+
"deepseek-v4.1-flash-free"
|
|
98
94
|
]
|
|
99
95
|
},
|
|
100
96
|
"doc": {
|
|
101
97
|
"model": "glm-5.3-flash-free",
|
|
102
98
|
"chain": [
|
|
103
99
|
"glm-5.3-flash-free",
|
|
104
|
-
"deepseek-v4.1-flash-free"
|
|
105
|
-
"longcat-2.5-preview-free"
|
|
100
|
+
"deepseek-v4.1-flash-free"
|
|
106
101
|
]
|
|
107
102
|
},
|
|
108
103
|
"schema": {
|
|
@@ -117,15 +112,13 @@
|
|
|
117
112
|
"model": "nemotron-3.5-super-vl-free",
|
|
118
113
|
"chain": [
|
|
119
114
|
"nemotron-3.5-super-vl-free",
|
|
120
|
-
"longcat-2.5-preview-free",
|
|
121
115
|
"beam-free"
|
|
122
116
|
]
|
|
123
117
|
},
|
|
124
118
|
"longform": {
|
|
125
|
-
"model": "
|
|
119
|
+
"model": "flint-40b-free",
|
|
126
120
|
"chain": [
|
|
127
|
-
"
|
|
128
|
-
"minimax-m3.1-flash-free",
|
|
121
|
+
"flint-40b-free",
|
|
129
122
|
"beam-free"
|
|
130
123
|
]
|
|
131
124
|
}
|
|
@@ -137,8 +130,7 @@
|
|
|
137
130
|
"keepUserMessages": true
|
|
138
131
|
},
|
|
139
132
|
"fallback": [
|
|
140
|
-
"
|
|
141
|
-
"deepseek-v4.1-flash-free"
|
|
142
|
-
"longcat-2.5-preview-free"
|
|
133
|
+
"flint-40b-free",
|
|
134
|
+
"deepseek-v4.1-flash-free"
|
|
143
135
|
]
|
|
144
136
|
}
|