opencode-harness-kit 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/README.md +217 -21
- package/bin/oc-harness.mjs +177 -10
- package/dist/index.js +54 -6
- package/dist/profile.d.ts +2 -1
- package/docs/KNOWN-LIMITS.md +22 -2
- package/package.json +1 -1
- package/profiles/beam-free.json +2 -3
- package/profiles/deepseek-v4.1-flash-free.json +1 -2
- package/profiles/fleet-all.json +6 -14
- package/profiles/fleet-safe.json +8 -19
- package/profiles/glm-5.3-flash-free.json +1 -2
- package/profiles/index.json +4 -11
- package/profiles/mimo-v2.6-flash-free.json +1 -2
- package/profiles/minimax-m3.1-flash-free.json +1 -2
- package/profiles/nemotron-3.5-lightning-free.json +2 -3
- package/profiles/nemotron-3.5-super-vl-free.json +1 -2
- package/profiles/schema.json +8 -0
- package/retention.yml +0 -9
- package/src/profile.js +14 -1
- package/src/prune.ts +90 -5
- package/src/research.js +462 -0
- package/profiles/longcat-2.5-preview-free.json +0 -45
package/bin/oc-harness.mjs
CHANGED
|
@@ -47,6 +47,7 @@ import { generateConfig, globalInstructionsPath, utilityModel, withMemoryPlugin,
|
|
|
47
47
|
import { ruleNames, BASH_RULES, WRITE_RULES } from "../src/rules.js"
|
|
48
48
|
import { planEffort } from "../src/effort.js"
|
|
49
49
|
import { findTelemetryFiles, readLaneUsage, averageMs } from "../src/lane-usage.js"
|
|
50
|
+
import { researchModels, isWithdrawn, pruneWithdrawnChains } from "../src/research.js"
|
|
50
51
|
import { main as trustMain } from "../src/trust.js"
|
|
51
52
|
|
|
52
53
|
const CONFIG_DIR =
|
|
@@ -1702,6 +1703,11 @@ function diffProfile(p, live) {
|
|
|
1702
1703
|
* Only machine-derived fields are rewritten. `retention`, `notes`, `roles` and
|
|
1703
1704
|
* the fallback queue are human judgements and are never touched by a script:
|
|
1704
1705
|
* losing a privacy verdict to an automated rewrite would be worse than drift.
|
|
1706
|
+
*
|
|
1707
|
+
* The ONE exception is a chain rung the provider has withdrawn. A chain names
|
|
1708
|
+
* the models that could serve a lane, and a withdrawn model is not one of them
|
|
1709
|
+
* any more - it cannot run. Removing it is a derivation from live metadata
|
|
1710
|
+
* and it only ever removes; nothing is ever put INTO a lane.
|
|
1705
1711
|
*/
|
|
1706
1712
|
/**
|
|
1707
1713
|
* Refresh every shipped profile against the live catalogue.
|
|
@@ -1742,8 +1748,18 @@ async function cmdRefreshAll() {
|
|
|
1742
1748
|
gone.push(`${id} (${p.provider}/${p.model})`)
|
|
1743
1749
|
continue
|
|
1744
1750
|
}
|
|
1751
|
+
// A model the provider has withdrawn is still IN the catalogue, so the
|
|
1752
|
+
// check above passes it. A profile whose own model was withdrawn has no
|
|
1753
|
+
// chain behind it to fall back to, so there is nothing a refresh can
|
|
1754
|
+
// repair - it has to be retired, and that is a person's call.
|
|
1755
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
1756
|
+
if (prune.dead) {
|
|
1757
|
+
gone.push(`${id} (${p.provider}/${p.model} was withdrawn from the provider)`)
|
|
1758
|
+
continue
|
|
1759
|
+
}
|
|
1745
1760
|
const before = diffProfile(p, live)
|
|
1746
|
-
|
|
1761
|
+
const pruned = prune.pruned.length > 0
|
|
1762
|
+
if (!before.length && !pruned) {
|
|
1747
1763
|
same.push(id)
|
|
1748
1764
|
continue
|
|
1749
1765
|
}
|
|
@@ -1751,8 +1767,18 @@ async function cmdRefreshAll() {
|
|
|
1751
1767
|
// `verified` is the field that says when a profile was last checked
|
|
1752
1768
|
// against live metadata. Leaving it stale after a refresh makes it a lie.
|
|
1753
1769
|
p.verified = new Date().toISOString().slice(0, 10)
|
|
1770
|
+
const notes = [...before]
|
|
1771
|
+
if (pruned) {
|
|
1772
|
+
notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
|
|
1773
|
+
for (const lane of prune.lanes) {
|
|
1774
|
+
notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
|
|
1775
|
+
}
|
|
1776
|
+
if (prune.utilityMoved) {
|
|
1777
|
+
notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1754
1780
|
if (flag("--dryrun")) {
|
|
1755
|
-
changed.push(`${id}: ${
|
|
1781
|
+
changed.push(`${id}: ${notes.join("; ")} (dry run, nothing written)`)
|
|
1756
1782
|
continue
|
|
1757
1783
|
}
|
|
1758
1784
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
@@ -1765,7 +1791,7 @@ async function cmdRefreshAll() {
|
|
|
1765
1791
|
continue
|
|
1766
1792
|
}
|
|
1767
1793
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
1768
|
-
changed.push(`${id}: ${
|
|
1794
|
+
changed.push(`${id}: ${notes.join("; ")}`)
|
|
1769
1795
|
}
|
|
1770
1796
|
console.log(
|
|
1771
1797
|
`\n${bold("refreshed")} ${stamp(`${checked} of ${ids.length} profile(s) against the live catalogue`)}\n`,
|
|
@@ -2303,8 +2329,30 @@ async function cmdRefresh(id) {
|
|
|
2303
2329
|
)
|
|
2304
2330
|
}
|
|
2305
2331
|
|
|
2332
|
+
// A withdrawn model is still in the catalogue, so the check above passes
|
|
2333
|
+
// it. A profile whose own model was withdrawn has no chain behind it, so
|
|
2334
|
+
// there is nothing to fall back to and nothing a refresh can repair.
|
|
2335
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
2336
|
+
if (prune.dead) {
|
|
2337
|
+
fail(
|
|
2338
|
+
`${p.provider}/${p.model} was withdrawn from the provider.\n` +
|
|
2339
|
+
` This profile has no chain to fall back to, so it cannot be repaired.\n` +
|
|
2340
|
+
` Remove it, or point it at a model that is still live.`,
|
|
2341
|
+
)
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2306
2344
|
const before = diffProfile(p, live)
|
|
2307
|
-
|
|
2345
|
+
const notes = [...before]
|
|
2346
|
+
if (prune.pruned.length > 0) {
|
|
2347
|
+
notes.push(`withdrawn rungs removed: ${prune.pruned.join(", ")}`)
|
|
2348
|
+
for (const lane of prune.lanes) {
|
|
2349
|
+
notes.push(`lane ${lane.lane}: ${lane.from.join(", ")} -> ${lane.to.join(", ")}`)
|
|
2350
|
+
}
|
|
2351
|
+
if (prune.utilityMoved) {
|
|
2352
|
+
notes.push(`utility route: ${prune.utilityMoved} -> ${p.utility.model}`)
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
if (notes.length === 0) {
|
|
2308
2356
|
console.log(`\n ${green("✓")} ${id} already matches live metadata\n`)
|
|
2309
2357
|
return
|
|
2310
2358
|
}
|
|
@@ -2337,7 +2385,7 @@ async function cmdRefresh(id) {
|
|
|
2337
2385
|
|
|
2338
2386
|
if (flag("--dryrun")) {
|
|
2339
2387
|
console.log(`\n ${stamp("dry run, nothing written")}`)
|
|
2340
|
-
for (const line of
|
|
2388
|
+
for (const line of notes) console.log(` ${line}`)
|
|
2341
2389
|
return
|
|
2342
2390
|
}
|
|
2343
2391
|
// A refresh overwrites a file the user may have edited by hand, and for
|
|
@@ -2352,9 +2400,9 @@ async function cmdRefresh(id) {
|
|
|
2352
2400
|
}
|
|
2353
2401
|
writeFileSync(path, `${JSON.stringify(p, null, 2)}\n`)
|
|
2354
2402
|
console.log(`\n ${green("✓")} refreshed ${path}`)
|
|
2355
|
-
for (const line of
|
|
2403
|
+
for (const line of notes) console.log(` ${line}`)
|
|
2356
2404
|
console.log(
|
|
2357
|
-
`\n ${stamp("kept as-is: retention, roles, notes
|
|
2405
|
+
`\n ${stamp("kept as-is: retention, roles, notes (human judgement)")}\n`,
|
|
2358
2406
|
)
|
|
2359
2407
|
}
|
|
2360
2408
|
|
|
@@ -2594,6 +2642,111 @@ async function cmdDoctor() {
|
|
|
2594
2642
|
// second doctor in the same process does not inherit the first one's findings.
|
|
2595
2643
|
const doctorReport = { ok: true, problems: 0, drift: 0, strict: false, findings: [], json: false }
|
|
2596
2644
|
|
|
2645
|
+
/**
|
|
2646
|
+
* Research the live tier for models the kit could use.
|
|
2647
|
+
*
|
|
2648
|
+
* `doctor` says when a model has gone. This says what has come to replace
|
|
2649
|
+
* it, and what is still a person's decision.
|
|
2650
|
+
*
|
|
2651
|
+
* It writes nothing. The shortlist it prints is a list to argue with, not a
|
|
2652
|
+
* list to accept: `retention` is a judgement about a provider's terms, and
|
|
2653
|
+
* that is the one field a script must not fill in.
|
|
2654
|
+
*/
|
|
2655
|
+
async function cmdResearch() {
|
|
2656
|
+
const wantJson = flag("--json")
|
|
2657
|
+
let catalog
|
|
2658
|
+
try {
|
|
2659
|
+
catalog = await fetchCatalog()
|
|
2660
|
+
} catch (err) {
|
|
2661
|
+
fail(err instanceof Error ? err.message : String(err), wantJson)
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2664
|
+
// A profile that cannot be read is `doctor`'s finding, not this command's.
|
|
2665
|
+
// Research is about the tier, so a broken profile is left out of the
|
|
2666
|
+
// comparison rather than failing the whole report.
|
|
2667
|
+
const profiles = []
|
|
2668
|
+
for (const id of listProfileIds()) {
|
|
2669
|
+
try {
|
|
2670
|
+
profiles.push(loadProfile(id))
|
|
2671
|
+
} catch {
|
|
2672
|
+
// doctor reports this, with the profile's own name on it.
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
const report = researchModels({
|
|
2677
|
+
catalog,
|
|
2678
|
+
profiles,
|
|
2679
|
+
retentionOf: (model) => modelRetention(model),
|
|
2680
|
+
defaultProfileId: index().default ?? null,
|
|
2681
|
+
})
|
|
2682
|
+
|
|
2683
|
+
if (wantJson) {
|
|
2684
|
+
console.log(JSON.stringify({ ok: true, ...report }, null, 2))
|
|
2685
|
+
return
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
console.log(
|
|
2689
|
+
`\n${bold("research")} ${stamp(`tier ${report.tier.provider}, ${report.tier.live} live of ${report.tier.total} free`)}\n`,
|
|
2690
|
+
)
|
|
2691
|
+
|
|
2692
|
+
// The models the kit uses that the provider has pulled. This is the part
|
|
2693
|
+
// that has to be acted on, so it comes first.
|
|
2694
|
+
if (report.shipped.gone.length) {
|
|
2695
|
+
console.log(` ${red("gone")} ${report.shipped.gone.length} model(s) in use have been withdrawn\n`)
|
|
2696
|
+
for (const g of report.shipped.gone) {
|
|
2697
|
+
console.log(` ${red("✗")} ${g.id}`)
|
|
2698
|
+
console.log(` ${g.reason}`)
|
|
2699
|
+
if (g.asBase.length) {
|
|
2700
|
+
console.log(` ${stamp("base model of")} ${g.asBase.join(", ")}`)
|
|
2701
|
+
}
|
|
2702
|
+
for (const [profile, laneList] of Object.entries(g.inChains ?? {})) {
|
|
2703
|
+
console.log(` ${stamp("fallback rung in")} ${profile}: ${laneList.join(", ")}`)
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
console.log("")
|
|
2707
|
+
} else {
|
|
2708
|
+
console.log(` ${green("✓")} every model in use is still live\n`)
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
// Per lane: which rungs of its chain are alive. A lane whose whole chain
|
|
2712
|
+
// has gone is a lane with no fallback at all.
|
|
2713
|
+
if (report.lanes.length) {
|
|
2714
|
+
console.log(` ${bold("lanes")}`)
|
|
2715
|
+
for (const l of report.lanes) {
|
|
2716
|
+
const dead = l.dead.length
|
|
2717
|
+
? yellow(` ${l.dead.length} dead (${l.dead.join(", ")})`)
|
|
2718
|
+
: green(" 0 dead")
|
|
2719
|
+
console.log(` ${l.lane.padEnd(12)} ${String(l.live.length).padStart(2)} live${dead}`)
|
|
2720
|
+
}
|
|
2721
|
+
console.log("")
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
// The shortlist. What each candidate can do, which lanes it could serve,
|
|
2725
|
+
// and what is still undecided about it.
|
|
2726
|
+
if (!report.candidates.length) {
|
|
2727
|
+
console.log(` ${stamp("no live free model outside the ones the kit already uses")}\n`)
|
|
2728
|
+
return
|
|
2729
|
+
}
|
|
2730
|
+
console.log(` ${bold("candidates")} ${stamp("live, free, and not in use yet")}\n`)
|
|
2731
|
+
for (const c of report.candidates) {
|
|
2732
|
+
const effort = c.effort.length ? c.effort.join(",") : "none published"
|
|
2733
|
+
console.log(
|
|
2734
|
+
` ${bold(c.id)} ${stamp(
|
|
2735
|
+
`ctx ${Number(c.context).toLocaleString("en-US")} out ${Number(c.output).toLocaleString("en-US")} effort ${effort}`,
|
|
2736
|
+
)}`,
|
|
2737
|
+
)
|
|
2738
|
+
console.log(` fits: ${c.fits.length ? c.fits.join(", ") : "no lane on its own terms"}`)
|
|
2739
|
+
console.log(
|
|
2740
|
+
` ${c.retention === "unknown" ? yellow("retention: unknown") : stamp(`retention: ${c.retention}`)}`,
|
|
2741
|
+
)
|
|
2742
|
+
for (const b of c.blockers) console.log(` ${yellow(b)}`)
|
|
2743
|
+
console.log("")
|
|
2744
|
+
}
|
|
2745
|
+
console.log(
|
|
2746
|
+
` ${stamp("nothing here is written anywhere. A retention verdict is a person reading a provider's terms.")}\n`,
|
|
2747
|
+
)
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2597
2750
|
async function runDoctor(catalog) {
|
|
2598
2751
|
const retention = loadRetention().models
|
|
2599
2752
|
const ids = listProfileIds()
|
|
@@ -2641,7 +2794,16 @@ async function runDoctor(catalog) {
|
|
|
2641
2794
|
// with no model at all, which no amount of `refresh` on the base profile
|
|
2642
2795
|
// would reveal.
|
|
2643
2796
|
for (const [lane, chain] of Object.entries(chainOf(p))) {
|
|
2644
|
-
|
|
2797
|
+
// A withdrawn model is still IN the catalog: models.dev keeps the
|
|
2798
|
+
// record and marks it deprecated rather than deleting it. So "is it
|
|
2799
|
+
// still there" is not the same question as "can it still run", and
|
|
2800
|
+
// asking the first one reported a dead model as a healthy fallback.
|
|
2801
|
+
// The message below always said "withdrawn" - only the filter was
|
|
2802
|
+
// wrong, which is why the fleet's dead rung stayed invisible.
|
|
2803
|
+
const alive = chain.filter((rung) => {
|
|
2804
|
+
const record = catalog?.[p.provider]?.models?.[rung]
|
|
2805
|
+
return record && !isWithdrawn(record)
|
|
2806
|
+
})
|
|
2645
2807
|
if (!alive.length) {
|
|
2646
2808
|
issues.push(red(`lane ${lane}: every model in its chain has left the catalog`))
|
|
2647
2809
|
} else if (alive.length < chain.length) {
|
|
@@ -3142,6 +3304,9 @@ try {
|
|
|
3142
3304
|
if (flag("--watch") || flag("--check-models")) await cmdDoctorWatch()
|
|
3143
3305
|
else await cmdDoctor()
|
|
3144
3306
|
break
|
|
3307
|
+
case "research":
|
|
3308
|
+
await cmdResearch()
|
|
3309
|
+
break
|
|
3145
3310
|
case "baseline":
|
|
3146
3311
|
await import("./baseline.mjs")
|
|
3147
3312
|
break
|
|
@@ -3174,6 +3339,8 @@ try {
|
|
|
3174
3339
|
lanes report which lanes the fleet actually used
|
|
3175
3340
|
trust set up npm trusted publishing for this package
|
|
3176
3341
|
doctor re-check profiles against models.dev and the installed config
|
|
3342
|
+
research find the live free models the kit could use, and the ones in
|
|
3343
|
+
use that have been withdrawn
|
|
3177
3344
|
upgrade check for a newer kit, refresh the profiles, rewrite the config
|
|
3178
3345
|
backups the configs that were replaced, newest first
|
|
3179
3346
|
uninstall remove the plugin, prompts, skills and GLOBAL.md again
|
|
@@ -3191,8 +3358,8 @@ try {
|
|
|
3191
3358
|
--strict doctor: fail on perishable catalogue drift too
|
|
3192
3359
|
--watch doctor: keep watching the catalogue, report what disappeared
|
|
3193
3360
|
--check-models doctor: one run, non-zero when a shipped model is gone
|
|
3194
|
-
--json doctor, show, lanes, auto: print the
|
|
3195
|
-
and nothing else
|
|
3361
|
+
--json doctor, show, lanes, auto, research: print the
|
|
3362
|
+
machine-readable form, and nothing else
|
|
3196
3363
|
--offline auto: skip the refresh, for a machine that cannot reach
|
|
3197
3364
|
models.dev
|
|
3198
3365
|
--hook install: also install the pre-commit hook into this
|
package/dist/index.js
CHANGED
|
@@ -1103,7 +1103,10 @@ function normalize(p) {
|
|
|
1103
1103
|
keepRecentToolCalls: p.prune?.keepRecentToolCalls ?? 6,
|
|
1104
1104
|
keepRecentTurns: p.prune?.keepRecentTurns ?? 3,
|
|
1105
1105
|
keepUserMessages: p.prune?.keepUserMessages ?? true,
|
|
1106
|
-
keepDistinctOutputs: p.prune?.keepDistinctOutputs ?? true
|
|
1106
|
+
keepDistinctOutputs: p.prune?.keepDistinctOutputs ?? true,
|
|
1107
|
+
// null means "the pruner's own budget decides" - the profile's context
|
|
1108
|
+
// budget. A number is a ceiling in characters.
|
|
1109
|
+
keepDistinctBudgetChars: p.prune?.keepDistinctBudgetChars ?? null
|
|
1107
1110
|
},
|
|
1108
1111
|
fallback: p.fallback ?? []
|
|
1109
1112
|
};
|
|
@@ -1399,6 +1402,13 @@ function validate(p, path = "<memory>") {
|
|
|
1399
1402
|
if (prune.keepDistinctOutputs !== void 0 && typeof prune.keepDistinctOutputs !== "boolean") {
|
|
1400
1403
|
throw new ProfileError(`${where}: prune.keepDistinctOutputs must be a boolean`);
|
|
1401
1404
|
}
|
|
1405
|
+
if (prune.keepDistinctBudgetChars !== void 0 && prune.keepDistinctBudgetChars !== null) {
|
|
1406
|
+
if (!Number.isInteger(prune.keepDistinctBudgetChars) || prune.keepDistinctBudgetChars < 0) {
|
|
1407
|
+
throw new ProfileError(
|
|
1408
|
+
`${where}: prune.keepDistinctBudgetChars must be an integer of at least 0`
|
|
1409
|
+
);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1402
1412
|
if (prune.enabled !== void 0 && typeof prune.enabled !== "boolean") {
|
|
1403
1413
|
throw new ProfileError(`${where}: prune.enabled must be a boolean`);
|
|
1404
1414
|
}
|
|
@@ -1705,33 +1715,58 @@ var PrunePlugin = async ({ worktree, directory }) => {
|
|
|
1705
1715
|
keepTurns: 0,
|
|
1706
1716
|
keepUsers: true,
|
|
1707
1717
|
keepDistinct: false,
|
|
1718
|
+
distinctBudget: 0,
|
|
1708
1719
|
threshold: 0
|
|
1709
1720
|
};
|
|
1710
1721
|
let cached = null;
|
|
1722
|
+
let reportedFailure = false;
|
|
1711
1723
|
const resolve4 = () => {
|
|
1712
1724
|
if (cached) return cached;
|
|
1713
1725
|
let profile = null;
|
|
1714
1726
|
try {
|
|
1715
1727
|
const id = activeProfileId(project);
|
|
1716
1728
|
if (id) profile = loadProfile(id);
|
|
1717
|
-
} catch {
|
|
1729
|
+
} catch (err) {
|
|
1718
1730
|
profile = null;
|
|
1731
|
+
if (!reportedFailure) {
|
|
1732
|
+
reportedFailure = true;
|
|
1733
|
+
console.error(
|
|
1734
|
+
`[opencode-harness-kit] prune disabled: ${err instanceof Error ? err.message : String(err)}`
|
|
1735
|
+
);
|
|
1736
|
+
}
|
|
1719
1737
|
}
|
|
1720
1738
|
if (!profile?.prune?.enabled) return DISABLED;
|
|
1739
|
+
const threshold = pruneThreshold(profile);
|
|
1721
1740
|
cached = {
|
|
1722
1741
|
profile,
|
|
1723
1742
|
keepCalls: profile.prune.keepRecentToolCalls ?? 6,
|
|
1724
1743
|
keepTurns: profile.prune.keepRecentTurns ?? 3,
|
|
1725
1744
|
keepUsers: profile.prune.keepUserMessages ?? true,
|
|
1726
1745
|
keepDistinct: profile.prune.keepDistinctOutputs ?? true,
|
|
1727
|
-
|
|
1746
|
+
// The budget the distinct outputs have to fit in.
|
|
1747
|
+
//
|
|
1748
|
+
// It defaults to the profile's own context budget - the number the
|
|
1749
|
+
// pruner is measured against - because a rule with no ceiling is not a
|
|
1750
|
+
// trade-off, it is a leak. Measured on a real 137-message session: the
|
|
1751
|
+
// request was 5.26 MB and the pruner removed 0.04 MB of it (0.8%),
|
|
1752
|
+
// because EVERY distinct output the session had ever produced was kept
|
|
1753
|
+
// (4.74 MB, 92% of the request) "however old it is". The threshold
|
|
1754
|
+
// that session was supposed to live under was 176,000 characters and
|
|
1755
|
+
// the request was 5,140,000 - twenty-nine times over.
|
|
1756
|
+
//
|
|
1757
|
+
// The ceiling keeps the property the rule was added for (the newest
|
|
1758
|
+
// copy of a file the agent read survives, so it does not read it
|
|
1759
|
+
// again) and drops the part that was never wanted (the copy of every
|
|
1760
|
+
// file the session has EVER touched, for ever).
|
|
1761
|
+
distinctBudget: profile.prune.keepDistinctBudgetChars ?? threshold,
|
|
1762
|
+
threshold
|
|
1728
1763
|
};
|
|
1729
1764
|
return cached;
|
|
1730
1765
|
};
|
|
1731
1766
|
return {
|
|
1732
1767
|
"experimental.chat.messages.transform": async (_input, output) => {
|
|
1733
1768
|
try {
|
|
1734
|
-
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, threshold } = resolve4();
|
|
1769
|
+
const { profile, keepCalls, keepTurns, keepUsers, keepDistinct, distinctBudget, threshold } = resolve4();
|
|
1735
1770
|
if (!profile) return;
|
|
1736
1771
|
const messages = output.messages;
|
|
1737
1772
|
if (!Array.isArray(messages) || messages.length === 0) return;
|
|
@@ -1750,17 +1785,30 @@ var PrunePlugin = async ({ worktree, directory }) => {
|
|
|
1750
1785
|
}
|
|
1751
1786
|
if (keepDistinct) {
|
|
1752
1787
|
const seenSignatures = /* @__PURE__ */ new Set();
|
|
1788
|
+
let kept = 0;
|
|
1753
1789
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1754
1790
|
const msg = messages[i];
|
|
1755
1791
|
if (!msg?.parts) continue;
|
|
1792
|
+
let added = 0;
|
|
1756
1793
|
for (const part of msg.parts) {
|
|
1757
1794
|
if (part?.type !== "tool") continue;
|
|
1758
1795
|
if (part.state?.status !== "completed") continue;
|
|
1759
1796
|
const sig = toolSignature(part);
|
|
1760
|
-
if (!sig
|
|
1797
|
+
if (!sig) continue;
|
|
1798
|
+
const seen2 = seenSignatures.has(sig);
|
|
1761
1799
|
seenSignatures.add(sig);
|
|
1762
|
-
|
|
1800
|
+
if (seen2) continue;
|
|
1801
|
+
if (i >= floor) continue;
|
|
1802
|
+
added += partSize(part);
|
|
1763
1803
|
}
|
|
1804
|
+
if (added === 0) continue;
|
|
1805
|
+
added = (msg.parts ?? []).reduce(
|
|
1806
|
+
(n, p) => n + partSize(p),
|
|
1807
|
+
0
|
|
1808
|
+
);
|
|
1809
|
+
if (kept + added > distinctBudget) break;
|
|
1810
|
+
kept += added;
|
|
1811
|
+
keepToolMessageIds.add(msg.info?.id ?? String(i));
|
|
1764
1812
|
}
|
|
1765
1813
|
}
|
|
1766
1814
|
const stats = {
|
package/dist/profile.d.ts
CHANGED
|
@@ -55,6 +55,7 @@ export type Profile = {
|
|
|
55
55
|
keepRecentTurns?: number;
|
|
56
56
|
keepUserMessages?: boolean;
|
|
57
57
|
keepDistinctOutputs?: boolean;
|
|
58
|
+
keepDistinctBudgetChars?: number | null;
|
|
58
59
|
};
|
|
59
60
|
fallback?: string[];
|
|
60
61
|
};
|
|
@@ -88,7 +89,7 @@ export type RetentionRecord = {
|
|
|
88
89
|
* @property {Record<string, RoleModel>} [roleModels]
|
|
89
90
|
* @property {{ enabled?: boolean, keepRecentToolCalls?: number,
|
|
90
91
|
* keepRecentTurns?: number, keepUserMessages?: boolean,
|
|
91
|
-
* keepDistinctOutputs?: boolean }} [prune]
|
|
92
|
+
* keepDistinctOutputs?: boolean, keepDistinctBudgetChars?: number | null }} [prune]
|
|
92
93
|
* @property {string[]} [fallback]
|
|
93
94
|
* @typedef {{ retention: Retention, note?: string, verified?: string, source?: string }} RetentionRecord
|
|
94
95
|
*/
|
package/docs/KNOWN-LIMITS.md
CHANGED
|
@@ -373,12 +373,32 @@ RECENT calls was a window on recency, and a long session walks out of it: after
|
|
|
373
373
|
six more tool calls, the only copy of a file the agent had read was gone, and
|
|
374
374
|
the agent read it again. The long-task A/B measured seventeen re-reads, five
|
|
375
375
|
times the input tokens and the same final context. The rule now keeps the
|
|
376
|
-
newest copy of each distinct output
|
|
377
|
-
REPEATS of it. The context grows by the size of the distinct outputs, and the
|
|
376
|
+
newest copy of each distinct output, and stubs only the REPEATS of it. The
|
|
378
377
|
re-reads stop: **-69% input tokens and -60% wall time** on that task, at the
|
|
379
378
|
cost of a bigger context. It is a setting - `prune.keepDistinctOutputs` - and
|
|
380
379
|
turning it off brings the old behaviour back.
|
|
381
380
|
|
|
381
|
+
**And the distinct outputs have a ceiling.** The rule as it first shipped had
|
|
382
|
+
no upper bound, which made it a leak rather than a trade-off: it kept every
|
|
383
|
+
output the session had ever produced, and a long session grew without limit.
|
|
384
|
+
Measured on a real 137-message session on 2026-10-22: the request was 5.26 MB,
|
|
385
|
+
the pruner removed 0.04 MB of it - 0.8% - and 4.74 MB of it, 92%, was the
|
|
386
|
+
distinct outputs of the whole session. The profile's own budget for that
|
|
387
|
+
request was 176,000 characters, so the request was twenty-nine times over it.
|
|
388
|
+
|
|
389
|
+
A request that size is not only expensive, it is what makes the server
|
|
390
|
+
slow: opencode persists a full copy of every updated part into its event
|
|
391
|
+
store, so every turn writes megabytes into a database that on that machine had
|
|
392
|
+
reached 4.2 GB, and the TUI - which is a client of that server - started
|
|
393
|
+
dropping keystrokes mid-word.
|
|
394
|
+
|
|
395
|
+
So the distinct outputs are now kept newest-first and only while they fit in
|
|
396
|
+
a budget: `prune.keepDistinctBudgetChars`, unset meaning the profile's own
|
|
397
|
+
context budget, `0` meaning none. The property the rule was added for still
|
|
398
|
+
holds - the copy of a recent output the agent is working with survives, so it
|
|
399
|
+
does not read the file again - and the part that was never wanted, the copy of
|
|
400
|
+
everything the session has ever touched for ever, is gone.
|
|
401
|
+
|
|
382
402
|
`savedChars` in `prune.jsonl` is the size of the request before the pass minus
|
|
383
403
|
the size after it, measured on the parts as they are serialised - not on the
|
|
384
404
|
text inside them. It used to measure the text only, and to ignore the state
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-harness-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",
|
package/profiles/beam-free.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "beam-free",
|
|
7
7
|
"family": "beam",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "retained",
|
|
10
10
|
"notes": "Reflection 501B-A23B, text only, and the freshest knowledge of the free set at 2026-06-30. Measured as the fastest route on the tier: 2.6s average over seven requests. Its provider does not train on prompts but retains them for 30 days, which is why it is `retained` rather than `zero-retention`.",
|
|
11
11
|
"limits": {
|
|
@@ -45,7 +45,6 @@
|
|
|
45
45
|
},
|
|
46
46
|
"fallback": [
|
|
47
47
|
"deepseek-v4.1-flash-free",
|
|
48
|
-
"glm-5.3-flash-free"
|
|
49
|
-
"longcat-2.5-preview-free"
|
|
48
|
+
"glm-5.3-flash-free"
|
|
50
49
|
]
|
|
51
50
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "deepseek-v4.1-flash-free",
|
|
7
7
|
"family": "deepseek-flash",
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"notes": "Reasoning plus agentic coding plus structured output on a zero-retention provider. The natural schema lane. Its knowledge cutoff is 2025-05, the oldest of the tier, so it is not the lane for anything that depends on recent facts.",
|
|
11
11
|
"limits": {
|
|
@@ -43,7 +43,6 @@
|
|
|
43
43
|
},
|
|
44
44
|
"fallback": [
|
|
45
45
|
"glm-5.3-flash-free",
|
|
46
|
-
"longcat-2.5-preview-free",
|
|
47
46
|
"beam-free"
|
|
48
47
|
]
|
|
49
48
|
}
|
package/profiles/fleet-all.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "minimax-m3.1-flash-free",
|
|
7
7
|
"family": null,
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "may-train",
|
|
10
10
|
"maxRetention": "may-train",
|
|
11
11
|
"notes": "Every lane gets the model built for it: beam plans, Nemotron 3 Ultra reviews, Lightning sweeps, MiMo sees, GLM audits, Super VL reasons with the effort turned off. Five of the ten providers state that collected data may be used to improve the model, or log the session, so the profile as a whole is may-train and the generated config says so in its header. Install it for public or throwaway repositories.",
|
|
@@ -52,16 +52,14 @@
|
|
|
52
52
|
"model": "minimax-m3.1-flash-free",
|
|
53
53
|
"chain": [
|
|
54
54
|
"minimax-m3.1-flash-free",
|
|
55
|
-
"deepseek-v4.1-flash-free"
|
|
56
|
-
"longcat-2.5-preview-free"
|
|
55
|
+
"deepseek-v4.1-flash-free"
|
|
57
56
|
]
|
|
58
57
|
},
|
|
59
58
|
"plan": {
|
|
60
59
|
"model": "beam-free",
|
|
61
60
|
"chain": [
|
|
62
61
|
"beam-free",
|
|
63
|
-
"deepseek-v4.1-flash-free"
|
|
64
|
-
"longcat-2.5-preview-free"
|
|
62
|
+
"deepseek-v4.1-flash-free"
|
|
65
63
|
]
|
|
66
64
|
},
|
|
67
65
|
"reviewer": {
|
|
@@ -77,7 +75,6 @@
|
|
|
77
75
|
"chain": [
|
|
78
76
|
"nemotron-3.5-lightning-free",
|
|
79
77
|
"big-pickle",
|
|
80
|
-
"longcat-2.5-preview-free",
|
|
81
78
|
"deepseek-v4.1-flash-free"
|
|
82
79
|
]
|
|
83
80
|
},
|
|
@@ -93,7 +90,6 @@
|
|
|
93
90
|
"model": "mimo-v2.6-flash-free",
|
|
94
91
|
"chain": [
|
|
95
92
|
"mimo-v2.6-flash-free",
|
|
96
|
-
"longcat-2.5-preview-free",
|
|
97
93
|
"minimax-m3.1-flash-free"
|
|
98
94
|
]
|
|
99
95
|
},
|
|
@@ -101,8 +97,7 @@
|
|
|
101
97
|
"model": "glm-5.3-flash-free",
|
|
102
98
|
"chain": [
|
|
103
99
|
"glm-5.3-flash-free",
|
|
104
|
-
"deepseek-v4.1-flash-free"
|
|
105
|
-
"longcat-2.5-preview-free"
|
|
100
|
+
"deepseek-v4.1-flash-free"
|
|
106
101
|
]
|
|
107
102
|
},
|
|
108
103
|
"schema": {
|
|
@@ -117,14 +112,12 @@
|
|
|
117
112
|
"model": "nemotron-3.5-super-vl-free",
|
|
118
113
|
"chain": [
|
|
119
114
|
"nemotron-3.5-super-vl-free",
|
|
120
|
-
"longcat-2.5-preview-free",
|
|
121
115
|
"beam-free"
|
|
122
116
|
]
|
|
123
117
|
},
|
|
124
118
|
"longform": {
|
|
125
|
-
"model": "
|
|
119
|
+
"model": "minimax-m3.1-flash-free",
|
|
126
120
|
"chain": [
|
|
127
|
-
"longcat-2.5-preview-free",
|
|
128
121
|
"minimax-m3.1-flash-free",
|
|
129
122
|
"beam-free"
|
|
130
123
|
]
|
|
@@ -138,7 +131,6 @@
|
|
|
138
131
|
},
|
|
139
132
|
"fallback": [
|
|
140
133
|
"minimax-m3.1-flash-free",
|
|
141
|
-
"deepseek-v4.1-flash-free"
|
|
142
|
-
"longcat-2.5-preview-free"
|
|
134
|
+
"deepseek-v4.1-flash-free"
|
|
143
135
|
]
|
|
144
136
|
}
|
package/profiles/fleet-safe.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"provider": "opencode",
|
|
6
6
|
"model": "minimax-m3.1-flash-free",
|
|
7
7
|
"family": null,
|
|
8
|
-
"verified": "2026-10-
|
|
8
|
+
"verified": "2026-10-22",
|
|
9
9
|
"retention": "zero-retention",
|
|
10
10
|
"maxRetention": "zero-retention",
|
|
11
11
|
"notes": "The default. Every lane resolves to a model whose provider states a zero-retention policy: no training, nothing kept. The cost is that six of the ten jobs run on a second-choice model - the planner loses beam (30-day retention) and the reviewer loses Nemotron 3 Ultra - so this is the floor, not the ceiling. The chains are ordered within the ceiling rather than copied from `fleet-all`: `locate` takes deepseek because it publishes a `low` effort level that longcat does not, and `vision` takes minimax because it reads an image in 7 seconds. `fleet-all` gives every lane the model built for it.",
|
|
@@ -52,31 +52,27 @@
|
|
|
52
52
|
"model": "minimax-m3.1-flash-free",
|
|
53
53
|
"chain": [
|
|
54
54
|
"minimax-m3.1-flash-free",
|
|
55
|
-
"deepseek-v4.1-flash-free"
|
|
56
|
-
"longcat-2.5-preview-free"
|
|
55
|
+
"deepseek-v4.1-flash-free"
|
|
57
56
|
]
|
|
58
57
|
},
|
|
59
58
|
"plan": {
|
|
60
59
|
"model": "deepseek-v4.1-flash-free",
|
|
61
60
|
"chain": [
|
|
62
61
|
"deepseek-v4.1-flash-free",
|
|
63
|
-
"glm-5.3-flash-free"
|
|
64
|
-
"longcat-2.5-preview-free"
|
|
62
|
+
"glm-5.3-flash-free"
|
|
65
63
|
]
|
|
66
64
|
},
|
|
67
65
|
"reviewer": {
|
|
68
66
|
"model": "glm-5.3-flash-free",
|
|
69
67
|
"chain": [
|
|
70
68
|
"glm-5.3-flash-free",
|
|
71
|
-
"deepseek-v4.1-flash-free"
|
|
72
|
-
"longcat-2.5-preview-free"
|
|
69
|
+
"deepseek-v4.1-flash-free"
|
|
73
70
|
]
|
|
74
71
|
},
|
|
75
72
|
"locate": {
|
|
76
73
|
"model": "deepseek-v4.1-flash-free",
|
|
77
74
|
"chain": [
|
|
78
75
|
"deepseek-v4.1-flash-free",
|
|
79
|
-
"longcat-2.5-preview-free",
|
|
80
76
|
"minimax-m3.1-flash-free"
|
|
81
77
|
]
|
|
82
78
|
},
|
|
@@ -84,15 +80,13 @@
|
|
|
84
80
|
"model": "deepseek-v4.1-flash-free",
|
|
85
81
|
"chain": [
|
|
86
82
|
"deepseek-v4.1-flash-free",
|
|
87
|
-
"glm-5.3-flash-free"
|
|
88
|
-
"longcat-2.5-preview-free"
|
|
83
|
+
"glm-5.3-flash-free"
|
|
89
84
|
]
|
|
90
85
|
},
|
|
91
86
|
"vision": {
|
|
92
87
|
"model": "minimax-m3.1-flash-free",
|
|
93
88
|
"chain": [
|
|
94
89
|
"minimax-m3.1-flash-free",
|
|
95
|
-
"longcat-2.5-preview-free",
|
|
96
90
|
"deepseek-v4.1-flash-free"
|
|
97
91
|
]
|
|
98
92
|
},
|
|
@@ -100,30 +94,26 @@
|
|
|
100
94
|
"model": "glm-5.3-flash-free",
|
|
101
95
|
"chain": [
|
|
102
96
|
"glm-5.3-flash-free",
|
|
103
|
-
"deepseek-v4.1-flash-free"
|
|
104
|
-
"longcat-2.5-preview-free"
|
|
97
|
+
"deepseek-v4.1-flash-free"
|
|
105
98
|
]
|
|
106
99
|
},
|
|
107
100
|
"schema": {
|
|
108
101
|
"model": "deepseek-v4.1-flash-free",
|
|
109
102
|
"chain": [
|
|
110
103
|
"deepseek-v4.1-flash-free",
|
|
111
|
-
"glm-5.3-flash-free"
|
|
112
|
-
"longcat-2.5-preview-free"
|
|
104
|
+
"glm-5.3-flash-free"
|
|
113
105
|
]
|
|
114
106
|
},
|
|
115
107
|
"sweep": {
|
|
116
108
|
"model": "deepseek-v4.1-flash-free",
|
|
117
109
|
"chain": [
|
|
118
110
|
"deepseek-v4.1-flash-free",
|
|
119
|
-
"longcat-2.5-preview-free",
|
|
120
111
|
"minimax-m3.1-flash-free"
|
|
121
112
|
]
|
|
122
113
|
},
|
|
123
114
|
"longform": {
|
|
124
|
-
"model": "
|
|
115
|
+
"model": "minimax-m3.1-flash-free",
|
|
125
116
|
"chain": [
|
|
126
|
-
"longcat-2.5-preview-free",
|
|
127
117
|
"minimax-m3.1-flash-free",
|
|
128
118
|
"deepseek-v4.1-flash-free"
|
|
129
119
|
]
|
|
@@ -137,7 +127,6 @@
|
|
|
137
127
|
},
|
|
138
128
|
"fallback": [
|
|
139
129
|
"deepseek-v4.1-flash-free",
|
|
140
|
-
"longcat-2.5-preview-free",
|
|
141
130
|
"glm-5.3-flash-free"
|
|
142
131
|
]
|
|
143
132
|
}
|