opencode-harness-kit 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oc-harness.mjs +165 -6
- package/dist/health.d.ts +15 -0
- package/dist/index.js +37 -13
- package/dist/router.d.ts +16 -0
- package/package.json +1 -1
- package/src/health.js +39 -1
- package/src/router.ts +72 -15
package/bin/oc-harness.mjs
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
} from "../src/profile.js"
|
|
46
46
|
import { generateConfig, globalInstructionsPath, utilityModel, withMemoryPlugin, canAddMemoryPlugin, kitVersion, MEMORY_PLUGIN } from "../src/config.js"
|
|
47
47
|
import { ruleNames, BASH_RULES, WRITE_RULES } from "../src/rules.js"
|
|
48
|
+
import { readHealth, unhealthyRoutes, writeHealth } from "../src/health.js"
|
|
48
49
|
import { planEffort } from "../src/effort.js"
|
|
49
50
|
import { findTelemetryFiles, readLaneUsage, averageMs } from "../src/lane-usage.js"
|
|
50
51
|
import { researchModels, isWithdrawn, pruneWithdrawnChains } from "../src/research.js"
|
|
@@ -489,6 +490,46 @@ function projectRoot() {
|
|
|
489
490
|
* one in the plural one, and a checker that only looks at `agents/` misses
|
|
490
491
|
* half of them.
|
|
491
492
|
*/
|
|
493
|
+
/**
|
|
494
|
+
* The health records whose model the catalogue lists again.
|
|
495
|
+
*
|
|
496
|
+
* Issue #9: a route recorded as withdrawn carried the quota's day-long
|
|
497
|
+
* window, so a pulled model was retried once a day, forever. The withdrawn
|
|
498
|
+
* window fixed the expiry half; this is the other half. A record whose
|
|
499
|
+
* model has COME BACK to the catalogue - back on the tier, or no longer
|
|
500
|
+
* flagged as withdrawn in it - describes a route that works again right
|
|
501
|
+
* now, and every hour the record still stands, the profile's config hook
|
|
502
|
+
* benches a live route and sends the lane to a rung down its chain.
|
|
503
|
+
*
|
|
504
|
+
* The clear is conservative in the way the health file itself is: a model
|
|
505
|
+
* that is IN the catalogue and not flagged withdrawn in it is trusted as
|
|
506
|
+
* returned. A model the fetch never saw stays recorded - a network error
|
|
507
|
+
* must not un-bench a route.
|
|
508
|
+
*
|
|
509
|
+
* @param {string} dir the project root the health file lives under
|
|
510
|
+
* @param {any} catalog the live model catalogue, or null
|
|
511
|
+
* @returns {string[]} the route keys that were cleared
|
|
512
|
+
*/
|
|
513
|
+
function clearRecoveredRoutes(dir, catalog) {
|
|
514
|
+
if (!catalog || typeof catalog !== "object") return []
|
|
515
|
+
const routes = unhealthyRoutes(dir)
|
|
516
|
+
const recovered = []
|
|
517
|
+
for (const key of Object.keys(routes)) {
|
|
518
|
+
const slash = key.indexOf("/")
|
|
519
|
+
// A key without its provider half was never written by the router.
|
|
520
|
+
if (slash < 1) continue
|
|
521
|
+
const provider = key.slice(0, slash)
|
|
522
|
+
const model = key.slice(slash + 1)
|
|
523
|
+
const record = catalog?.[provider]?.models?.[model]
|
|
524
|
+
if (record && !isWithdrawn(record)) recovered.push(key)
|
|
525
|
+
}
|
|
526
|
+
if (!recovered.length) return []
|
|
527
|
+
const health = readHealth(dir)
|
|
528
|
+
for (const key of recovered) delete health.unhealthy?.[key]
|
|
529
|
+
writeHealth(dir, health)
|
|
530
|
+
return recovered
|
|
531
|
+
}
|
|
532
|
+
|
|
492
533
|
function agentDirs() {
|
|
493
534
|
const project = projectRoot()
|
|
494
535
|
return [
|
|
@@ -876,6 +917,9 @@ async function cmdAuto() {
|
|
|
876
917
|
try {
|
|
877
918
|
// 1. The catalogue. A model that was withdrawn has to be caught before
|
|
878
919
|
// it is used, and the profile files are the only place that lives.
|
|
920
|
+
// The same fetch feeds the health cleanup and the use step below;
|
|
921
|
+
// offline, it stays null and both of those stand down with it.
|
|
922
|
+
let refreshed = { changed: [], gone: [], same: [], skipped: [], catalog: null }
|
|
879
923
|
if (flag("--offline")) {
|
|
880
924
|
steps.push({ step: "refresh", status: "skipped", detail: "--offline" })
|
|
881
925
|
} else {
|
|
@@ -884,7 +928,7 @@ async function cmdAuto() {
|
|
|
884
928
|
// profile that could not be read used to vanish from the report - and
|
|
885
929
|
// the report said "ok": true while a profile was never checked against
|
|
886
930
|
// the live catalogue.
|
|
887
|
-
|
|
931
|
+
refreshed = await cmdRefreshAll()
|
|
888
932
|
for (const g of refreshed.gone) {
|
|
889
933
|
problems.push(`profile ${g} is no longer in the live catalogue`)
|
|
890
934
|
}
|
|
@@ -901,14 +945,35 @@ async function cmdAuto() {
|
|
|
901
945
|
})
|
|
902
946
|
}
|
|
903
947
|
|
|
948
|
+
// 1.5. The health file. A record whose model the catalogue lists again
|
|
949
|
+
// describes a route that works, and the profile's config hook benches
|
|
950
|
+
// a live route while it stands. This step runs with the same fetch the
|
|
951
|
+
// refresh made, so it costs nothing extra. A `--offline` doctor skips
|
|
952
|
+
// it with the refresh it belongs to: a catalogue that was not read
|
|
953
|
+
// must not be trusted to un-bench anything.
|
|
954
|
+
const recovered = clearRecoveredRoutes(projectRoot(), refreshed.catalog)
|
|
955
|
+
if (recovered.length) {
|
|
956
|
+
for (const key of recovered) console.log(` ${green("+")} ${key} is in the live catalogue again; the spent record was cleared`)
|
|
957
|
+
steps.push({
|
|
958
|
+
step: "health",
|
|
959
|
+
status: "ok",
|
|
960
|
+
detail: `${recovered.length} spent record(s) cleared`,
|
|
961
|
+
})
|
|
962
|
+
}
|
|
963
|
+
|
|
904
964
|
// 2. The behavioural layer: the plugin link, the prompts, the skills, the
|
|
905
965
|
// global rules. Without them the config means nothing.
|
|
906
966
|
await cmdInstall()
|
|
907
967
|
steps.push({ step: "install", status: "ok" })
|
|
908
968
|
|
|
909
|
-
// 3. The config, recompiled from the profile
|
|
910
|
-
|
|
911
|
-
|
|
969
|
+
// 3. The config, recompiled from the profile - against the SAME catalogue
|
|
970
|
+
// fetch, so the step does not pull models.dev a second time.
|
|
971
|
+
const used = await cmdUse(id, refreshed.catalog, { wantJson, report: { problems } })
|
|
972
|
+
if (used === "refused") {
|
|
973
|
+
steps.push({ step: "use", status: "failed", detail: id })
|
|
974
|
+
} else {
|
|
975
|
+
steps.push({ step: "use", status: "ok", detail: id })
|
|
976
|
+
}
|
|
912
977
|
|
|
913
978
|
// 4. The check that says whether all of that worked.
|
|
914
979
|
const verifyProblems = cmdVerify()
|
|
@@ -1452,7 +1517,28 @@ function memorySpec() {
|
|
|
1452
1517
|
return MEMORY_PLUGIN
|
|
1453
1518
|
}
|
|
1454
1519
|
|
|
1455
|
-
|
|
1520
|
+
/**
|
|
1521
|
+
* Install a profile's config.
|
|
1522
|
+
*
|
|
1523
|
+
* @param {string} id
|
|
1524
|
+
* @param {any} [catalog] a catalogue an earlier step already fetched
|
|
1525
|
+
* @param {{ wantJson?: boolean, report?: { problems: string[] } | null }} [sink]
|
|
1526
|
+
* where a refusal goes instead of `fail()`. The one caller that runs with
|
|
1527
|
+
* an calmed stdout is `auto --json`: a script reads its report, the report
|
|
1528
|
+
* names the refusal as a problem, and the run still gets to print the
|
|
1529
|
+
* report. Channels an exit here and the report is never printed, which is
|
|
1530
|
+
* exactly the bug the JSON report exists to prevent.
|
|
1531
|
+
*/
|
|
1532
|
+
async function cmdUse(id, catalog = null, sink = {}) {
|
|
1533
|
+
const wantJson = sink.wantJson === true
|
|
1534
|
+
/** Refuse: issue the message the way the caller's stdout wants it said. */
|
|
1535
|
+
const refuse = (msg) => {
|
|
1536
|
+
if (sink.report) {
|
|
1537
|
+
sink.report.problems.push(msg)
|
|
1538
|
+
return false
|
|
1539
|
+
}
|
|
1540
|
+
fail(msg, wantJson)
|
|
1541
|
+
}
|
|
1456
1542
|
if (!id) fail("usage: oc-harness use <profile> [--print] [--yes]")
|
|
1457
1543
|
let p = null
|
|
1458
1544
|
try {
|
|
@@ -1507,6 +1593,76 @@ async function cmdUse(id) {
|
|
|
1507
1593
|
console.error(msg + " Continuing: there is no terminal to ask.\n")
|
|
1508
1594
|
}
|
|
1509
1595
|
|
|
1596
|
+
// The live catalogue, checked at install time because the profile that gets
|
|
1597
|
+
// installed is a promise about what will serve tomorrow.
|
|
1598
|
+
//
|
|
1599
|
+
// Issue #11: `use` used to install a profile without reading the catalogue
|
|
1600
|
+
// at all, so the first run of the harness was the first thing that knew a
|
|
1601
|
+
// lane's model was gone. The profile's file was written, the header said
|
|
1602
|
+
// every lane was healthy, and the harness said so by failing.
|
|
1603
|
+
//
|
|
1604
|
+
// - A dead BASE model ends the install: the profile has nothing to fall
|
|
1605
|
+
// back to, and writing a config that fails on the session's first token
|
|
1606
|
+
// is the bug, not the fix.
|
|
1607
|
+
// - Dead rungs are pruned IN MEMORY before the config is compiled, so the
|
|
1608
|
+
// written config runs on what the catalogue still serves. The profile
|
|
1609
|
+
// file on disk keeps the rung until `oc-harness refresh` makes the fix
|
|
1610
|
+
// durable - an install must not rewrite the package's own data.
|
|
1611
|
+
// - A lane with no rung left fails the install: a lane with no model is
|
|
1612
|
+
// not a lane, and the profile's author has to name the replacement.
|
|
1613
|
+
//
|
|
1614
|
+
// Fail-open: a catalogue that cannot be read is a warning, not a refusal.
|
|
1615
|
+
// Refusing installs on a flaky network takes the tool away from the
|
|
1616
|
+
// machine that needs to build offline, and `--offline` says that out loud.
|
|
1617
|
+
if (!catalog && !flag("--offline")) {
|
|
1618
|
+
try {
|
|
1619
|
+
catalog = await fetchCatalog()
|
|
1620
|
+
} catch (err) {
|
|
1621
|
+
console.error(
|
|
1622
|
+
yellow(
|
|
1623
|
+
` could not check the live catalogue (${err instanceof Error ? err.message : String(err)}). Install continues with the profile as it ships.`,
|
|
1624
|
+
),
|
|
1625
|
+
)
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
if (catalog) {
|
|
1629
|
+
const models = catalog?.[p.provider]?.models ?? {}
|
|
1630
|
+
const record = models[p.model]
|
|
1631
|
+
if (!record || isWithdrawn(record)) {
|
|
1632
|
+
if (!refuse(
|
|
1633
|
+
`profile ${id}'s base model ${p.provider}/${p.model} is no longer in the live catalogue.` +
|
|
1634
|
+
` Pick another profile: oc-harness list <no-args>`,
|
|
1635
|
+
)) return "refused"
|
|
1636
|
+
}
|
|
1637
|
+
// The prune is silent until it has something to say. Its costs are the
|
|
1638
|
+
// output below: the rungs it dropped and the one route that moved, and
|
|
1639
|
+
// the way to make both durable.
|
|
1640
|
+
const prune = pruneWithdrawnChains(p, catalog, p.provider)
|
|
1641
|
+
if (prune.emptied.length) {
|
|
1642
|
+
const lanes = prune.emptied.map((lane) => {
|
|
1643
|
+
const rm = prune.lanes.find((l) => l.lane === lane)
|
|
1644
|
+
return `${lane} (its models were: ${rm?.removed.join(", ")})`
|
|
1645
|
+
})
|
|
1646
|
+
if (!refuse(
|
|
1647
|
+
`profile ${id}: every rung of the ${prune.emptied.join(", ")} lane is no longer in the live catalogue.` +
|
|
1648
|
+
` No model can serve that lane, so the profile needs a replacement before it installs.`,
|
|
1649
|
+
)) return "refused"
|
|
1650
|
+
}
|
|
1651
|
+
if (prune.pruned.length || prune.utilityMoved) {
|
|
1652
|
+
const drops = []
|
|
1653
|
+
for (const lane of prune.lanes) drops.push(`lane ${lane.lane}: ${lane.from.join(" -> ")} becomes ${lane.to.join(" -> ")}`)
|
|
1654
|
+
if (prune.fallback) drops.push(`fallback: ${prune.fallback.removed.join(", ")} dropped`)
|
|
1655
|
+
if (prune.utilityMoved) drops.push(`the utility route moved to ${p.utility.model}`)
|
|
1656
|
+
console.error(
|
|
1657
|
+
yellow(
|
|
1658
|
+
` the live catalogue no longer serves ${prune.pruned.join(", ")}: the config this run compiled drops them.` +
|
|
1659
|
+
` Run oc-harness refresh to make the fix durable in the profile file itself.`,
|
|
1660
|
+
) +
|
|
1661
|
+
`\n ${drops.join("\n ")}`,
|
|
1662
|
+
)
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1510
1666
|
const base = readConfigJson()
|
|
1511
1667
|
// `--memory` turns on the memory plugin. The kit does not implement memory
|
|
1512
1668
|
// - a local vector store with embeddings, a web UI and auto-capture is a
|
|
@@ -1833,7 +1989,10 @@ async function cmdRefreshAll() {
|
|
|
1833
1989
|
// dropped, so a caller that silences the human output - `auto --json` -
|
|
1834
1990
|
// never saw them: a withdrawn model or a profile that was never checked
|
|
1835
1991
|
// against the catalogue went into a report that said everything was fine.
|
|
1836
|
-
|
|
1992
|
+
// The catalogue itself comes back with the report: `doctor` runs the
|
|
1993
|
+
// refresh as its opening step, and the steps that follow it check against
|
|
1994
|
+
// the same fetch instead of pulling models.dev twice.
|
|
1995
|
+
return { changed, gone, same, skipped, catalog }
|
|
1837
1996
|
}
|
|
1838
1997
|
|
|
1839
1998
|
/**
|
package/dist/health.d.ts
CHANGED
|
@@ -7,6 +7,13 @@ export type SpentEntry = {
|
|
|
7
7
|
* the pattern that matched
|
|
8
8
|
*/
|
|
9
9
|
reason: string;
|
|
10
|
+
/**
|
|
11
|
+
* why the route was written off.
|
|
12
|
+
* `withdrawn` entries expire on the catalogue's clock, not the quota's:
|
|
13
|
+
* a rate limit resets at 00:00 UTC, a pulled model does not. Entries
|
|
14
|
+
* written before the field existed behave as `spent`.
|
|
15
|
+
*/
|
|
16
|
+
class?: ("spent" | "withdrawn");
|
|
10
17
|
};
|
|
11
18
|
export type Health = {
|
|
12
19
|
unhealthy: Record<string, SpentEntry>;
|
|
@@ -21,6 +28,14 @@ export type Health = {
|
|
|
21
28
|
* @returns {number}
|
|
22
29
|
*/
|
|
23
30
|
export declare function spentWindowMs(): number;
|
|
31
|
+
/**
|
|
32
|
+
* The window, in milliseconds, a route recorded as withdrawn is honoured for.
|
|
33
|
+
*
|
|
34
|
+
* Read at call time, like `spentWindowMs`.
|
|
35
|
+
*
|
|
36
|
+
* @returns {number}
|
|
37
|
+
*/
|
|
38
|
+
export declare function withdrawnWindowMs(): number;
|
|
24
39
|
/**
|
|
25
40
|
* Whether a recorded route is still spent.
|
|
26
41
|
*
|
package/dist/index.js
CHANGED
|
@@ -1935,6 +1935,7 @@ function utilityModel(p) {
|
|
|
1935
1935
|
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync3, renameSync as renameSync2, unlinkSync, writeFileSync } from "node:fs";
|
|
1936
1936
|
import { join as join7 } from "node:path";
|
|
1937
1937
|
var DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1e3;
|
|
1938
|
+
var DEFAULT_WITHDRAWN_WINDOW_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
1938
1939
|
function spentWindowMs() {
|
|
1939
1940
|
const raw = process.env.HARNESS_SPENT_WINDOW_HOURS;
|
|
1940
1941
|
if (!raw) return DEFAULT_WINDOW_MS;
|
|
@@ -1942,6 +1943,13 @@ function spentWindowMs() {
|
|
|
1942
1943
|
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WINDOW_MS;
|
|
1943
1944
|
return hours * 60 * 60 * 1e3;
|
|
1944
1945
|
}
|
|
1946
|
+
function withdrawnWindowMs() {
|
|
1947
|
+
const raw = process.env.HARNESS_WITHDRAWN_WINDOW_HOURS;
|
|
1948
|
+
if (!raw) return DEFAULT_WITHDRAWN_WINDOW_MS;
|
|
1949
|
+
const hours = Number(raw);
|
|
1950
|
+
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WITHDRAWN_WINDOW_MS;
|
|
1951
|
+
return hours * 60 * 60 * 1e3;
|
|
1952
|
+
}
|
|
1945
1953
|
function isSpent(entry, now = Date.now()) {
|
|
1946
1954
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
|
|
1947
1955
|
const at = (
|
|
@@ -1951,7 +1959,11 @@ function isSpent(entry, now = Date.now()) {
|
|
|
1951
1959
|
if (typeof at !== "string") return true;
|
|
1952
1960
|
const when = Date.parse(at);
|
|
1953
1961
|
if (Number.isNaN(when)) return true;
|
|
1954
|
-
|
|
1962
|
+
const kind = (
|
|
1963
|
+
/** @type {any} */
|
|
1964
|
+
entry.class
|
|
1965
|
+
);
|
|
1966
|
+
return now - when <= (kind === "withdrawn" ? withdrawnWindowMs() : spentWindowMs());
|
|
1955
1967
|
}
|
|
1956
1968
|
function liveUnhealthy(health, now = Date.now()) {
|
|
1957
1969
|
const out = {};
|
|
@@ -2018,18 +2030,18 @@ var SPENT = [
|
|
|
2018
2030
|
/rate[_ -]?limit/i,
|
|
2019
2031
|
/\b429\b/,
|
|
2020
2032
|
/quota (?:exceeded|exhausted)/i,
|
|
2021
|
-
/overloaded_error/i
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
// "ModelNotFound" - because the text this runs against is the provider's
|
|
2025
|
-
// own error and that can be a whole response body. The old pattern ran
|
|
2026
|
-
// over the entire event, where a body that mentioned a model in one place
|
|
2027
|
-
// and "not found" in another read as a withdrawn model: a live route
|
|
2028
|
-
// written off by a coincidence.
|
|
2033
|
+
/overloaded_error/i
|
|
2034
|
+
];
|
|
2035
|
+
var WITHDRAWN = [
|
|
2029
2036
|
/\bmodel[\W_]*not[\W_]*found\b/i,
|
|
2030
2037
|
/\bmodel\b[\w.:/-]*(?:\s+[\w.:/-]+){0,2}\s+not found\b/i,
|
|
2031
2038
|
/no available provider/i
|
|
2032
2039
|
];
|
|
2040
|
+
var TRANSIENT = [
|
|
2041
|
+
/\b502\b|\b503\b|\b529\b/,
|
|
2042
|
+
/bad gateway/i,
|
|
2043
|
+
/\bupstream\b/
|
|
2044
|
+
];
|
|
2033
2045
|
function errorText(err) {
|
|
2034
2046
|
if (err == null) return "";
|
|
2035
2047
|
if (typeof err === "string") return err;
|
|
@@ -2059,7 +2071,12 @@ function errorText(err) {
|
|
|
2059
2071
|
function classifyError(err) {
|
|
2060
2072
|
const text = errorText(err);
|
|
2061
2073
|
if (!text) return null;
|
|
2062
|
-
for (const p of SPENT)
|
|
2074
|
+
for (const p of SPENT)
|
|
2075
|
+
if (p.test(text)) return { kind: "spent", reason: p.source };
|
|
2076
|
+
for (const p of WITHDRAWN)
|
|
2077
|
+
if (p.test(text)) return { kind: "withdrawn", reason: p.source };
|
|
2078
|
+
for (const p of TRANSIENT)
|
|
2079
|
+
if (p.test(text)) return { kind: "transient", reason: p.source };
|
|
2063
2080
|
return null;
|
|
2064
2081
|
}
|
|
2065
2082
|
function errorOf(e) {
|
|
@@ -2175,14 +2192,21 @@ var RouterPlugin = async ({ worktree, directory }) => {
|
|
|
2175
2192
|
}
|
|
2176
2193
|
const err = errorOf(e);
|
|
2177
2194
|
if (!err) return;
|
|
2178
|
-
const
|
|
2179
|
-
if (!
|
|
2195
|
+
const verdict = classifyError(err);
|
|
2196
|
+
if (!verdict) return;
|
|
2180
2197
|
const sessionID = sessionIdOf(e);
|
|
2181
2198
|
const key = seen ?? (sessionID ? sessionRoutes.get(sessionID) : void 0) ?? configuredRoute ?? lastRoute;
|
|
2182
2199
|
if (!key) return;
|
|
2200
|
+
if (verdict.kind === "transient") return;
|
|
2183
2201
|
const health = readHealth(project);
|
|
2184
2202
|
if (unhealthyRoutes(project)[key]) return;
|
|
2185
|
-
health.unhealthy[key] = {
|
|
2203
|
+
health.unhealthy[key] = {
|
|
2204
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2205
|
+
reason: verdict.reason,
|
|
2206
|
+
// The class decides the window the entry expires under. A model the
|
|
2207
|
+
// provider pulled is not back at 00:00 UTC, whichever the quota says.
|
|
2208
|
+
class: verdict.kind
|
|
2209
|
+
};
|
|
2186
2210
|
writeHealth(project, health);
|
|
2187
2211
|
failedSessions++;
|
|
2188
2212
|
},
|
package/dist/router.d.ts
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
1
|
import type { Plugin } from "./plugin.js";
|
|
2
|
+
/**
|
|
3
|
+
* Which of the three classes a provider error belongs to.
|
|
4
|
+
*
|
|
5
|
+
* The classes decide what happens to the health file: `spent` is recorded
|
|
6
|
+
* under the quota's window, `withdrawn` under the catalogue's, and
|
|
7
|
+
* `transient` is not recorded at all. Exported so a test can feed an error
|
|
8
|
+
* straight through the classifier instead of inventing an event to carry
|
|
9
|
+
* it.
|
|
10
|
+
*
|
|
11
|
+
* @param {any} err
|
|
12
|
+
* @returns {{ kind: "spent" | "withdrawn" | "transient", reason: string } | null}
|
|
13
|
+
*/
|
|
14
|
+
export declare function classifyError(err: any): {
|
|
15
|
+
kind: "spent" | "withdrawn" | "transient";
|
|
16
|
+
reason: string;
|
|
17
|
+
} | null;
|
|
2
18
|
export declare const RouterPlugin: Plugin;
|
|
3
19
|
export default RouterPlugin;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-harness-kit",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "Harness kit for OpenCode: capability profiles, a fleet of job lanes with one free model each, role-based reasoning effort, context pruning, and measured baselines.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/pham-canh/opencode-harness-kit#readme",
|
package/src/health.js
CHANGED
|
@@ -19,6 +19,10 @@ import { join } from "node:path"
|
|
|
19
19
|
* @typedef {object} SpentEntry
|
|
20
20
|
* @property {string} at when the route was recorded as spent
|
|
21
21
|
* @property {string} reason the pattern that matched
|
|
22
|
+
* @property {("spent"|"withdrawn")} [class] why the route was written off.
|
|
23
|
+
* `withdrawn` entries expire on the catalogue's clock, not the quota's:
|
|
24
|
+
* a rate limit resets at 00:00 UTC, a pulled model does not. Entries
|
|
25
|
+
* written before the field existed behave as `spent`.
|
|
22
26
|
*/
|
|
23
27
|
|
|
24
28
|
/**
|
|
@@ -42,6 +46,20 @@ import { join } from "node:path"
|
|
|
42
46
|
*/
|
|
43
47
|
const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000
|
|
44
48
|
|
|
49
|
+
/**
|
|
50
|
+
* How long a route recorded as WITHDRAWN is honoured for, by default.
|
|
51
|
+
*
|
|
52
|
+
* A rate limit resets at 00:00 UTC. A withdrawal does not: the provider
|
|
53
|
+
* pulled the model, and no number of days' sleep brings it back. Treating
|
|
54
|
+
* both with the one window meant a model that had left the catalogue was
|
|
55
|
+
* retried once a day, forever, and every lane that named it first paid a
|
|
56
|
+
* failed request before falling through.
|
|
57
|
+
*
|
|
58
|
+
* Thirty days is the scale the toolkit's catalogue turns on - free tiers
|
|
59
|
+
* rotate monthly - and HARNESS_WITHDRAWN_WINDOW_HOURS overrides it.
|
|
60
|
+
*/
|
|
61
|
+
const DEFAULT_WITHDRAWN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000
|
|
62
|
+
|
|
45
63
|
/**
|
|
46
64
|
* The window, in milliseconds, a recorded spent route is honoured for.
|
|
47
65
|
*
|
|
@@ -61,6 +79,21 @@ export function spentWindowMs() {
|
|
|
61
79
|
return hours * 60 * 60 * 1000
|
|
62
80
|
}
|
|
63
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The window, in milliseconds, a route recorded as withdrawn is honoured for.
|
|
84
|
+
*
|
|
85
|
+
* Read at call time, like `spentWindowMs`.
|
|
86
|
+
*
|
|
87
|
+
* @returns {number}
|
|
88
|
+
*/
|
|
89
|
+
export function withdrawnWindowMs() {
|
|
90
|
+
const raw = process.env.HARNESS_WITHDRAWN_WINDOW_HOURS
|
|
91
|
+
if (!raw) return DEFAULT_WITHDRAWN_WINDOW_MS
|
|
92
|
+
const hours = Number(raw)
|
|
93
|
+
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WITHDRAWN_WINDOW_MS
|
|
94
|
+
return hours * 60 * 60 * 1000
|
|
95
|
+
}
|
|
96
|
+
|
|
64
97
|
/**
|
|
65
98
|
* Whether a recorded route is still spent.
|
|
66
99
|
*
|
|
@@ -79,7 +112,12 @@ export function isSpent(entry, now = Date.now()) {
|
|
|
79
112
|
if (typeof at !== "string") return true
|
|
80
113
|
const when = Date.parse(at)
|
|
81
114
|
if (Number.isNaN(when)) return true
|
|
82
|
-
|
|
115
|
+
// The window the entry was written under. An entry that carries no class
|
|
116
|
+
// was written by an older version of the kit, or by a hand, and gets the
|
|
117
|
+
// quota's window: the quota resets, which is the assumption every entry
|
|
118
|
+
// used to carry.
|
|
119
|
+
const kind = /** @type {any} */ (entry).class
|
|
120
|
+
return now - when <= (kind === "withdrawn" ? withdrawnWindowMs() : spentWindowMs())
|
|
83
121
|
}
|
|
84
122
|
|
|
85
123
|
/**
|
package/src/router.ts
CHANGED
|
@@ -36,25 +36,53 @@ import {
|
|
|
36
36
|
resolveRoles,
|
|
37
37
|
} from "./profile.js"
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Patterns that mean "this route is spent", not "this request was bad": the
|
|
41
|
+
* provider's quota for the free tier ran out. The quota resets, which is why
|
|
42
|
+
* these expire on the day's window.
|
|
43
|
+
*/
|
|
40
44
|
const SPENT = [
|
|
41
45
|
/FreeUsageLimitError/i,
|
|
42
46
|
/rate[_ -]?limit/i,
|
|
43
47
|
/\b429\b/,
|
|
44
48
|
/quota (?:exceeded|exhausted)/i,
|
|
45
49
|
/overloaded_error/i,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Patterns that mean "this model is gone": the provider pulled it.
|
|
54
|
+
*
|
|
55
|
+
* The words have to sit next to each other - "model minimax-m3.1-flash-free
|
|
56
|
+
* not found", "model_not_found", "ModelNotFound" - because the text this
|
|
57
|
+
* runs against is the provider's own error and that can be a whole response
|
|
58
|
+
* body. The old pattern ran over the entire event, where a body that
|
|
59
|
+
* mentioned a model in one place and "not found" in another read as a
|
|
60
|
+
* withdrawn model: a live route written off by a coincidence.
|
|
61
|
+
*
|
|
62
|
+
* These get their own class and their own window: no number of days' sleep
|
|
63
|
+
* returns a pulled model, so benching one for 24h retried a death once a
|
|
64
|
+
* day, forever.
|
|
65
|
+
*/
|
|
66
|
+
const WITHDRAWN = [
|
|
53
67
|
/\bmodel[\W_]*not[\W_]*found\b/i,
|
|
54
68
|
/\bmodel\b[\w.:/-]*(?:\s+[\w.:/-]+){0,2}\s+not found\b/i,
|
|
55
69
|
/no available provider/i,
|
|
56
70
|
]
|
|
57
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Patterns that mean "this request could not be answered" - the host was at
|
|
74
|
+
* capacity, a gateway hiccuped - and nothing about the ROUTE is worse.
|
|
75
|
+
*
|
|
76
|
+
* Benching a route for one of these benches a model that answered minutes
|
|
77
|
+
* later. They are not spent, not withdrawn, and they record nothing: the
|
|
78
|
+
* next request tries the same route again.
|
|
79
|
+
*/
|
|
80
|
+
const TRANSIENT = [
|
|
81
|
+
/\b502\b|\b503\b|\b529\b/,
|
|
82
|
+
/bad gateway/i,
|
|
83
|
+
/\bupstream\b/,
|
|
84
|
+
]
|
|
85
|
+
|
|
58
86
|
/**
|
|
59
87
|
* The text of a provider error, and nothing else.
|
|
60
88
|
*
|
|
@@ -99,15 +127,26 @@ function errorText(err: any): string {
|
|
|
99
127
|
}
|
|
100
128
|
|
|
101
129
|
/**
|
|
102
|
-
*
|
|
130
|
+
* Which of the three classes a provider error belongs to.
|
|
131
|
+
*
|
|
132
|
+
* The classes decide what happens to the health file: `spent` is recorded
|
|
133
|
+
* under the quota's window, `withdrawn` under the catalogue's, and
|
|
134
|
+
* `transient` is not recorded at all. Exported so a test can feed an error
|
|
135
|
+
* straight through the classifier instead of inventing an event to carry
|
|
136
|
+
* it.
|
|
103
137
|
*
|
|
104
138
|
* @param {any} err
|
|
105
|
-
* @returns {
|
|
139
|
+
* @returns {{ kind: "spent" | "withdrawn" | "transient", reason: string } | null}
|
|
106
140
|
*/
|
|
107
|
-
function classifyError(err: any): string | null {
|
|
141
|
+
export function classifyError(err: any): { kind: "spent" | "withdrawn" | "transient"; reason: string } | null {
|
|
108
142
|
const text = errorText(err)
|
|
109
143
|
if (!text) return null
|
|
110
|
-
for (const p of SPENT)
|
|
144
|
+
for (const p of SPENT)
|
|
145
|
+
if (p.test(text)) return { kind: "spent", reason: p.source }
|
|
146
|
+
for (const p of WITHDRAWN)
|
|
147
|
+
if (p.test(text)) return { kind: "withdrawn", reason: p.source }
|
|
148
|
+
for (const p of TRANSIENT)
|
|
149
|
+
if (p.test(text)) return { kind: "transient", reason: p.source }
|
|
111
150
|
return null
|
|
112
151
|
}
|
|
113
152
|
|
|
@@ -352,8 +391,8 @@ export const RouterPlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
352
391
|
// anything about the route.
|
|
353
392
|
const err = errorOf(e)
|
|
354
393
|
if (!err) return
|
|
355
|
-
const
|
|
356
|
-
if (!
|
|
394
|
+
const verdict = classifyError(err)
|
|
395
|
+
if (!verdict) return
|
|
357
396
|
|
|
358
397
|
// The route the failure belongs to: the event's own, else the route
|
|
359
398
|
// this session was last seen on, else the route it was configured
|
|
@@ -367,9 +406,27 @@ export const RouterPlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
367
406
|
lastRoute
|
|
368
407
|
if (!key) return
|
|
369
408
|
|
|
409
|
+
// A transient failure is the host failing to answer one request, not
|
|
410
|
+
// the route becoming unusable. Nothing is written, so the next request
|
|
411
|
+
// tries the same rung again - benching the route under the quota's
|
|
412
|
+
// window is how a capacity blip cost a model a day.
|
|
413
|
+
//
|
|
414
|
+
// ("Advances to the next rung inside the same request" is not on the
|
|
415
|
+
// table: `chat.params` carries only temperature and sampling, the model
|
|
416
|
+
// on `chat.message` is read-only, and `config` has already run. What a
|
|
417
|
+
// transient verdict buys - and what this buys - is a route that is
|
|
418
|
+
// still live when the next request arrives.)
|
|
419
|
+
if (verdict.kind === "transient") return
|
|
420
|
+
|
|
370
421
|
const health = readHealth(project)
|
|
371
422
|
if (unhealthyRoutes(project)[key]) return // already recorded, do not spam
|
|
372
|
-
health.unhealthy[key] = {
|
|
423
|
+
health.unhealthy[key] = {
|
|
424
|
+
at: new Date().toISOString(),
|
|
425
|
+
reason: verdict.reason,
|
|
426
|
+
// The class decides the window the entry expires under. A model the
|
|
427
|
+
// provider pulled is not back at 00:00 UTC, whichever the quota says.
|
|
428
|
+
class: verdict.kind,
|
|
429
|
+
}
|
|
373
430
|
writeHealth(project, health)
|
|
374
431
|
failedSessions++
|
|
375
432
|
},
|