opencode-harness-kit 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +207 -17
- package/bin/oc-harness.mjs +278 -45
- package/dist/config.d.ts +5 -0
- package/dist/denylist.d.ts +5 -1
- package/dist/guard.d.ts +17 -1
- package/dist/health.d.ts +69 -7
- package/dist/index.js +452 -68
- package/dist/profile.d.ts +32 -2
- package/dist/telemetry-store.d.ts +22 -0
- package/docs/EXIT-CODES.md +7 -0
- package/docs/KNOWN-LIMITS.md +18 -7
- package/eval/measure-prune.mjs +13 -4
- package/package.json +1 -1
- package/src/config.js +126 -3
- package/src/denylist.js +46 -2
- package/src/guard.ts +78 -7
- package/src/health.js +130 -5
- package/src/lane-usage.js +13 -6
- package/src/profile.js +187 -25
- package/src/prune.ts +58 -13
- package/src/router.ts +217 -17
- package/src/rules.js +329 -19
- package/src/sqlite.js +80 -0
- package/src/tasks.js +193 -40
- package/src/telemetry-store.ts +27 -4
- package/src/trust.js +182 -54
package/src/health.js
CHANGED
|
@@ -15,12 +15,96 @@
|
|
|
15
15
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
16
16
|
import { join } from "node:path"
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* @typedef {object} SpentEntry
|
|
20
|
+
* @property {string} at when the route was recorded as spent
|
|
21
|
+
* @property {string} reason the pattern that matched
|
|
22
|
+
*/
|
|
23
|
+
|
|
18
24
|
/**
|
|
19
25
|
* @typedef {object} Health
|
|
20
|
-
* @property {Record<string,
|
|
26
|
+
* @property {Record<string, SpentEntry>} unhealthy
|
|
21
27
|
* @property {string} [active]
|
|
22
28
|
*/
|
|
23
29
|
|
|
30
|
+
/**
|
|
31
|
+
* How long a recorded spent route is honoured for, by default.
|
|
32
|
+
*
|
|
33
|
+
* The provider's free window resets at 00:00 UTC, so a route recorded as
|
|
34
|
+
* spent is expected to come back within a day. Without an expiry the file is
|
|
35
|
+
* a one-way ratchet: a fleet that hit one rate limit never uses that model
|
|
36
|
+
* again, its lanes drain onto the last rung of their chains, and the only
|
|
37
|
+
* way back is a human editing a JSON file by hand.
|
|
38
|
+
*
|
|
39
|
+
* 24h is the default because the provider's window is a day. The window
|
|
40
|
+
* belongs to the provider rather than to the kit, so it is configurable:
|
|
41
|
+
* HARNESS_SPENT_WINDOW_HOURS overrides it.
|
|
42
|
+
*/
|
|
43
|
+
const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The window, in milliseconds, a recorded spent route is honoured for.
|
|
47
|
+
*
|
|
48
|
+
* Read at call time rather than at import time, so a caller that changes it
|
|
49
|
+
* does not have to reload the module to see the effect.
|
|
50
|
+
*
|
|
51
|
+
* @returns {number}
|
|
52
|
+
*/
|
|
53
|
+
export function spentWindowMs() {
|
|
54
|
+
const raw = process.env.HARNESS_SPENT_WINDOW_HOURS
|
|
55
|
+
if (!raw) return DEFAULT_WINDOW_MS
|
|
56
|
+
const hours = Number(raw)
|
|
57
|
+
// A window of zero would expire every route the moment it was written, and
|
|
58
|
+
// a negative one would expire routes that were never spent. Neither is a
|
|
59
|
+
// setting anybody meant, so the default stands.
|
|
60
|
+
if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WINDOW_MS
|
|
61
|
+
return hours * 60 * 60 * 1000
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Whether a recorded route is still spent.
|
|
66
|
+
*
|
|
67
|
+
* An entry with no usable timestamp is KEPT. The router always stamps one,
|
|
68
|
+
* so an entry without one was written by a hand, and honouring a standing
|
|
69
|
+
* statement is safer than putting a route that may be dead back into a
|
|
70
|
+
* fleet. An entry that cannot be shown to be stale is not dropped.
|
|
71
|
+
*
|
|
72
|
+
* @param {unknown} entry
|
|
73
|
+
* @param {number} [now]
|
|
74
|
+
* @returns {boolean}
|
|
75
|
+
*/
|
|
76
|
+
export function isSpent(entry, now = Date.now()) {
|
|
77
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false
|
|
78
|
+
const at = /** @type {any} */ (entry).at
|
|
79
|
+
if (typeof at !== "string") return true
|
|
80
|
+
const when = Date.parse(at)
|
|
81
|
+
if (Number.isNaN(when)) return true
|
|
82
|
+
return now - when <= spentWindowMs()
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The spent routes that are still spent.
|
|
87
|
+
*
|
|
88
|
+
* Every reader of the file goes through here, so the router and the effort
|
|
89
|
+
* hook cannot disagree about which routes are down. A reader that looked at
|
|
90
|
+
* the raw file would see a route that came back an hour ago as spent, and
|
|
91
|
+
* would fail the fleet over onto a model that works.
|
|
92
|
+
*
|
|
93
|
+
* @param {Health} health
|
|
94
|
+
* @param {number} [now]
|
|
95
|
+
* @returns {Record<string, SpentEntry>}
|
|
96
|
+
*/
|
|
97
|
+
export function liveUnhealthy(health, now = Date.now()) {
|
|
98
|
+
/** @type {Record<string, SpentEntry>} */
|
|
99
|
+
const out = {}
|
|
100
|
+
const source = health?.unhealthy
|
|
101
|
+
if (!source || typeof source !== "object" || Array.isArray(source)) return out
|
|
102
|
+
for (const [key, entry] of Object.entries(source)) {
|
|
103
|
+
if (isSpent(entry, now)) out[key] = entry
|
|
104
|
+
}
|
|
105
|
+
return out
|
|
106
|
+
}
|
|
107
|
+
|
|
24
108
|
/**
|
|
25
109
|
* @param {string} dir the project root
|
|
26
110
|
* @returns {string}
|
|
@@ -30,31 +114,69 @@ export function healthPath(dir) {
|
|
|
30
114
|
}
|
|
31
115
|
|
|
32
116
|
/**
|
|
117
|
+
* The health file, or an empty one.
|
|
118
|
+
*
|
|
119
|
+
* A file that is not an object - null, an array, a number, a string - is not
|
|
120
|
+
* a health file, and reading it as one made `unhealthyModels` throw on a
|
|
121
|
+
* null. That call sits inside a bare catch in src/index.ts, so the throw
|
|
122
|
+
* silently skipped the effort plan AND the write of `.space/active` that the
|
|
123
|
+
* pruner depends on: a corrupt file switched the pruner off without a word.
|
|
124
|
+
*
|
|
33
125
|
* @param {string} dir the project root
|
|
34
126
|
* @returns {Health}
|
|
35
127
|
*/
|
|
36
128
|
export function readHealth(dir) {
|
|
129
|
+
/** @type {any} */
|
|
130
|
+
let parsed
|
|
37
131
|
try {
|
|
38
|
-
|
|
132
|
+
parsed = JSON.parse(readFileSync(healthPath(dir), "utf8"))
|
|
39
133
|
} catch {
|
|
40
134
|
return { unhealthy: {} }
|
|
41
135
|
}
|
|
136
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
137
|
+
return { unhealthy: {} }
|
|
138
|
+
}
|
|
139
|
+
const unhealthy = parsed.unhealthy
|
|
140
|
+
if (!unhealthy || typeof unhealthy !== "object" || Array.isArray(unhealthy)) {
|
|
141
|
+
// The other keys are kept: `active` names the profile the fleet is
|
|
142
|
+
// running, and dropping it because the routes are malformed would lose
|
|
143
|
+
// the pruner's target as well.
|
|
144
|
+
return { ...parsed, unhealthy: {} }
|
|
145
|
+
}
|
|
146
|
+
return parsed
|
|
42
147
|
}
|
|
43
148
|
|
|
44
149
|
/**
|
|
150
|
+
* Write the health file, without the routes whose window has passed.
|
|
151
|
+
*
|
|
152
|
+
* The expiry is applied on the way out as well as on the way in, so a file
|
|
153
|
+
* that is read and written again - which the router does on every event -
|
|
154
|
+
* does not carry its dead routes forever.
|
|
155
|
+
*
|
|
45
156
|
* @param {string} dir the project root
|
|
46
157
|
* @param {Health} health
|
|
47
158
|
*/
|
|
48
159
|
export function writeHealth(dir, health) {
|
|
49
160
|
try {
|
|
50
161
|
if (!existsSync(join(dir, ".space"))) mkdirSync(join(dir, ".space"), { recursive: true })
|
|
51
|
-
|
|
162
|
+
const pruned = { ...(health ?? {}), unhealthy: liveUnhealthy(health) }
|
|
163
|
+
writeFileSync(healthPath(dir), JSON.stringify(pruned, null, 2))
|
|
52
164
|
} catch {
|
|
53
165
|
// Best effort. A health file that cannot be written means no failover,
|
|
54
166
|
// and failing to write it must not break the session.
|
|
55
167
|
}
|
|
56
168
|
}
|
|
57
169
|
|
|
170
|
+
/**
|
|
171
|
+
* The routes on disk that are still spent.
|
|
172
|
+
*
|
|
173
|
+
* @param {string} dir the project root
|
|
174
|
+
* @returns {Record<string, SpentEntry>}
|
|
175
|
+
*/
|
|
176
|
+
export function unhealthyRoutes(dir) {
|
|
177
|
+
return liveUnhealthy(readHealth(dir))
|
|
178
|
+
}
|
|
179
|
+
|
|
58
180
|
/**
|
|
59
181
|
* The models a previous session found spent.
|
|
60
182
|
*
|
|
@@ -67,13 +189,16 @@ export function writeHealth(dir, health) {
|
|
|
67
189
|
* lane chains list bare model ids, so the prefix is dropped here. Comparing
|
|
68
190
|
* the two directly would never match, and a spent route would look healthy.
|
|
69
191
|
*
|
|
192
|
+
* A route whose window has passed is not spent any more, and is not returned
|
|
193
|
+
* here: the provider's quota comes back at 00:00 UTC, and a lane that stays
|
|
194
|
+
* on its fallback forever is a lane that never comes back either.
|
|
195
|
+
*
|
|
70
196
|
* @param {string} dir the project root
|
|
71
197
|
* @returns {Set<string>}
|
|
72
198
|
*/
|
|
73
199
|
export function unhealthyModels(dir) {
|
|
74
|
-
const health = readHealth(dir)
|
|
75
200
|
const out = new Set()
|
|
76
|
-
for (const key of Object.keys(
|
|
201
|
+
for (const key of Object.keys(unhealthyRoutes(dir))) {
|
|
77
202
|
out.add(key.includes("/") ? key.slice(key.indexOf("/") + 1) : key)
|
|
78
203
|
}
|
|
79
204
|
return out
|
package/src/lane-usage.js
CHANGED
|
@@ -34,10 +34,11 @@ import { dirname, join, resolve } from "node:path"
|
|
|
34
34
|
* The telemetry files for the project containing `startDir`.
|
|
35
35
|
*
|
|
36
36
|
* ALL of them, oldest first. The plugin rotates the file when it passes 4 MB
|
|
37
|
-
* and
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
37
|
+
* and keeps THREE generations - `telemetry.jsonl`, `telemetry.jsonl.1` and
|
|
38
|
+
* `telemetry.jsonl.2` - and the reader used to look at `telemetry.jsonl`
|
|
39
|
+
* alone, so a rotation silently threw away the history and every lane
|
|
40
|
+
* reported "never used". The number that decides whether the fleet stays at
|
|
41
|
+
* ten lanes was disappearing on a schedule.
|
|
41
42
|
*
|
|
42
43
|
* @param {string} startDir
|
|
43
44
|
* @returns {string[]} the paths, or an empty array when the project has none
|
|
@@ -62,13 +63,19 @@ export function findTelemetryFiles(startDir) {
|
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
/**
|
|
65
|
-
* The telemetry file for the project containing `startDir`.
|
|
66
|
+
* The CURRENT telemetry file for the project containing `startDir`.
|
|
67
|
+
*
|
|
68
|
+
* The current one, not the oldest one. The files are enumerated oldest
|
|
69
|
+
* first, so the current file is the LAST of them - and taking the first
|
|
70
|
+
* handed back the file that had been rotated away twice, which is the
|
|
71
|
+
* history rather than the present.
|
|
66
72
|
*
|
|
67
73
|
* @param {string} startDir
|
|
68
74
|
* @returns {string | null} the path, or null when the project has no telemetry
|
|
69
75
|
*/
|
|
70
76
|
export function findTelemetryFile(startDir) {
|
|
71
|
-
|
|
77
|
+
const files = findTelemetryFiles(startDir)
|
|
78
|
+
return files.length ? files[files.length - 1] : null
|
|
72
79
|
}
|
|
73
80
|
|
|
74
81
|
/**
|
package/src/profile.js
CHANGED
|
@@ -180,6 +180,16 @@ export function loadProfile(id) {
|
|
|
180
180
|
const msg = err instanceof Error ? err.message : String(err)
|
|
181
181
|
throw new ProfileError(`cannot read ${path}: ${msg}`)
|
|
182
182
|
}
|
|
183
|
+
// The file name IS the id. The generated header tells the user to edit
|
|
184
|
+
// `profiles/<id>.json` and to re-run `oc-harness use <id>`, and both
|
|
185
|
+
// come from this field, so a profile whose id disagrees with its file
|
|
186
|
+
// name sends them to a file that does not exist and to a command that
|
|
187
|
+
// cannot resolve. Nothing else would say so.
|
|
188
|
+
if (raw?.id && raw.id !== id) {
|
|
189
|
+
throw new ProfileError(
|
|
190
|
+
`${path}: profile id is "${raw.id}", which does not match the file name "${id}"`,
|
|
191
|
+
)
|
|
192
|
+
}
|
|
183
193
|
validate(raw, path)
|
|
184
194
|
return normalize(raw)
|
|
185
195
|
}
|
|
@@ -463,25 +473,43 @@ export function resolveLaneChain(p, role) {
|
|
|
463
473
|
* @returns {{ model: string, levels: string[] }}
|
|
464
474
|
*/
|
|
465
475
|
export function resolveLaneModel(p, role, opts = {}) {
|
|
476
|
+
const model = laneModelId(p, role, opts)
|
|
477
|
+
return { model, levels: levelsOfModel(model) }
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* The model a lane runs on, decided without loading any profile.
|
|
482
|
+
*
|
|
483
|
+
* The split is what makes the answer safe to ask from inside `validate()`.
|
|
484
|
+
* Resolving a lane used to mean loading that model's profile to read its
|
|
485
|
+
* effort levels, and loading a profile validates it, and validating it
|
|
486
|
+
* resolves its lanes: a loop. The levels are not part of this question, so
|
|
487
|
+
* they are left out of it.
|
|
488
|
+
*
|
|
489
|
+
* @param {Profile} p
|
|
490
|
+
* @param {string} role
|
|
491
|
+
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
492
|
+
* @returns {string}
|
|
493
|
+
*/
|
|
494
|
+
function laneModelId(p, role, opts = {}) {
|
|
466
495
|
const unhealthy = opts.unhealthy ?? new Set()
|
|
467
496
|
const ceiling = /** @type {Retention} */ (p.maxRetention ?? p.retention)
|
|
468
497
|
for (const candidate of resolveLaneChain(p, role)) {
|
|
469
498
|
if (unhealthy.has(candidate)) continue
|
|
470
499
|
if (!retentionAllowed(candidate, ceiling)) continue
|
|
471
|
-
return
|
|
500
|
+
return candidate
|
|
472
501
|
}
|
|
473
502
|
// Nothing in the chain qualifies. The base model is the next thing to try:
|
|
474
503
|
// it is the profile's own route, and a lane pinned to a rung the router has
|
|
475
504
|
// already recorded as spent is worse than a lane on a route that was never
|
|
476
505
|
// preferred for this job but is alive.
|
|
477
506
|
if (p.model && !unhealthy.has(p.model) && retentionAllowed(p.model, ceiling)) {
|
|
478
|
-
return
|
|
507
|
+
return p.model
|
|
479
508
|
}
|
|
480
509
|
// Last resort: the first rung anyway rather than leaving the lane without a
|
|
481
510
|
// model, and let the config header say what the effective retention turned
|
|
482
511
|
// out to be.
|
|
483
|
-
|
|
484
|
-
return { model: fallback, levels: levelsOfModel(fallback) }
|
|
512
|
+
return resolveLaneChain(p, role)[0] ?? p.model
|
|
485
513
|
}
|
|
486
514
|
|
|
487
515
|
/**
|
|
@@ -578,29 +606,42 @@ export function resolveEffort(levels, requested) {
|
|
|
578
606
|
// no variant" and let the model use its own default.
|
|
579
607
|
if (requested === "none") return null
|
|
580
608
|
|
|
609
|
+
// `none` is not a candidate either, for the same reason: a lane that asked
|
|
610
|
+
// for `minimal` and was handed `none` had its reasoning switched off, which
|
|
611
|
+
// is the opposite of what it asked for. The distance search used to count
|
|
612
|
+
// `none` as the rung below `minimal`, so it won every near miss.
|
|
613
|
+
const rungs = levels.filter(
|
|
614
|
+
(level) => level !== "none" && CANON.indexOf(level) >= 0,
|
|
615
|
+
)
|
|
616
|
+
|
|
581
617
|
const want = CANON.indexOf(requested)
|
|
582
618
|
if (want < 0) {
|
|
583
|
-
// Unknown label. Snap to the top
|
|
619
|
+
// Unknown label. Snap to the top rung, which is the safe reading of a
|
|
584
620
|
// role that asked for something strong.
|
|
585
|
-
return
|
|
621
|
+
return rungs[rungs.length - 1] ?? null
|
|
586
622
|
}
|
|
587
623
|
let best = null
|
|
588
624
|
let bestDist = Infinity
|
|
589
|
-
for (const level of
|
|
625
|
+
for (const level of rungs) {
|
|
590
626
|
const have = CANON.indexOf(level)
|
|
591
|
-
if (have < 0) continue
|
|
592
627
|
const d = Math.abs(have - want)
|
|
593
628
|
if (d < bestDist) {
|
|
594
629
|
bestDist = d
|
|
595
630
|
best = level
|
|
596
631
|
}
|
|
597
632
|
}
|
|
598
|
-
return best ??
|
|
633
|
+
return best ?? rungs[rungs.length - 1] ?? null
|
|
599
634
|
}
|
|
600
635
|
|
|
601
636
|
// --- validation -------------------------------------------------------------
|
|
602
637
|
|
|
603
|
-
|
|
638
|
+
// The retention levels a profile may DECLARE. `unknown` is not one of them:
|
|
639
|
+
// it is the absence of a level, and a profile that says "nobody has stated
|
|
640
|
+
// the terms of this provider" is not a profile. It used to be accepted, and
|
|
641
|
+
// `normalize()` then derived a `maxRetention: "unknown"` from it - a value
|
|
642
|
+
// `validate()` refuses when it is written out by hand, and one the profile
|
|
643
|
+
// schema does not allow for `maxRetention` either.
|
|
644
|
+
const DECLARABLE_RETENTION = ["zero-retention", "retained", "may-train"]
|
|
604
645
|
|
|
605
646
|
/** @param {Profile} p @param {string} [path] */
|
|
606
647
|
export function validate(p, path = "<memory>") {
|
|
@@ -608,25 +649,49 @@ export function validate(p, path = "<memory>") {
|
|
|
608
649
|
if (!p?.id) throw new ProfileError(`${path}: missing "id"`)
|
|
609
650
|
if (!p.provider) throw new ProfileError(`${where}: missing "provider"`)
|
|
610
651
|
if (!p.model) throw new ProfileError(`${where}: missing "model"`)
|
|
611
|
-
if (
|
|
612
|
-
throw new ProfileError(
|
|
613
|
-
`${where}: retention must be one of ${VALID_RETENTION.join(", ")}`,
|
|
614
|
-
)
|
|
652
|
+
if (p.retention === "unknown") {
|
|
653
|
+
throw new ProfileError(`${where}: retention cannot be unknown`)
|
|
615
654
|
}
|
|
616
|
-
if (
|
|
655
|
+
if (!DECLARABLE_RETENTION.includes(p.retention)) {
|
|
617
656
|
throw new ProfileError(
|
|
618
|
-
`${where}:
|
|
657
|
+
`${where}: retention must be one of ${DECLARABLE_RETENTION.join(", ")}`,
|
|
619
658
|
)
|
|
620
659
|
}
|
|
621
660
|
if (p.maxRetention === "unknown") {
|
|
622
661
|
throw new ProfileError(`${where}: maxRetention cannot be unknown`)
|
|
623
662
|
}
|
|
624
|
-
if (
|
|
663
|
+
if (p.maxRetention && !DECLARABLE_RETENTION.includes(p.maxRetention)) {
|
|
664
|
+
throw new ProfileError(
|
|
665
|
+
`${where}: maxRetention must be one of ${DECLARABLE_RETENTION.join(", ")}`,
|
|
666
|
+
)
|
|
667
|
+
}
|
|
668
|
+
// The message said "must be a positive integer" and the check said "greater
|
|
669
|
+
// than zero", so 1.5 and "200000" were accepted. A context of 1.5 makes
|
|
670
|
+
// the prune threshold NEGATIVE, because the reserve has a floor of 8,000
|
|
671
|
+
// tokens and the cap is one and a half.
|
|
672
|
+
if (!Number.isInteger(p.limits?.context) || p.limits.context <= 0) {
|
|
625
673
|
throw new ProfileError(`${where}: limits.context must be a positive integer`)
|
|
626
674
|
}
|
|
627
|
-
if (!p.limits?.output || p.limits.output <= 0) {
|
|
675
|
+
if (!Number.isInteger(p.limits?.output) || p.limits.output <= 0) {
|
|
628
676
|
throw new ProfileError(`${where}: limits.output must be a positive integer`)
|
|
629
677
|
}
|
|
678
|
+
// The levels are written into the config as each lane's `variant`, and
|
|
679
|
+
// opencode cannot resolve a variant the model does not publish. The scale
|
|
680
|
+
// is the canonical one: `none` is reasoning off, and the rest are the
|
|
681
|
+
// reasoning levels.
|
|
682
|
+
for (const level of p.effort?.levels ?? []) {
|
|
683
|
+
if (!CANON.includes(level)) {
|
|
684
|
+
throw new ProfileError(
|
|
685
|
+
`${where}: effort.levels contains "${level}", which is not one of ${CANON.join(", ")}`,
|
|
686
|
+
)
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
const wanted = p.effort?.default
|
|
690
|
+
if (wanted !== undefined && wanted !== null && !CANON.includes(wanted)) {
|
|
691
|
+
throw new ProfileError(
|
|
692
|
+
`${where}: effort.default is "${wanted}", which is not one of ${CANON.join(", ")}`,
|
|
693
|
+
)
|
|
694
|
+
}
|
|
630
695
|
const share = p.effort?.reserveShare
|
|
631
696
|
if (share !== undefined && (share < 0 || share > 0.5)) {
|
|
632
697
|
throw new ProfileError(
|
|
@@ -743,14 +808,25 @@ export function validate(p, path = "<memory>") {
|
|
|
743
808
|
)
|
|
744
809
|
}
|
|
745
810
|
}
|
|
811
|
+
// The base model is the model the config names as the session model, and
|
|
812
|
+
// the one every lane falls back to. It was never checked against
|
|
813
|
+
// retention.yml, so a profile could name a model nobody had stated the
|
|
814
|
+
// terms of and the header would print a level for a model that does not
|
|
815
|
+
// have one.
|
|
816
|
+
if (!retentionFor(p.model)) {
|
|
817
|
+
throw new ProfileError(
|
|
818
|
+
`${where}: model ${p.model} has no retention.yml entry`,
|
|
819
|
+
)
|
|
820
|
+
}
|
|
746
821
|
// The utility route has to be a model this profile can actually run, or a
|
|
747
822
|
// failover leaves opencode's short auxiliary calls pointing at a model that
|
|
748
823
|
// is not in the fleet.
|
|
749
824
|
//
|
|
750
825
|
// The check is STATIC - the union of the profile's own model and every
|
|
751
|
-
// model its chains name
|
|
752
|
-
//
|
|
753
|
-
//
|
|
826
|
+
// model its chains name - and it has to be: the route has to be reachable
|
|
827
|
+
// at all, not reachable today. A chain rung that is not resolving right
|
|
828
|
+
// now is still a route the profile runs when the rung above it is spent,
|
|
829
|
+
// and the utility route is written into the config once and kept.
|
|
754
830
|
if (p.utility) {
|
|
755
831
|
if (typeof p.utility.model !== "string" || !p.utility.model) {
|
|
756
832
|
throw new ProfileError(`${where}: utility.model must be a model id`)
|
|
@@ -765,6 +841,15 @@ export function validate(p, path = "<memory>") {
|
|
|
765
841
|
`${where}: utility.model is ${p.utility.model}, which this profile does not resolve to (it has ${[...reachable].join(", ")})`,
|
|
766
842
|
)
|
|
767
843
|
}
|
|
844
|
+
// The route is written into the config and serves real requests, so its
|
|
845
|
+
// terms have to be on record. A utility route with no retention.yml
|
|
846
|
+
// entry is an `unknown` in the header and a refusal at install time;
|
|
847
|
+
// saying so here names the field instead of leaving a surprise.
|
|
848
|
+
if (!retentionFor(p.utility.model)) {
|
|
849
|
+
throw new ProfileError(
|
|
850
|
+
`${where}: utility.model ${p.utility.model} has no retention.yml entry`,
|
|
851
|
+
)
|
|
852
|
+
}
|
|
768
853
|
}
|
|
769
854
|
|
|
770
855
|
// The fallback queue is walked at startup, and an id that does not resolve
|
|
@@ -777,6 +862,37 @@ export function validate(p, path = "<memory>") {
|
|
|
777
862
|
)
|
|
778
863
|
}
|
|
779
864
|
}
|
|
865
|
+
|
|
866
|
+
// The retention the profile DECLARES may be equal to the worst level
|
|
867
|
+
// among the models its own base and lanes resolve to, or WORSE than that
|
|
868
|
+
// level. It may not be BETTER.
|
|
869
|
+
//
|
|
870
|
+
// The claim was never checked against retention.yml, so a profile could
|
|
871
|
+
// say "zero-retention" while a lane ran on a provider that may train -
|
|
872
|
+
// and the config header printed the claim as fact.
|
|
873
|
+
//
|
|
874
|
+
// The check is by SEVERITY and not by equality. Equality refused the
|
|
875
|
+
// conservative declaration: a profile that says "retained" while every
|
|
876
|
+
// model it runs on is zero-retention was rejected as though it had lied.
|
|
877
|
+
// It had not - it had promised less than the models give, and a promise
|
|
878
|
+
// that is stricter than the truth is the safe direction to err in. Only
|
|
879
|
+
// one direction is a lie, and it is the one the check now refuses.
|
|
880
|
+
//
|
|
881
|
+
// `retentionSeverity` is the same table the ceiling check uses, so the
|
|
882
|
+
// two cannot disagree about which of two levels is the worse one.
|
|
883
|
+
//
|
|
884
|
+
// The utility route is NOT part of this comparison. It is an extra route
|
|
885
|
+
// the profile adds on top of its own models, and it is reported where it
|
|
886
|
+
// belongs: the header names it as the reason the file is worse than the
|
|
887
|
+
// profile claims, and `oc-harness use` warns about it. A profile that
|
|
888
|
+
// names a may-train utility route is therefore installable, and the file
|
|
889
|
+
// it writes says may-train.
|
|
890
|
+
const actual = worstRetention(laneModelsInPlay(p))
|
|
891
|
+
if (retentionSeverity(p.retention) < retentionSeverity(actual.retention)) {
|
|
892
|
+
throw new ProfileError(
|
|
893
|
+
`${where}: retention is ${p.retention}, but ${actual.worst} is ${actual.retention} - a profile cannot be more private than the models it runs on`,
|
|
894
|
+
)
|
|
895
|
+
}
|
|
780
896
|
}
|
|
781
897
|
|
|
782
898
|
/**
|
|
@@ -840,17 +956,53 @@ export function pruneThreshold(p, opts = {}) {
|
|
|
840
956
|
}
|
|
841
957
|
|
|
842
958
|
/**
|
|
843
|
-
* Every model a profile can put behind the user: the base model
|
|
844
|
-
* each lane actually resolves to.
|
|
959
|
+
* Every model a profile can put behind the user: the base model, the model
|
|
960
|
+
* each lane actually resolves to, and the utility route.
|
|
961
|
+
*
|
|
962
|
+
* The utility route belongs in this list because it is WRITTEN into the
|
|
963
|
+
* config as `small_model` and opencode sends real requests down it - a
|
|
964
|
+
* title, a one-line summary, a rewritten question. It used to be a chain
|
|
965
|
+
* rung only, so it never appeared here, and a profile could name a
|
|
966
|
+
* may-train utility route under a zero-retention claim: the file said
|
|
967
|
+
* "zero-retention" while a training provider served the calls.
|
|
968
|
+
*
|
|
969
|
+
* Every caller of this list wants the same thing - the models that can
|
|
970
|
+
* serve a request - so the list is widened rather than a second one
|
|
971
|
+
* invented beside it. The one caller that needs the lanes alone is
|
|
972
|
+
* `validate()`, which cross-checks the profile's DECLARED retention against
|
|
973
|
+
* the base model and the lanes; that is `laneModelsInPlay`.
|
|
845
974
|
*
|
|
846
975
|
* @param {Profile} p
|
|
847
976
|
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
848
977
|
* @returns {string[]}
|
|
849
978
|
*/
|
|
850
979
|
export function modelsInPlay(p, opts = {}) {
|
|
980
|
+
const out = laneModelsInPlay(p, opts)
|
|
981
|
+
const utility = p.utility?.model
|
|
982
|
+
if (typeof utility === "string" && utility && !out.includes(utility)) {
|
|
983
|
+
out.push(utility)
|
|
984
|
+
}
|
|
985
|
+
return out
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* The models the profile's own base and lanes put behind the user, without
|
|
990
|
+
* the utility route.
|
|
991
|
+
*
|
|
992
|
+
* This is the set the profile's own `retention` field is about: the base
|
|
993
|
+
* model and the lanes it resolves to. The utility route is an extra route
|
|
994
|
+
* the profile adds on top of them, and it is reported where it belongs - the
|
|
995
|
+
* config header names it, and `oc-harness use` warns about it - so it does
|
|
996
|
+
* not decide what the profile may declare about itself.
|
|
997
|
+
*
|
|
998
|
+
* @param {Profile} p
|
|
999
|
+
* @param {{ unhealthy?: Set<string> }} [opts]
|
|
1000
|
+
* @returns {string[]}
|
|
1001
|
+
*/
|
|
1002
|
+
export function laneModelsInPlay(p, opts = {}) {
|
|
851
1003
|
const out = [p.model]
|
|
852
1004
|
for (const role of Object.keys(p.roleModels ?? {})) {
|
|
853
|
-
const model =
|
|
1005
|
+
const model = laneModelId(p, role, opts)
|
|
854
1006
|
if (!out.includes(model)) out.push(model)
|
|
855
1007
|
}
|
|
856
1008
|
return out
|
|
@@ -894,10 +1046,20 @@ export function roleSplitModels(p, opts = {}) {
|
|
|
894
1046
|
* @returns {{ retention: string, worst: string | null }}
|
|
895
1047
|
*/
|
|
896
1048
|
export function effectiveRetention(p, opts = {}) {
|
|
1049
|
+
return worstRetention(modelsInPlay(p, opts))
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* The worst retention among a list of models, and the model responsible.
|
|
1054
|
+
*
|
|
1055
|
+
* @param {string[]} ids
|
|
1056
|
+
* @returns {{ retention: string, worst: string | null }}
|
|
1057
|
+
*/
|
|
1058
|
+
function worstRetention(ids) {
|
|
897
1059
|
/** @type {string | null} */
|
|
898
1060
|
let worst = null
|
|
899
1061
|
let retention = "zero-retention"
|
|
900
|
-
for (const id of
|
|
1062
|
+
for (const id of ids) {
|
|
901
1063
|
const declared = modelRetention(id)
|
|
902
1064
|
if (
|
|
903
1065
|
!worst ||
|
package/src/prune.ts
CHANGED
|
@@ -66,7 +66,6 @@ interface StubStats {
|
|
|
66
66
|
bytesAfter: number
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
/** Rough size of a part, used only for the stats line. */
|
|
70
69
|
/**
|
|
71
70
|
* The identity of a tool call's output: the tool and what it was asked for.
|
|
72
71
|
*
|
|
@@ -88,14 +87,37 @@ function toolSignature(part: any): string | null {
|
|
|
88
87
|
return `${tool} ${input}`
|
|
89
88
|
}
|
|
90
89
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
90
|
+
/**
|
|
91
|
+
* The size of a PART, as it sits in the request.
|
|
92
|
+
*
|
|
93
|
+
* The whole part, not the text inside it. The saving this plugin reports is
|
|
94
|
+
* a claim about the REQUEST, and the request is built out of parts - so
|
|
95
|
+
* measuring `part.text` measured a piece of a part and called the difference
|
|
96
|
+
* a saving. The stub replaces a tool result with a new state object that
|
|
97
|
+
* carries its own metadata, and that metadata was invisible to the measure,
|
|
98
|
+
* so a turn whose request GREW was reported as a saving of the difference
|
|
99
|
+
* between two texts. That is the worst bug this plugin can have: a number
|
|
100
|
+
* that says "I saved you 322 characters" for a request that grew by 406.
|
|
101
|
+
*
|
|
102
|
+
* The metadata the stub adds is counted as part of the stub, deliberately:
|
|
103
|
+
* it is bytes the request really does carry, so a stub that costs more than
|
|
104
|
+
* the output it replaced has to show up as a cost rather than as a saving.
|
|
105
|
+
*
|
|
106
|
+
* CHARACTERS, not bytes. A UTF-8 character is more than one byte, and a
|
|
107
|
+
* report that says "bytes" and counts characters is off by a factor on
|
|
108
|
+
* any text that is not ASCII - which is most of the code in a repository.
|
|
109
|
+
* The field is named for what it measures.
|
|
110
|
+
*/
|
|
111
|
+
function partSize(part: any): number {
|
|
112
|
+
try {
|
|
113
|
+
const json = JSON.stringify(part)
|
|
114
|
+
return typeof json === "string" ? json.length : 0
|
|
115
|
+
} catch {
|
|
116
|
+
// A part that cannot be serialised cannot be measured either. It counts
|
|
117
|
+
// as zero rather than throwing: a prune that cannot measure itself must
|
|
118
|
+
// still prune.
|
|
119
|
+
return 0
|
|
120
|
+
}
|
|
99
121
|
}
|
|
100
122
|
|
|
101
123
|
/**
|
|
@@ -261,7 +283,7 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
261
283
|
if (!msg?.parts) continue
|
|
262
284
|
const isUser = msg.info?.role === "user"
|
|
263
285
|
for (const part of msg.parts) {
|
|
264
|
-
const size =
|
|
286
|
+
const size = partSize(part)
|
|
265
287
|
stats.bytesBefore += size
|
|
266
288
|
|
|
267
289
|
if (isUser && keepUsers) {
|
|
@@ -295,8 +317,14 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
295
317
|
// part schema has no signature, so replacing its text is
|
|
296
318
|
// safe. A part that grew one would have to be removed rather
|
|
297
319
|
// than rewritten.
|
|
320
|
+
//
|
|
321
|
+
// There is NO floor under which a reasoning part is left
|
|
322
|
+
// alone. A short one is replaced too, and the stub is about
|
|
323
|
+
// seventy-eight characters - so a reasoning part shorter
|
|
324
|
+
// than that makes the request BIGGER. The log says so,
|
|
325
|
+
// because the log is a measurement and not a claim.
|
|
298
326
|
part.text = REASONING_STUB
|
|
299
|
-
stats.bytesAfter +=
|
|
327
|
+
stats.bytesAfter += partSize(part)
|
|
300
328
|
stats.stubbed++
|
|
301
329
|
break
|
|
302
330
|
}
|
|
@@ -310,6 +338,15 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
310
338
|
stats.kept++
|
|
311
339
|
break
|
|
312
340
|
}
|
|
341
|
+
// The stub is a NEW state object, and the new state carries
|
|
342
|
+
// its own metadata: a flag and a timestamp, about fifty
|
|
343
|
+
// characters per stubbed part.
|
|
344
|
+
//
|
|
345
|
+
// Those characters are part of the stub, and they are
|
|
346
|
+
// measured as part of it. They used to be invisible to the
|
|
347
|
+
// measurement, so the log reported the difference between
|
|
348
|
+
// two TEXTS while the request carried the metadata as well -
|
|
349
|
+
// and a request that grew was reported as a saving.
|
|
313
350
|
part.state = {
|
|
314
351
|
status: "completed",
|
|
315
352
|
input: state.input,
|
|
@@ -318,7 +355,7 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
318
355
|
time: state.time,
|
|
319
356
|
metadata: { ...(state.metadata ?? {}), pruned: true, prunedAt: new Date().toISOString() },
|
|
320
357
|
}
|
|
321
|
-
stats.bytesAfter +=
|
|
358
|
+
stats.bytesAfter += partSize(part)
|
|
322
359
|
stats.stubbed++
|
|
323
360
|
break
|
|
324
361
|
}
|
|
@@ -336,7 +373,15 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
|
336
373
|
}
|
|
337
374
|
|
|
338
375
|
if (stats.stubbed > 0) {
|
|
339
|
-
|
|
376
|
+
// The delta of the REQUEST, exactly, and not floored at zero.
|
|
377
|
+
//
|
|
378
|
+
// The saving is a claim about the request, and a request that grew
|
|
379
|
+
// has to say so. A zero would be read as "nothing was saved" and
|
|
380
|
+
// would hide the fact that the pruner made the request BIGGER -
|
|
381
|
+
// which is the failure this number exists to prevent. The line is
|
|
382
|
+
// still logged either way, because a part was stubbed, and that is
|
|
383
|
+
// the event worth recording.
|
|
384
|
+
const saved = stats.bytesBefore - stats.bytesAfter
|
|
340
385
|
await logPrune(project, profile.id, {
|
|
341
386
|
...stats,
|
|
342
387
|
saved,
|