opencode-harness-kit 0.5.4 → 0.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/health.js CHANGED
@@ -15,12 +15,96 @@
15
15
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
16
16
  import { join } from "node:path"
17
17
 
18
+ /**
19
+ * @typedef {object} SpentEntry
20
+ * @property {string} at when the route was recorded as spent
21
+ * @property {string} reason the pattern that matched
22
+ */
23
+
18
24
  /**
19
25
  * @typedef {object} Health
20
- * @property {Record<string, { at: string, reason: string }>} unhealthy
26
+ * @property {Record<string, SpentEntry>} unhealthy
21
27
  * @property {string} [active]
22
28
  */
23
29
 
30
+ /**
31
+ * How long a recorded spent route is honoured for, by default.
32
+ *
33
+ * The provider's free window resets at 00:00 UTC, so a route recorded as
34
+ * spent is expected to come back within a day. Without an expiry the file is
35
+ * a one-way ratchet: a fleet that hit one rate limit never uses that model
36
+ * again, its lanes drain onto the last rung of their chains, and the only
37
+ * way back is a human editing a JSON file by hand.
38
+ *
39
+ * 24h is the default because the provider's window is a day. The window
40
+ * belongs to the provider rather than to the kit, so it is configurable:
41
+ * HARNESS_SPENT_WINDOW_HOURS overrides it.
42
+ */
43
+ const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000
44
+
45
+ /**
46
+ * The window, in milliseconds, a recorded spent route is honoured for.
47
+ *
48
+ * Read at call time rather than at import time, so a caller that changes it
49
+ * does not have to reload the module to see the effect.
50
+ *
51
+ * @returns {number}
52
+ */
53
+ export function spentWindowMs() {
54
+ const raw = process.env.HARNESS_SPENT_WINDOW_HOURS
55
+ if (!raw) return DEFAULT_WINDOW_MS
56
+ const hours = Number(raw)
57
+ // A window of zero would expire every route the moment it was written, and
58
+ // a negative one would expire routes that were never spent. Neither is a
59
+ // setting anybody meant, so the default stands.
60
+ if (!Number.isFinite(hours) || hours <= 0) return DEFAULT_WINDOW_MS
61
+ return hours * 60 * 60 * 1000
62
+ }
63
+
64
+ /**
65
+ * Whether a recorded route is still spent.
66
+ *
67
+ * An entry with no usable timestamp is KEPT. The router always stamps one,
68
+ * so an entry without one was written by a hand, and honouring a standing
69
+ * statement is safer than putting a route that may be dead back into a
70
+ * fleet. An entry that cannot be shown to be stale is not dropped.
71
+ *
72
+ * @param {unknown} entry
73
+ * @param {number} [now]
74
+ * @returns {boolean}
75
+ */
76
+ export function isSpent(entry, now = Date.now()) {
77
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false
78
+ const at = /** @type {any} */ (entry).at
79
+ if (typeof at !== "string") return true
80
+ const when = Date.parse(at)
81
+ if (Number.isNaN(when)) return true
82
+ return now - when <= spentWindowMs()
83
+ }
84
+
85
+ /**
86
+ * The spent routes that are still spent.
87
+ *
88
+ * Every reader of the file goes through here, so the router and the effort
89
+ * hook cannot disagree about which routes are down. A reader that looked at
90
+ * the raw file would see a route that came back an hour ago as spent, and
91
+ * would fail the fleet over onto a model that works.
92
+ *
93
+ * @param {Health} health
94
+ * @param {number} [now]
95
+ * @returns {Record<string, SpentEntry>}
96
+ */
97
+ export function liveUnhealthy(health, now = Date.now()) {
98
+ /** @type {Record<string, SpentEntry>} */
99
+ const out = {}
100
+ const source = health?.unhealthy
101
+ if (!source || typeof source !== "object" || Array.isArray(source)) return out
102
+ for (const [key, entry] of Object.entries(source)) {
103
+ if (isSpent(entry, now)) out[key] = entry
104
+ }
105
+ return out
106
+ }
107
+
24
108
  /**
25
109
  * @param {string} dir the project root
26
110
  * @returns {string}
@@ -30,31 +114,69 @@ export function healthPath(dir) {
30
114
  }
31
115
 
32
116
  /**
117
+ * The health file, or an empty one.
118
+ *
119
+ * A file that is not an object - null, an array, a number, a string - is not
120
+ * a health file, and reading it as one made `unhealthyModels` throw on a
121
+ * null. That call sits inside a bare catch in src/index.ts, so the throw
122
+ * silently skipped the effort plan AND the write of `.space/active` that the
123
+ * pruner depends on: a corrupt file switched the pruner off without a word.
124
+ *
33
125
  * @param {string} dir the project root
34
126
  * @returns {Health}
35
127
  */
36
128
  export function readHealth(dir) {
129
+ /** @type {any} */
130
+ let parsed
37
131
  try {
38
- return JSON.parse(readFileSync(healthPath(dir), "utf8"))
132
+ parsed = JSON.parse(readFileSync(healthPath(dir), "utf8"))
39
133
  } catch {
40
134
  return { unhealthy: {} }
41
135
  }
136
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
137
+ return { unhealthy: {} }
138
+ }
139
+ const unhealthy = parsed.unhealthy
140
+ if (!unhealthy || typeof unhealthy !== "object" || Array.isArray(unhealthy)) {
141
+ // The other keys are kept: `active` names the profile the fleet is
142
+ // running, and dropping it because the routes are malformed would lose
143
+ // the pruner's target as well.
144
+ return { ...parsed, unhealthy: {} }
145
+ }
146
+ return parsed
42
147
  }
43
148
 
44
149
  /**
150
+ * Write the health file, without the routes whose window has passed.
151
+ *
152
+ * The expiry is applied on the way out as well as on the way in, so a file
153
+ * that is read and written again - which the router does on every event -
154
+ * does not carry its dead routes forever.
155
+ *
45
156
  * @param {string} dir the project root
46
157
  * @param {Health} health
47
158
  */
48
159
  export function writeHealth(dir, health) {
49
160
  try {
50
161
  if (!existsSync(join(dir, ".space"))) mkdirSync(join(dir, ".space"), { recursive: true })
51
- writeFileSync(healthPath(dir), JSON.stringify(health, null, 2))
162
+ const pruned = { ...(health ?? {}), unhealthy: liveUnhealthy(health) }
163
+ writeFileSync(healthPath(dir), JSON.stringify(pruned, null, 2))
52
164
  } catch {
53
165
  // Best effort. A health file that cannot be written means no failover,
54
166
  // and failing to write it must not break the session.
55
167
  }
56
168
  }
57
169
 
170
+ /**
171
+ * The routes on disk that are still spent.
172
+ *
173
+ * @param {string} dir the project root
174
+ * @returns {Record<string, SpentEntry>}
175
+ */
176
+ export function unhealthyRoutes(dir) {
177
+ return liveUnhealthy(readHealth(dir))
178
+ }
179
+
58
180
  /**
59
181
  * The models a previous session found spent.
60
182
  *
@@ -67,13 +189,16 @@ export function writeHealth(dir, health) {
67
189
  * lane chains list bare model ids, so the prefix is dropped here. Comparing
68
190
  * the two directly would never match, and a spent route would look healthy.
69
191
  *
192
+ * A route whose window has passed is not spent any more, and is not returned
193
+ * here: the provider's quota comes back at 00:00 UTC, and a lane that stays
194
+ * on its fallback forever is a lane that never comes back either.
195
+ *
70
196
  * @param {string} dir the project root
71
197
  * @returns {Set<string>}
72
198
  */
73
199
  export function unhealthyModels(dir) {
74
- const health = readHealth(dir)
75
200
  const out = new Set()
76
- for (const key of Object.keys(health.unhealthy ?? {})) {
201
+ for (const key of Object.keys(unhealthyRoutes(dir))) {
77
202
  out.add(key.includes("/") ? key.slice(key.indexOf("/") + 1) : key)
78
203
  }
79
204
  return out
package/src/lane-usage.js CHANGED
@@ -34,10 +34,11 @@ import { dirname, join, resolve } from "node:path"
34
34
  * The telemetry files for the project containing `startDir`.
35
35
  *
36
36
  * ALL of them, oldest first. The plugin rotates the file when it passes 4 MB
37
- * and renames the old one to `telemetry.jsonl.1` - and the reader used to
38
- * look at `telemetry.jsonl` alone, so a rotation silently threw away the
39
- * history and every lane reported "never used". The number that decides
40
- * whether the fleet stays at ten lanes was disappearing on a schedule.
37
+ * and keeps THREE generations - `telemetry.jsonl`, `telemetry.jsonl.1` and
38
+ * `telemetry.jsonl.2` - and the reader used to look at `telemetry.jsonl`
39
+ * alone, so a rotation silently threw away the history and every lane
40
+ * reported "never used". The number that decides whether the fleet stays at
41
+ * ten lanes was disappearing on a schedule.
41
42
  *
42
43
  * @param {string} startDir
43
44
  * @returns {string[]} the paths, or an empty array when the project has none
@@ -62,13 +63,19 @@ export function findTelemetryFiles(startDir) {
62
63
  }
63
64
 
64
65
  /**
65
- * The telemetry file for the project containing `startDir`.
66
+ * The CURRENT telemetry file for the project containing `startDir`.
67
+ *
68
+ * The current one, not the oldest one. The files are enumerated oldest
69
+ * first, so the current file is the LAST of them - and taking the first
70
+ * handed back the file that had been rotated away twice, which is the
71
+ * history rather than the present.
66
72
  *
67
73
  * @param {string} startDir
68
74
  * @returns {string | null} the path, or null when the project has no telemetry
69
75
  */
70
76
  export function findTelemetryFile(startDir) {
71
- return findTelemetryFiles(startDir)[0] ?? null
77
+ const files = findTelemetryFiles(startDir)
78
+ return files.length ? files[files.length - 1] : null
72
79
  }
73
80
 
74
81
  /**
package/src/profile.js CHANGED
@@ -180,6 +180,16 @@ export function loadProfile(id) {
180
180
  const msg = err instanceof Error ? err.message : String(err)
181
181
  throw new ProfileError(`cannot read ${path}: ${msg}`)
182
182
  }
183
+ // The file name IS the id. The generated header tells the user to edit
184
+ // `profiles/<id>.json` and to re-run `oc-harness use <id>`, and both
185
+ // come from this field, so a profile whose id disagrees with its file
186
+ // name sends them to a file that does not exist and to a command that
187
+ // cannot resolve. Nothing else would say so.
188
+ if (raw?.id && raw.id !== id) {
189
+ throw new ProfileError(
190
+ `${path}: profile id is "${raw.id}", which does not match the file name "${id}"`,
191
+ )
192
+ }
183
193
  validate(raw, path)
184
194
  return normalize(raw)
185
195
  }
@@ -463,25 +473,43 @@ export function resolveLaneChain(p, role) {
463
473
  * @returns {{ model: string, levels: string[] }}
464
474
  */
465
475
  export function resolveLaneModel(p, role, opts = {}) {
476
+ const model = laneModelId(p, role, opts)
477
+ return { model, levels: levelsOfModel(model) }
478
+ }
479
+
480
+ /**
481
+ * The model a lane runs on, decided without loading any profile.
482
+ *
483
+ * The split is what makes the answer safe to ask from inside `validate()`.
484
+ * Resolving a lane used to mean loading that model's profile to read its
485
+ * effort levels, and loading a profile validates it, and validating it
486
+ * resolves its lanes: a loop. The levels are not part of this question, so
487
+ * they are left out of it.
488
+ *
489
+ * @param {Profile} p
490
+ * @param {string} role
491
+ * @param {{ unhealthy?: Set<string> }} [opts]
492
+ * @returns {string}
493
+ */
494
+ function laneModelId(p, role, opts = {}) {
466
495
  const unhealthy = opts.unhealthy ?? new Set()
467
496
  const ceiling = /** @type {Retention} */ (p.maxRetention ?? p.retention)
468
497
  for (const candidate of resolveLaneChain(p, role)) {
469
498
  if (unhealthy.has(candidate)) continue
470
499
  if (!retentionAllowed(candidate, ceiling)) continue
471
- return { model: candidate, levels: levelsOfModel(candidate) }
500
+ return candidate
472
501
  }
473
502
  // Nothing in the chain qualifies. The base model is the next thing to try:
474
503
  // it is the profile's own route, and a lane pinned to a rung the router has
475
504
  // already recorded as spent is worse than a lane on a route that was never
476
505
  // preferred for this job but is alive.
477
506
  if (p.model && !unhealthy.has(p.model) && retentionAllowed(p.model, ceiling)) {
478
- return { model: p.model, levels: levelsOfModel(p.model) }
507
+ return p.model
479
508
  }
480
509
  // Last resort: the first rung anyway rather than leaving the lane without a
481
510
  // model, and let the config header say what the effective retention turned
482
511
  // out to be.
483
- const fallback = resolveLaneChain(p, role)[0] ?? p.model
484
- return { model: fallback, levels: levelsOfModel(fallback) }
512
+ return resolveLaneChain(p, role)[0] ?? p.model
485
513
  }
486
514
 
487
515
  /**
@@ -578,29 +606,42 @@ export function resolveEffort(levels, requested) {
578
606
  // no variant" and let the model use its own default.
579
607
  if (requested === "none") return null
580
608
 
609
+ // `none` is not a candidate either, for the same reason: a lane that asked
610
+ // for `minimal` and was handed `none` had its reasoning switched off, which
611
+ // is the opposite of what it asked for. The distance search used to count
612
+ // `none` as the rung below `minimal`, so it won every near miss.
613
+ const rungs = levels.filter(
614
+ (level) => level !== "none" && CANON.indexOf(level) >= 0,
615
+ )
616
+
581
617
  const want = CANON.indexOf(requested)
582
618
  if (want < 0) {
583
- // Unknown label. Snap to the top level, which is the safe reading of a
619
+ // Unknown label. Snap to the top rung, which is the safe reading of a
584
620
  // role that asked for something strong.
585
- return levels[levels.length - 1] ?? null
621
+ return rungs[rungs.length - 1] ?? null
586
622
  }
587
623
  let best = null
588
624
  let bestDist = Infinity
589
- for (const level of levels) {
625
+ for (const level of rungs) {
590
626
  const have = CANON.indexOf(level)
591
- if (have < 0) continue
592
627
  const d = Math.abs(have - want)
593
628
  if (d < bestDist) {
594
629
  bestDist = d
595
630
  best = level
596
631
  }
597
632
  }
598
- return best ?? levels[levels.length - 1] ?? null
633
+ return best ?? rungs[rungs.length - 1] ?? null
599
634
  }
600
635
 
601
636
  // --- validation -------------------------------------------------------------
602
637
 
603
- const VALID_RETENTION = ["zero-retention", "retained", "may-train", "unknown"]
638
+ // The retention levels a profile may DECLARE. `unknown` is not one of them:
639
+ // it is the absence of a level, and a profile that says "nobody has stated
640
+ // the terms of this provider" is not a profile. It used to be accepted, and
641
+ // `normalize()` then derived a `maxRetention: "unknown"` from it - a value
642
+ // `validate()` refuses when it is written out by hand, and one the profile
643
+ // schema does not allow for `maxRetention` either.
644
+ const DECLARABLE_RETENTION = ["zero-retention", "retained", "may-train"]
604
645
 
605
646
  /** @param {Profile} p @param {string} [path] */
606
647
  export function validate(p, path = "<memory>") {
@@ -608,25 +649,49 @@ export function validate(p, path = "<memory>") {
608
649
  if (!p?.id) throw new ProfileError(`${path}: missing "id"`)
609
650
  if (!p.provider) throw new ProfileError(`${where}: missing "provider"`)
610
651
  if (!p.model) throw new ProfileError(`${where}: missing "model"`)
611
- if (!VALID_RETENTION.includes(p.retention)) {
612
- throw new ProfileError(
613
- `${where}: retention must be one of ${VALID_RETENTION.join(", ")}`,
614
- )
652
+ if (p.retention === "unknown") {
653
+ throw new ProfileError(`${where}: retention cannot be unknown`)
615
654
  }
616
- if (p.maxRetention && !VALID_RETENTION.includes(p.maxRetention)) {
655
+ if (!DECLARABLE_RETENTION.includes(p.retention)) {
617
656
  throw new ProfileError(
618
- `${where}: maxRetention must be one of ${VALID_RETENTION.join(", ")}`,
657
+ `${where}: retention must be one of ${DECLARABLE_RETENTION.join(", ")}`,
619
658
  )
620
659
  }
621
660
  if (p.maxRetention === "unknown") {
622
661
  throw new ProfileError(`${where}: maxRetention cannot be unknown`)
623
662
  }
624
- if (!p.limits?.context || p.limits.context <= 0) {
663
+ if (p.maxRetention && !DECLARABLE_RETENTION.includes(p.maxRetention)) {
664
+ throw new ProfileError(
665
+ `${where}: maxRetention must be one of ${DECLARABLE_RETENTION.join(", ")}`,
666
+ )
667
+ }
668
+ // The message said "must be a positive integer" and the check said "greater
669
+ // than zero", so 1.5 and "200000" were accepted. A context of 1.5 makes
670
+ // the prune threshold NEGATIVE, because the reserve has a floor of 8,000
671
+ // tokens and the cap is one and a half.
672
+ if (!Number.isInteger(p.limits?.context) || p.limits.context <= 0) {
625
673
  throw new ProfileError(`${where}: limits.context must be a positive integer`)
626
674
  }
627
- if (!p.limits?.output || p.limits.output <= 0) {
675
+ if (!Number.isInteger(p.limits?.output) || p.limits.output <= 0) {
628
676
  throw new ProfileError(`${where}: limits.output must be a positive integer`)
629
677
  }
678
+ // The levels are written into the config as each lane's `variant`, and
679
+ // opencode cannot resolve a variant the model does not publish. The scale
680
+ // is the canonical one: `none` is reasoning off, and the rest are the
681
+ // reasoning levels.
682
+ for (const level of p.effort?.levels ?? []) {
683
+ if (!CANON.includes(level)) {
684
+ throw new ProfileError(
685
+ `${where}: effort.levels contains "${level}", which is not one of ${CANON.join(", ")}`,
686
+ )
687
+ }
688
+ }
689
+ const wanted = p.effort?.default
690
+ if (wanted !== undefined && wanted !== null && !CANON.includes(wanted)) {
691
+ throw new ProfileError(
692
+ `${where}: effort.default is "${wanted}", which is not one of ${CANON.join(", ")}`,
693
+ )
694
+ }
630
695
  const share = p.effort?.reserveShare
631
696
  if (share !== undefined && (share < 0 || share > 0.5)) {
632
697
  throw new ProfileError(
@@ -743,14 +808,25 @@ export function validate(p, path = "<memory>") {
743
808
  )
744
809
  }
745
810
  }
811
+ // The base model is the model the config names as the session model, and
812
+ // the one every lane falls back to. It was never checked against
813
+ // retention.yml, so a profile could name a model nobody had stated the
814
+ // terms of and the header would print a level for a model that does not
815
+ // have one.
816
+ if (!retentionFor(p.model)) {
817
+ throw new ProfileError(
818
+ `${where}: model ${p.model} has no retention.yml entry`,
819
+ )
820
+ }
746
821
  // The utility route has to be a model this profile can actually run, or a
747
822
  // failover leaves opencode's short auxiliary calls pointing at a model that
748
823
  // is not in the fleet.
749
824
  //
750
825
  // The check is STATIC - the union of the profile's own model and every
751
- // model its chains name. Asking `modelsInPlay` would be the obvious way,
752
- // and it is a loop: resolving a lane reads the levels of a model, which
753
- // loads that model's profile, which validates it, which resolves a lane.
826
+ // model its chains name - and it has to be: the route has to be reachable
827
+ // at all, not reachable today. A chain rung that is not resolving right
828
+ // now is still a route the profile runs when the rung above it is spent,
829
+ // and the utility route is written into the config once and kept.
754
830
  if (p.utility) {
755
831
  if (typeof p.utility.model !== "string" || !p.utility.model) {
756
832
  throw new ProfileError(`${where}: utility.model must be a model id`)
@@ -765,6 +841,15 @@ export function validate(p, path = "<memory>") {
765
841
  `${where}: utility.model is ${p.utility.model}, which this profile does not resolve to (it has ${[...reachable].join(", ")})`,
766
842
  )
767
843
  }
844
+ // The route is written into the config and serves real requests, so its
845
+ // terms have to be on record. A utility route with no retention.yml
846
+ // entry is an `unknown` in the header and a refusal at install time;
847
+ // saying so here names the field instead of leaving a surprise.
848
+ if (!retentionFor(p.utility.model)) {
849
+ throw new ProfileError(
850
+ `${where}: utility.model ${p.utility.model} has no retention.yml entry`,
851
+ )
852
+ }
768
853
  }
769
854
 
770
855
  // The fallback queue is walked at startup, and an id that does not resolve
@@ -777,6 +862,37 @@ export function validate(p, path = "<memory>") {
777
862
  )
778
863
  }
779
864
  }
865
+
866
+ // The retention the profile DECLARES may be equal to the worst level
867
+ // among the models its own base and lanes resolve to, or WORSE than that
868
+ // level. It may not be BETTER.
869
+ //
870
+ // The claim was never checked against retention.yml, so a profile could
871
+ // say "zero-retention" while a lane ran on a provider that may train -
872
+ // and the config header printed the claim as fact.
873
+ //
874
+ // The check is by SEVERITY and not by equality. Equality refused the
875
+ // conservative declaration: a profile that says "retained" while every
876
+ // model it runs on is zero-retention was rejected as though it had lied.
877
+ // It had not - it had promised less than the models give, and a promise
878
+ // that is stricter than the truth is the safe direction to err in. Only
879
+ // one direction is a lie, and it is the one the check now refuses.
880
+ //
881
+ // `retentionSeverity` is the same table the ceiling check uses, so the
882
+ // two cannot disagree about which of two levels is the worse one.
883
+ //
884
+ // The utility route is NOT part of this comparison. It is an extra route
885
+ // the profile adds on top of its own models, and it is reported where it
886
+ // belongs: the header names it as the reason the file is worse than the
887
+ // profile claims, and `oc-harness use` warns about it. A profile that
888
+ // names a may-train utility route is therefore installable, and the file
889
+ // it writes says may-train.
890
+ const actual = worstRetention(laneModelsInPlay(p))
891
+ if (retentionSeverity(p.retention) < retentionSeverity(actual.retention)) {
892
+ throw new ProfileError(
893
+ `${where}: retention is ${p.retention}, but ${actual.worst} is ${actual.retention} - a profile cannot be more private than the models it runs on`,
894
+ )
895
+ }
780
896
  }
781
897
 
782
898
  /**
@@ -840,17 +956,53 @@ export function pruneThreshold(p, opts = {}) {
840
956
  }
841
957
 
842
958
  /**
843
- * Every model a profile can put behind the user: the base model plus the model
844
- * each lane actually resolves to.
959
+ * Every model a profile can put behind the user: the base model, the model
960
+ * each lane actually resolves to, and the utility route.
961
+ *
962
+ * The utility route belongs in this list because it is WRITTEN into the
963
+ * config as `small_model` and opencode sends real requests down it - a
964
+ * title, a one-line summary, a rewritten question. It used to be a chain
965
+ * rung only, so it never appeared here, and a profile could name a
966
+ * may-train utility route under a zero-retention claim: the file said
967
+ * "zero-retention" while a training provider served the calls.
968
+ *
969
+ * Every caller of this list wants the same thing - the models that can
970
+ * serve a request - so the list is widened rather than a second one
971
+ * invented beside it. The one caller that needs the lanes alone is
972
+ * `validate()`, which cross-checks the profile's DECLARED retention against
973
+ * the base model and the lanes; that is `laneModelsInPlay`.
845
974
  *
846
975
  * @param {Profile} p
847
976
  * @param {{ unhealthy?: Set<string> }} [opts]
848
977
  * @returns {string[]}
849
978
  */
850
979
  export function modelsInPlay(p, opts = {}) {
980
+ const out = laneModelsInPlay(p, opts)
981
+ const utility = p.utility?.model
982
+ if (typeof utility === "string" && utility && !out.includes(utility)) {
983
+ out.push(utility)
984
+ }
985
+ return out
986
+ }
987
+
988
+ /**
989
+ * The models the profile's own base and lanes put behind the user, without
990
+ * the utility route.
991
+ *
992
+ * This is the set the profile's own `retention` field is about: the base
993
+ * model and the lanes it resolves to. The utility route is an extra route
994
+ * the profile adds on top of them, and it is reported where it belongs - the
995
+ * config header names it, and `oc-harness use` warns about it - so it does
996
+ * not decide what the profile may declare about itself.
997
+ *
998
+ * @param {Profile} p
999
+ * @param {{ unhealthy?: Set<string> }} [opts]
1000
+ * @returns {string[]}
1001
+ */
1002
+ export function laneModelsInPlay(p, opts = {}) {
851
1003
  const out = [p.model]
852
1004
  for (const role of Object.keys(p.roleModels ?? {})) {
853
- const model = resolveLaneModel(p, role, opts).model
1005
+ const model = laneModelId(p, role, opts)
854
1006
  if (!out.includes(model)) out.push(model)
855
1007
  }
856
1008
  return out
@@ -894,10 +1046,20 @@ export function roleSplitModels(p, opts = {}) {
894
1046
  * @returns {{ retention: string, worst: string | null }}
895
1047
  */
896
1048
  export function effectiveRetention(p, opts = {}) {
1049
+ return worstRetention(modelsInPlay(p, opts))
1050
+ }
1051
+
1052
+ /**
1053
+ * The worst retention among a list of models, and the model responsible.
1054
+ *
1055
+ * @param {string[]} ids
1056
+ * @returns {{ retention: string, worst: string | null }}
1057
+ */
1058
+ function worstRetention(ids) {
897
1059
  /** @type {string | null} */
898
1060
  let worst = null
899
1061
  let retention = "zero-retention"
900
- for (const id of modelsInPlay(p, opts)) {
1062
+ for (const id of ids) {
901
1063
  const declared = modelRetention(id)
902
1064
  if (
903
1065
  !worst ||
package/src/prune.ts CHANGED
@@ -66,7 +66,6 @@ interface StubStats {
66
66
  bytesAfter: number
67
67
  }
68
68
 
69
- /** Rough size of a part, used only for the stats line. */
70
69
  /**
71
70
  * The identity of a tool call's output: the tool and what it was asked for.
72
71
  *
@@ -88,14 +87,37 @@ function toolSignature(part: any): string | null {
88
87
  return `${tool} ${input}`
89
88
  }
90
89
 
91
- function sizeOf(part: any): number {
92
- // CHARACTERS, not bytes. A UTF-8 character is more than one byte, and a
93
- // report that says "bytes" and counts characters is off by a factor on
94
- // any text that is not ASCII - which is most of the code in a repository.
95
- // The field is named for what it measures.
96
- if (typeof part?.text === "string") return part.text.length
97
- if (typeof part?.state?.output === "string") return part.state.output.length
98
- return 0
90
+ /**
91
+ * The size of a PART, as it sits in the request.
92
+ *
93
+ * The whole part, not the text inside it. The saving this plugin reports is
94
+ * a claim about the REQUEST, and the request is built out of parts - so
95
+ * measuring `part.text` measured a piece of a part and called the difference
96
+ * a saving. The stub replaces a tool result with a new state object that
97
+ * carries its own metadata, and that metadata was invisible to the measure,
98
+ * so a turn whose request GREW was reported as a saving of the difference
99
+ * between two texts. That is the worst bug this plugin can have: a number
100
+ * that says "I saved you 322 characters" for a request that grew by 406.
101
+ *
102
+ * The metadata the stub adds is counted as part of the stub, deliberately:
103
+ * it is bytes the request really does carry, so a stub that costs more than
104
+ * the output it replaced has to show up as a cost rather than as a saving.
105
+ *
106
+ * CHARACTERS, not bytes. A UTF-8 character is more than one byte, and a
107
+ * report that says "bytes" and counts characters is off by a factor on
108
+ * any text that is not ASCII - which is most of the code in a repository.
109
+ * The field is named for what it measures.
110
+ */
111
+ function partSize(part: any): number {
112
+ try {
113
+ const json = JSON.stringify(part)
114
+ return typeof json === "string" ? json.length : 0
115
+ } catch {
116
+ // A part that cannot be serialised cannot be measured either. It counts
117
+ // as zero rather than throwing: a prune that cannot measure itself must
118
+ // still prune.
119
+ return 0
120
+ }
99
121
  }
100
122
 
101
123
  /**
@@ -261,7 +283,7 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
261
283
  if (!msg?.parts) continue
262
284
  const isUser = msg.info?.role === "user"
263
285
  for (const part of msg.parts) {
264
- const size = sizeOf(part)
286
+ const size = partSize(part)
265
287
  stats.bytesBefore += size
266
288
 
267
289
  if (isUser && keepUsers) {
@@ -295,8 +317,14 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
295
317
  // part schema has no signature, so replacing its text is
296
318
  // safe. A part that grew one would have to be removed rather
297
319
  // than rewritten.
320
+ //
321
+ // There is NO floor under which a reasoning part is left
322
+ // alone. A short one is replaced too, and the stub is about
323
+ // seventy-eight characters - so a reasoning part shorter
324
+ // than that makes the request BIGGER. The log says so,
325
+ // because the log is a measurement and not a claim.
298
326
  part.text = REASONING_STUB
299
- stats.bytesAfter += sizeOf(part)
327
+ stats.bytesAfter += partSize(part)
300
328
  stats.stubbed++
301
329
  break
302
330
  }
@@ -310,6 +338,15 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
310
338
  stats.kept++
311
339
  break
312
340
  }
341
+ // The stub is a NEW state object, and the new state carries
342
+ // its own metadata: a flag and a timestamp, about fifty
343
+ // characters per stubbed part.
344
+ //
345
+ // Those characters are part of the stub, and they are
346
+ // measured as part of it. They used to be invisible to the
347
+ // measurement, so the log reported the difference between
348
+ // two TEXTS while the request carried the metadata as well -
349
+ // and a request that grew was reported as a saving.
313
350
  part.state = {
314
351
  status: "completed",
315
352
  input: state.input,
@@ -318,7 +355,7 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
318
355
  time: state.time,
319
356
  metadata: { ...(state.metadata ?? {}), pruned: true, prunedAt: new Date().toISOString() },
320
357
  }
321
- stats.bytesAfter += sizeOf(part)
358
+ stats.bytesAfter += partSize(part)
322
359
  stats.stubbed++
323
360
  break
324
361
  }
@@ -336,7 +373,15 @@ export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
336
373
  }
337
374
 
338
375
  if (stats.stubbed > 0) {
339
- const saved = Math.max(0, stats.bytesBefore - stats.bytesAfter)
376
+ // The delta of the REQUEST, exactly, and not floored at zero.
377
+ //
378
+ // The saving is a claim about the request, and a request that grew
379
+ // has to say so. A zero would be read as "nothing was saved" and
380
+ // would hide the fact that the pruner made the request BIGGER -
381
+ // which is the failure this number exists to prevent. The line is
382
+ // still logged either way, because a part was stubbed, and that is
383
+ // the event worth recording.
384
+ const saved = stats.bytesBefore - stats.bytesAfter
340
385
  await logPrune(project, profile.id, {
341
386
  ...stats,
342
387
  saved,