opencode-harness-kit 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +527 -18
- package/CONTRIBUTING.md +242 -0
- package/GLOBAL.md +34 -5
- package/MIGRATION.md +151 -0
- package/README.md +307 -37
- package/SECURITY.md +30 -0
- package/bin/baseline.mjs +15 -7
- package/bin/git-hooks/pre-commit +41 -9
- package/bin/oc-harness.mjs +1265 -82
- package/dist/config.d.ts +127 -0
- package/dist/denylist.d.ts +33 -0
- package/dist/effort.d.ts +64 -0
- package/dist/guard.d.ts +23 -0
- package/dist/health.d.ts +43 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +532 -149
- package/dist/plugin.d.ts +15 -0
- package/dist/profile.d.ts +399 -0
- package/dist/project.d.ts +8 -0
- package/dist/prune.d.ts +6 -0
- package/dist/router.d.ts +3 -0
- package/dist/rules.d.ts +109 -0
- package/dist/safe.d.ts +7 -0
- package/dist/telemetry-store.d.ts +12 -0
- package/dist/telemetry.d.ts +3 -0
- package/docs/EXIT-CODES.md +71 -0
- package/docs/KNOWN-LIMITS.md +45 -21
- package/docs/RESEARCH.md +9 -4
- package/docs/ROUTING.md +53 -8
- package/eval/RESULTS.md +443 -0
- package/eval/ab.mjs +743 -0
- package/eval/attribute.mjs +170 -0
- package/eval/flags.mjs +88 -0
- package/eval/measure-install.mjs +323 -0
- package/eval/measure-prune.mjs +339 -0
- package/eval/metrics.js +79 -0
- package/eval/premises.mjs +200 -0
- package/eval/prepare.mjs +233 -0
- package/eval/run.mjs +376 -0
- package/eval/tasks-self.json +81 -0
- package/eval/tasks-ts.json +50 -0
- package/eval/tasks.json +83 -0
- package/eval/thresholds.mjs +134 -0
- package/eval/verify.mjs +287 -0
- package/package.json +44 -17
- package/profiles/fleet-all.json +6 -1
- package/profiles/fleet-safe.json +7 -3
- package/profiles/schema.json +182 -51
- package/prompts/build.md +4 -2
- package/skills/space-build/SKILL.md +4 -2
- package/skills/space-finish/SKILL.md +6 -2
- package/skills/space-plan/reference/task-template.md +13 -5
- package/src/config.js +182 -32
- package/src/denylist.js +95 -0
- package/src/driver/run.mjs +81 -16
- package/src/effort.js +15 -4
- package/src/guard.ts +165 -13
- package/src/health.js +80 -0
- package/src/index.ts +15 -4
- package/src/lane-usage.js +72 -18
- package/src/plugin-type-check.ts +38 -0
- package/src/plugin.ts +39 -0
- package/src/profile.js +225 -26
- package/src/project.js +47 -0
- package/src/prune.ts +110 -31
- package/src/router.ts +32 -63
- package/src/rules.js +99 -7
- package/src/safe.ts +27 -0
- package/src/sqlite.js +144 -0
- package/src/tasks.js +112 -29
- package/src/telemetry-store.ts +62 -0
- package/src/telemetry.ts +60 -57
- package/src/trust.js +70 -9
package/src/profile.js
CHANGED
|
@@ -42,7 +42,7 @@ const HERE = dirname(fileURLToPath(import.meta.url))
|
|
|
42
42
|
/** @param {string} start @returns {string} */
|
|
43
43
|
function findRoot(start) {
|
|
44
44
|
let dir = start
|
|
45
|
-
for (let i = 0; i <
|
|
45
|
+
for (let i = 0; i < 12; i++) {
|
|
46
46
|
if (existsSync(join(dir, "profiles"))) return dir
|
|
47
47
|
const parent = dirname(dir)
|
|
48
48
|
if (parent === dir) break
|
|
@@ -64,6 +64,7 @@ export const ROOT = findRoot(resolve(HERE, ".."))
|
|
|
64
64
|
* @property {string} provider
|
|
65
65
|
* @property {string} model
|
|
66
66
|
* @property {string | null} [family]
|
|
67
|
+
* @property {{ model: string, why?: string }} [utility] the route for opencode's short auxiliary calls
|
|
67
68
|
* @property {string} [verified]
|
|
68
69
|
* @property {Retention} retention
|
|
69
70
|
* @property {Retention} [maxRetention]
|
|
@@ -105,14 +106,41 @@ export const WORKFLOW_LANES = ["build", "plan", "reviewer", "locate", "verify"]
|
|
|
105
106
|
|
|
106
107
|
// Retention, ordered safest-first. A fleet is labelled with the worst level
|
|
107
108
|
// among the models it actually resolves to.
|
|
108
|
-
|
|
109
|
+
//
|
|
110
|
+
// `unknown` is the worst of the known levels on purpose. `retentionAllowed`
|
|
111
|
+
// refuses it outright - a lane that would run on a model nobody has stated
|
|
112
|
+
// the terms of is worse than a lane that runs on a worse model with known
|
|
113
|
+
// terms - and a severity table that ranked it as milder than `may-train`
|
|
114
|
+
// contradicted that: a profile with an unknown-retention model in it was
|
|
115
|
+
// reported as safer than one with a training provider.
|
|
116
|
+
const RETENTION_SEVERITY = ["zero-retention", "retained", "may-train", "unknown"]
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* How bad a retention level is, as a number. Bigger is worse.
|
|
120
|
+
*
|
|
121
|
+
* `unknown` is the worst of them. `retentionAllowed` refuses it outright - a
|
|
122
|
+
* lane that would run on a model nobody has stated the terms of is worse
|
|
123
|
+
* than a lane that runs on a worse model with known terms - so a severity
|
|
124
|
+
* table that ranked it as milder than `may-train` contradicted the rule it
|
|
125
|
+
* sits next to.
|
|
126
|
+
*
|
|
127
|
+
* @param {string} level
|
|
128
|
+
* @returns {number} the rank, or -1 for a level that is not one of them
|
|
129
|
+
*/
|
|
130
|
+
export function retentionSeverity(level) {
|
|
131
|
+
return RETENTION_SEVERITY.indexOf(/** @type {Retention} */ (level))
|
|
132
|
+
}
|
|
109
133
|
|
|
110
134
|
/**
|
|
111
135
|
* Profiles live in profiles/ (verified). schema.json and index.json are not
|
|
112
136
|
* profiles.
|
|
113
137
|
*/
|
|
114
138
|
export function profileDirs() {
|
|
115
|
-
|
|
139
|
+
// The profiles can live somewhere else. A test that refreshes a profile
|
|
140
|
+
// would otherwise rewrite the kit's own data - which is what happened the
|
|
141
|
+
// first time this was tried - and a user who keeps profiles in their own
|
|
142
|
+
// tree cannot use the kit at all.
|
|
143
|
+
return [process.env.OPENCODE_HARNESS_PROFILES ?? join(ROOT, "profiles")]
|
|
116
144
|
}
|
|
117
145
|
|
|
118
146
|
/** @param {string} id @returns {string | null} */
|
|
@@ -229,15 +257,31 @@ export function laneNames() {
|
|
|
229
257
|
* blocks and plain scalars costs about forty lines, which is less than a
|
|
230
258
|
* dependency for a file this shape.
|
|
231
259
|
*
|
|
232
|
-
*
|
|
260
|
+
* Cached, like the lane table: `retentionFor` is called for every candidate
|
|
261
|
+
* of every lane, and every candidate of every lane re-read and re-parsed
|
|
262
|
+
* the whole file.
|
|
263
|
+
*
|
|
264
|
+
* The `defaults:` block is read, because it LOOKS like configuration and
|
|
265
|
+
* was not: the fallback for a model with no entry was hardcoded in the
|
|
266
|
+
* reader, so changing `defaults.unknown` in the file did nothing.
|
|
267
|
+
*
|
|
268
|
+
* @returns {{ models: Record<string, RetentionRecord>, defaults: Record<string, string> }}
|
|
233
269
|
*/
|
|
270
|
+
/** @type {{ models: Record<string, RetentionRecord>, defaults: Record<string, string> } | null} */
|
|
271
|
+
let retentionCache = null
|
|
234
272
|
export function loadRetention() {
|
|
273
|
+
if (retentionCache) return retentionCache
|
|
235
274
|
const file = join(ROOT, "retention.yml")
|
|
236
|
-
if (!existsSync(file))
|
|
237
|
-
|
|
275
|
+
if (!existsSync(file)) {
|
|
276
|
+
retentionCache = { models: {}, defaults: {} }
|
|
277
|
+
return retentionCache
|
|
278
|
+
}
|
|
279
|
+
const models = /** @type {Record<string, any>} */ ({})
|
|
280
|
+
const defaults = /** @type {Record<string, string>} */ ({})
|
|
238
281
|
const lines = readFileSync(file, "utf8").split("\n")
|
|
239
282
|
|
|
240
283
|
let current = null
|
|
284
|
+
let inDefaults = false
|
|
241
285
|
let i = 0
|
|
242
286
|
|
|
243
287
|
while (i < lines.length) {
|
|
@@ -249,12 +293,31 @@ export function loadRetention() {
|
|
|
249
293
|
i++
|
|
250
294
|
continue
|
|
251
295
|
}
|
|
252
|
-
//
|
|
253
|
-
|
|
296
|
+
// `defaults:` is a top-level block, and its values are plain scalars.
|
|
297
|
+
if (indent === 0 && line === "defaults:") {
|
|
298
|
+
inDefaults = true
|
|
299
|
+
current = null
|
|
300
|
+
i++
|
|
301
|
+
continue
|
|
302
|
+
}
|
|
303
|
+
if (indent === 0 && line === "models:") {
|
|
304
|
+
inDefaults = false
|
|
305
|
+
current = null
|
|
306
|
+
i++
|
|
307
|
+
continue
|
|
308
|
+
}
|
|
309
|
+
if (inDefaults) {
|
|
310
|
+
const pair = line.match(/^([a-z-]+)\s*:\s*(\S+)\s*$/)
|
|
311
|
+
if (pair) defaults[pair[1]] = pair[2]
|
|
312
|
+
i++
|
|
313
|
+
continue
|
|
314
|
+
}
|
|
315
|
+
// A model key sits at two-space indent under `models:`. A key with no
|
|
316
|
+
// provider prefix gets one, so a lookup by bare model id finds it.
|
|
254
317
|
if (indent === 2 && line.endsWith(":")) {
|
|
255
318
|
const key = line.slice(0, -1).trim()
|
|
256
319
|
current = key.includes("/") ? key : `opencode/${key}`
|
|
257
|
-
|
|
320
|
+
models[current] = { retention: "unknown" }
|
|
258
321
|
i++
|
|
259
322
|
continue
|
|
260
323
|
}
|
|
@@ -281,17 +344,18 @@ export function loadRetention() {
|
|
|
281
344
|
if (l.length - l.trimStart().length <= indent) break
|
|
282
345
|
parts.push(l.trim())
|
|
283
346
|
}
|
|
284
|
-
|
|
347
|
+
models[current][name] = parts.join(" ").replace(/\s+/g, " ").trim()
|
|
285
348
|
i = j
|
|
286
349
|
continue
|
|
287
350
|
}
|
|
288
|
-
|
|
351
|
+
models[current][name] = rest.replace(/^["']|["']$/g, "")
|
|
289
352
|
i++
|
|
290
353
|
continue
|
|
291
354
|
}
|
|
292
355
|
i++
|
|
293
356
|
}
|
|
294
|
-
|
|
357
|
+
retentionCache = { models, defaults }
|
|
358
|
+
return retentionCache
|
|
295
359
|
}
|
|
296
360
|
|
|
297
361
|
/**
|
|
@@ -301,17 +365,27 @@ export function loadRetention() {
|
|
|
301
365
|
* @returns {RetentionRecord | undefined}
|
|
302
366
|
*/
|
|
303
367
|
export function retentionFor(id) {
|
|
304
|
-
const table = loadRetention()
|
|
368
|
+
const table = loadRetention().models
|
|
305
369
|
return table[id] ?? table[id.includes("/") ? id : `opencode/${id}`]
|
|
306
370
|
}
|
|
307
371
|
|
|
308
372
|
/**
|
|
309
|
-
* The retention of a model id,
|
|
373
|
+
* The retention of a model id, out of a retention table.
|
|
374
|
+
*
|
|
375
|
+
* Pure, so a test can hand it a table whose `defaults.unknown` says
|
|
376
|
+
* something else. The shipped table says "unknown", which is what the
|
|
377
|
+
* hardcoded fallback used to be - so a test that only checked the value
|
|
378
|
+
* could not tell the two apart.
|
|
379
|
+
*
|
|
380
|
+
* @param {{ models?: Record<string, RetentionRecord>, defaults?: Record<string, string> }} table
|
|
310
381
|
* @param {string} model
|
|
311
382
|
* @returns {Retention}
|
|
312
383
|
*/
|
|
313
|
-
export function
|
|
314
|
-
const
|
|
384
|
+
export function retentionOf(table, model) {
|
|
385
|
+
const fallback = table?.defaults?.unknown ?? "unknown"
|
|
386
|
+
const models = table?.models ?? {}
|
|
387
|
+
const record = models[model] ?? models[model.includes("/") ? model : `opencode/${model}`]
|
|
388
|
+
const declared = record?.retention ?? fallback
|
|
315
389
|
return /** @type {Retention} */ (
|
|
316
390
|
["zero-retention", "retained", "may-train", "unknown"].includes(declared)
|
|
317
391
|
? declared
|
|
@@ -319,6 +393,21 @@ export function modelRetention(model) {
|
|
|
319
393
|
)
|
|
320
394
|
}
|
|
321
395
|
|
|
396
|
+
/**
|
|
397
|
+
* The retention of a model id, qualified or not.
|
|
398
|
+
*
|
|
399
|
+
* A model with no entry in retention.yml gets the `defaults.unknown` value
|
|
400
|
+
* from that file, which is `unknown` today. The fallback used to be
|
|
401
|
+
* hardcoded here, so the `defaults:` block in retention.yml looked like
|
|
402
|
+
* configuration and was not.
|
|
403
|
+
*
|
|
404
|
+
* @param {string} model
|
|
405
|
+
* @returns {Retention}
|
|
406
|
+
*/
|
|
407
|
+
export function modelRetention(model) {
|
|
408
|
+
return retentionOf(loadRetention(), model)
|
|
409
|
+
}
|
|
410
|
+
|
|
322
411
|
/**
|
|
323
412
|
* Whether a model's retention is at or below a ceiling.
|
|
324
413
|
*
|
|
@@ -333,8 +422,8 @@ export function modelRetention(model) {
|
|
|
333
422
|
export function retentionAllowed(model, ceiling) {
|
|
334
423
|
const level = modelRetention(model)
|
|
335
424
|
if (level === "unknown") return false
|
|
336
|
-
const max =
|
|
337
|
-
const have =
|
|
425
|
+
const max = retentionSeverity(ceiling)
|
|
426
|
+
const have = retentionSeverity(level)
|
|
338
427
|
if (max < 0) return false
|
|
339
428
|
return have <= max
|
|
340
429
|
}
|
|
@@ -363,7 +452,8 @@ export function resolveLaneChain(p, role) {
|
|
|
363
452
|
*
|
|
364
453
|
* The first rung of the chain that is allowed by the retention ceiling and is
|
|
365
454
|
* not marked unhealthy wins. When no rung qualifies, the lane falls back to
|
|
366
|
-
* the profile's base model, so a lane is never left without a model
|
|
455
|
+
* the profile's base model, so a lane is never left without a model - and a
|
|
456
|
+
* base model that is itself spent is the last resort rather than the first.
|
|
367
457
|
*
|
|
368
458
|
* @param {Profile} p
|
|
369
459
|
* @param {string} role
|
|
@@ -378,9 +468,16 @@ export function resolveLaneModel(p, role, opts = {}) {
|
|
|
378
468
|
if (!retentionAllowed(candidate, ceiling)) continue
|
|
379
469
|
return { model: candidate, levels: levelsOfModel(candidate) }
|
|
380
470
|
}
|
|
381
|
-
// Nothing in the chain qualifies.
|
|
382
|
-
//
|
|
383
|
-
//
|
|
471
|
+
// Nothing in the chain qualifies. The base model is the next thing to try:
|
|
472
|
+
// it is the profile's own route, and a lane pinned to a rung the router has
|
|
473
|
+
// already recorded as spent is worse than a lane on a route that was never
|
|
474
|
+
// preferred for this job but is alive.
|
|
475
|
+
if (p.model && !unhealthy.has(p.model) && retentionAllowed(p.model, ceiling)) {
|
|
476
|
+
return { model: p.model, levels: levelsOfModel(p.model) }
|
|
477
|
+
}
|
|
478
|
+
// Last resort: the first rung anyway rather than leaving the lane without a
|
|
479
|
+
// model, and let the config header say what the effective retention turned
|
|
480
|
+
// out to be.
|
|
384
481
|
const fallback = resolveLaneChain(p, role)[0] ?? p.model
|
|
385
482
|
return { model: fallback, levels: levelsOfModel(fallback) }
|
|
386
483
|
}
|
|
@@ -471,6 +568,14 @@ export function resolveEffort(levels, requested) {
|
|
|
471
568
|
if (!levels || levels.length === 0) return null
|
|
472
569
|
if (levels.includes(requested)) return requested
|
|
473
570
|
|
|
571
|
+
// `none` is not a rung on the reasoning scale. It is reasoning OFF, and
|
|
572
|
+
// "off" is not "the lowest level the model has". Snapping it onto `low`
|
|
573
|
+
// would turn the one lane that exists to stop reasoning into the lane that
|
|
574
|
+
// reasons the most, and it would do so silently. A model that does not
|
|
575
|
+
// publish `none` cannot turn reasoning off, so the honest answer is "emit
|
|
576
|
+
// no variant" and let the model use its own default.
|
|
577
|
+
if (requested === "none") return null
|
|
578
|
+
|
|
474
579
|
const want = CANON.indexOf(requested)
|
|
475
580
|
if (want < 0) {
|
|
476
581
|
// Unknown label. Snap to the top level, which is the safe reading of a
|
|
@@ -538,6 +643,32 @@ export function validate(p, path = "<memory>") {
|
|
|
538
643
|
)
|
|
539
644
|
}
|
|
540
645
|
}
|
|
646
|
+
// The pruning knobs are limits, and a limit of zero or less is not a limit.
|
|
647
|
+
// `keepRecentTurns: -1` made the prunable span reach the last message,
|
|
648
|
+
// which is the one message the file's own invariants say never to touch.
|
|
649
|
+
if (p.prune) {
|
|
650
|
+
const prune = p.prune
|
|
651
|
+
if (prune.keepRecentToolCalls !== undefined) {
|
|
652
|
+
if (!Number.isInteger(prune.keepRecentToolCalls) || prune.keepRecentToolCalls < 1) {
|
|
653
|
+
throw new ProfileError(
|
|
654
|
+
`${where}: prune.keepRecentToolCalls must be an integer of at least 1`,
|
|
655
|
+
)
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
if (prune.keepRecentTurns !== undefined) {
|
|
659
|
+
if (!Number.isInteger(prune.keepRecentTurns) || prune.keepRecentTurns < 1) {
|
|
660
|
+
throw new ProfileError(
|
|
661
|
+
`${where}: prune.keepRecentTurns must be an integer of at least 1`,
|
|
662
|
+
)
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (prune.keepUserMessages !== undefined && typeof prune.keepUserMessages !== "boolean") {
|
|
666
|
+
throw new ProfileError(`${where}: prune.keepUserMessages must be a boolean`)
|
|
667
|
+
}
|
|
668
|
+
if (prune.enabled !== undefined && typeof prune.enabled !== "boolean") {
|
|
669
|
+
throw new ProfileError(`${where}: prune.enabled must be a boolean`)
|
|
670
|
+
}
|
|
671
|
+
}
|
|
541
672
|
for (const [role, rm] of Object.entries(p.roleModels ?? {})) {
|
|
542
673
|
if (!LANE_NAMES.includes(role)) {
|
|
543
674
|
throw new ProfileError(
|
|
@@ -554,9 +685,13 @@ export function validate(p, path = "<memory>") {
|
|
|
554
685
|
if (!rm?.model || typeof rm.model !== "string") {
|
|
555
686
|
throw new ProfileError(`${where}: roleModels.${role} needs a model`)
|
|
556
687
|
}
|
|
557
|
-
if (rm.levels !== undefined
|
|
688
|
+
if (rm.levels !== undefined) {
|
|
689
|
+
// The field used to be in the schema and validated as an array, and
|
|
690
|
+
// then nothing read it. A profile author could fill it with a lie and
|
|
691
|
+
// get no error. It is out of the schema now, and a profile that still
|
|
692
|
+
// carries it is told so rather than having it ignored.
|
|
558
693
|
throw new ProfileError(
|
|
559
|
-
`${where}: roleModels.${role}.levels
|
|
694
|
+
`${where}: roleModels.${role}.levels is not read by anything - the lane takes its levels from the model it runs on. Remove it.`,
|
|
560
695
|
)
|
|
561
696
|
}
|
|
562
697
|
if (rm.chain !== undefined) {
|
|
@@ -576,6 +711,15 @@ export function validate(p, path = "<memory>") {
|
|
|
576
711
|
`${where}: roleModels.${role}.chain names ${rung}, which has no retention.yml entry`,
|
|
577
712
|
)
|
|
578
713
|
}
|
|
714
|
+
// A rung with no profile cannot be resolved: its levels are unknown
|
|
715
|
+
// and its limits are unknown, so the lane quietly runs on a model it
|
|
716
|
+
// did not ask for. The retention check above passed it, because
|
|
717
|
+
// retention.yml lists a few paid routes the kit has no profile for.
|
|
718
|
+
if (!profileExists(rung)) {
|
|
719
|
+
throw new ProfileError(
|
|
720
|
+
`${where}: roleModels.${role}.chain names ${rung}, which has no profile in this directory`,
|
|
721
|
+
)
|
|
722
|
+
}
|
|
579
723
|
}
|
|
580
724
|
}
|
|
581
725
|
// retention.yml is the only place retention is decided, so a role model
|
|
@@ -585,9 +729,61 @@ export function validate(p, path = "<memory>") {
|
|
|
585
729
|
`${where}: roleModels.${role} names ${rm.model}, which has no retention.yml entry`,
|
|
586
730
|
)
|
|
587
731
|
}
|
|
732
|
+
if (!profileExists(rm.model)) {
|
|
733
|
+
throw new ProfileError(
|
|
734
|
+
`${where}: roleModels.${role} names ${rm.model}, which has no profile in this directory`,
|
|
735
|
+
)
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
// The utility route has to be a model this profile can actually run, or a
|
|
739
|
+
// failover leaves opencode's short auxiliary calls pointing at a model that
|
|
740
|
+
// is not in the fleet.
|
|
741
|
+
//
|
|
742
|
+
// The check is STATIC - the union of the profile's own model and every
|
|
743
|
+
// model its chains name. Asking `modelsInPlay` would be the obvious way,
|
|
744
|
+
// and it is a loop: resolving a lane reads the levels of a model, which
|
|
745
|
+
// loads that model's profile, which validates it, which resolves a lane.
|
|
746
|
+
if (p.utility) {
|
|
747
|
+
if (typeof p.utility.model !== "string" || !p.utility.model) {
|
|
748
|
+
throw new ProfileError(`${where}: utility.model must be a model id`)
|
|
749
|
+
}
|
|
750
|
+
const reachable = new Set([p.model])
|
|
751
|
+
for (const rm of Object.values(p.roleModels ?? {})) {
|
|
752
|
+
if (rm?.model) reachable.add(rm.model)
|
|
753
|
+
for (const rung of rm?.chain ?? []) reachable.add(rung)
|
|
754
|
+
}
|
|
755
|
+
if (!reachable.has(p.utility.model)) {
|
|
756
|
+
throw new ProfileError(
|
|
757
|
+
`${where}: utility.model is ${p.utility.model}, which this profile does not resolve to (it has ${[...reachable].join(", ")})`,
|
|
758
|
+
)
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
// The fallback queue is walked at startup, and an id that does not resolve
|
|
763
|
+
// is skipped without a word - so the queue quietly shortens and the
|
|
764
|
+
// session stays on a route that is already spent.
|
|
765
|
+
for (const id of p.fallback ?? []) {
|
|
766
|
+
if (!profileExists(id)) {
|
|
767
|
+
throw new ProfileError(
|
|
768
|
+
`${where}: fallback names ${id}, which has no profile in this directory`,
|
|
769
|
+
)
|
|
770
|
+
}
|
|
588
771
|
}
|
|
589
772
|
}
|
|
590
773
|
|
|
774
|
+
/**
|
|
775
|
+
* Whether a model id has a profile in this directory.
|
|
776
|
+
*
|
|
777
|
+
* The file is looked for rather than loaded: `loadProfile` validates, and
|
|
778
|
+
* validation is what asks this question.
|
|
779
|
+
*
|
|
780
|
+
* @param {string} id
|
|
781
|
+
* @returns {boolean}
|
|
782
|
+
*/
|
|
783
|
+
function profileExists(id) {
|
|
784
|
+
return profilePath(id) !== null
|
|
785
|
+
}
|
|
786
|
+
|
|
591
787
|
// --- context sizing ---------------------------------------------------------
|
|
592
788
|
|
|
593
789
|
/**
|
|
@@ -697,7 +893,7 @@ export function effectiveRetention(p, opts = {}) {
|
|
|
697
893
|
const declared = modelRetention(id)
|
|
698
894
|
if (
|
|
699
895
|
!worst ||
|
|
700
|
-
|
|
896
|
+
retentionSeverity(declared) > retentionSeverity(retention)
|
|
701
897
|
) {
|
|
702
898
|
worst = id
|
|
703
899
|
retention = declared
|
|
@@ -710,8 +906,11 @@ export function effectiveRetention(p, opts = {}) {
|
|
|
710
906
|
|
|
711
907
|
/** Where opencode keeps its config, honouring the same env var it does. */
|
|
712
908
|
export function configDir() {
|
|
909
|
+
// `homedir()` and not `process.env.HOME`: with HOME unset the other form
|
|
910
|
+
// produced a RELATIVE path, so `use` wrote to one place and
|
|
911
|
+
// `profileIdFromConfig` read another.
|
|
713
912
|
return (
|
|
714
|
-
process.env.OPENCODE_CONFIG_DIR ?? join(
|
|
913
|
+
process.env.OPENCODE_CONFIG_DIR ?? join(homedir(), ".config", "opencode")
|
|
715
914
|
)
|
|
716
915
|
}
|
|
717
916
|
|
package/src/project.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Where the kit's files live, and where its scope ends.
|
|
2
|
+
//
|
|
3
|
+
// `ctx.worktree` is opencode's WORKTREE: the git root of the project. For a
|
|
4
|
+
// directory that is not a git repository it is the FILESYSTEM ROOT, which is
|
|
5
|
+
// not a place to write a project's telemetry.
|
|
6
|
+
//
|
|
7
|
+
// That was not visible from the code, because the kit only ever ran in git
|
|
8
|
+
// repositories. In every other project the config hook's write to
|
|
9
|
+
// `.space/active` failed with a permission error that the plugin swallowed,
|
|
10
|
+
// so:
|
|
11
|
+
//
|
|
12
|
+
// - the pruner never found an active profile and did nothing at all,
|
|
13
|
+
// - the router never recorded a spent route, so no lane ever failed over,
|
|
14
|
+
// - the telemetry wrote to a place nobody looks.
|
|
15
|
+
//
|
|
16
|
+
// The session's own DIRECTORY is the reliable answer: it is where the user is
|
|
17
|
+
// working, and it is set for every session. The git root is preferred when
|
|
18
|
+
// there is one, because a project's files live there and a subdirectory of a
|
|
19
|
+
// monorepo is not the project.
|
|
20
|
+
|
|
21
|
+
import { existsSync } from "node:fs"
|
|
22
|
+
import { dirname, join, resolve } from "node:path"
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The root of the project a session is working in.
|
|
26
|
+
*
|
|
27
|
+
* @param {string | null | undefined} directory the session's working directory
|
|
28
|
+
* @param {string | null | undefined} worktree opencode's worktree, as a fallback
|
|
29
|
+
* @returns {string}
|
|
30
|
+
*/
|
|
31
|
+
export function projectRoot(directory, worktree) {
|
|
32
|
+
const start = resolve(directory || worktree || ".")
|
|
33
|
+
let dir = start
|
|
34
|
+
// The walk is bounded by the filesystem, not by a number: a session more
|
|
35
|
+
// than a dozen directories below the git root is a monorepo, and a cap
|
|
36
|
+
// that stops early would put the kit's files in a subdirectory.
|
|
37
|
+
for (let i = 0; i < 64; i++) {
|
|
38
|
+
if (existsSync(join(dir, ".git"))) return dir
|
|
39
|
+
const parent = dirname(dir)
|
|
40
|
+
if (parent === dir) break
|
|
41
|
+
dir = parent
|
|
42
|
+
}
|
|
43
|
+
// No git root anywhere above: the session's own directory is the project.
|
|
44
|
+
// The worktree is deliberately NOT used here - in a non-git project it is
|
|
45
|
+
// the filesystem root.
|
|
46
|
+
return start
|
|
47
|
+
}
|
package/src/prune.ts
CHANGED
|
@@ -16,23 +16,26 @@
|
|
|
16
16
|
// - never touch patch or file parts, which carry diffs the model needs
|
|
17
17
|
// - never stub a user message unless asked to
|
|
18
18
|
|
|
19
|
-
import type { Plugin } from "
|
|
19
|
+
import type { Plugin } from "./plugin.js"
|
|
20
20
|
import { loadProfile, pruneThreshold, ROOT } from "./profile.js"
|
|
21
|
+
import { projectRoot } from "./project.js"
|
|
21
22
|
import { readFileSync, existsSync } from "node:fs"
|
|
22
23
|
import { join } from "node:path"
|
|
23
24
|
|
|
24
25
|
/**
|
|
25
|
-
* Which profile is active. In order: an explicit environment variable, the
|
|
26
|
-
* marker the
|
|
26
|
+
* Which profile is active. In order: an explicit environment variable, then the
|
|
27
|
+
* marker the config hook writes into the project.
|
|
27
28
|
* Returns null when nothing is set, and pruning then stays off rather than
|
|
28
29
|
* guessing at thresholds.
|
|
29
30
|
*/
|
|
30
31
|
function activeProfileId(worktree: string): string | null {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
// The environment variable is a PROFILE ID, not a path.
|
|
33
|
+
//
|
|
34
|
+
// It was being checked with existsSync, like the marker below it, so it
|
|
35
|
+
// never matched a file and the documented override had never worked.
|
|
36
|
+
const fromEnv = process.env.OPENCODE_HARNESS_PROFILE
|
|
37
|
+
if (fromEnv && fromEnv.trim()) return fromEnv.trim()
|
|
38
|
+
const candidates = [join(worktree, ".space", "active")]
|
|
36
39
|
for (const f of candidates) {
|
|
37
40
|
if (!f) continue
|
|
38
41
|
try {
|
|
@@ -47,6 +50,15 @@ function activeProfileId(worktree: string): string | null {
|
|
|
47
50
|
return null
|
|
48
51
|
}
|
|
49
52
|
|
|
53
|
+
/**
|
|
54
|
+
* What an old reasoning part is replaced with.
|
|
55
|
+
*
|
|
56
|
+
* Short, and it says what happened: the model should not go looking for the
|
|
57
|
+
* reasoning it can no longer see.
|
|
58
|
+
*/
|
|
59
|
+
const REASONING_STUB =
|
|
60
|
+
"[reasoning] dropped from context - it was from an earlier turn and is stale."
|
|
61
|
+
|
|
50
62
|
interface StubStats {
|
|
51
63
|
stubbed: number
|
|
52
64
|
kept: number
|
|
@@ -56,6 +68,10 @@ interface StubStats {
|
|
|
56
68
|
|
|
57
69
|
/** Rough size of a part, used only for the stats line. */
|
|
58
70
|
function sizeOf(part: any): number {
|
|
71
|
+
// CHARACTERS, not bytes. A UTF-8 character is more than one byte, and a
|
|
72
|
+
// report that says "bytes" and counts characters is off by a factor on
|
|
73
|
+
// any text that is not ASCII - which is most of the code in a repository.
|
|
74
|
+
// The field is named for what it measures.
|
|
59
75
|
if (typeof part?.text === "string") return part.text.length
|
|
60
76
|
if (typeof part?.state?.output === "string") return part.state.output.length
|
|
61
77
|
return 0
|
|
@@ -76,7 +92,10 @@ function stub(part: any): string {
|
|
|
76
92
|
const status = state.status ?? "done"
|
|
77
93
|
let tail = ""
|
|
78
94
|
const out = String(state.output ?? "")
|
|
79
|
-
|
|
95
|
+
// Whole words only. The pattern used to match "ok" anywhere, so a command
|
|
96
|
+
// whose output contained "took 3 seconds" was reported as "ok" - a
|
|
97
|
+
// passing verdict that was never observed.
|
|
98
|
+
const m = out.match(/\b(?:\d+\s+(?:passed|failed)|passed|failed|error|ok)\b/gi)
|
|
80
99
|
if (m && m.length) tail = `, ${m[m.length - 1]}`
|
|
81
100
|
return `[${tool}] ${cmd} -> ${status}${tail} | full output was pruned from context; re-run the command if you need it`
|
|
82
101
|
}
|
|
@@ -87,28 +106,67 @@ function stub(part: any): string {
|
|
|
87
106
|
return `[${tool}] result pruned from context (${state.status ?? "done"}); re-run if you need it`
|
|
88
107
|
}
|
|
89
108
|
|
|
90
|
-
export const PrunePlugin: Plugin = async ({ worktree,
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
109
|
+
export const PrunePlugin: Plugin = async ({ worktree, directory }) => {
|
|
110
|
+
// The project root, not the worktree: in a project that is not a git
|
|
111
|
+
// repository the worktree is the filesystem root, so the marker the pruner
|
|
112
|
+
// reads was never where it looked, and the pruner did nothing.
|
|
113
|
+
const project = projectRoot(directory, worktree)
|
|
114
|
+
// The profile is resolved on the FIRST TRANSFORM, not here.
|
|
115
|
+
//
|
|
116
|
+
// OpenCode builds the plugin factories before it runs the config hooks, and
|
|
117
|
+
// the config hook is what writes the marker file naming the active profile.
|
|
118
|
+
// Resolving here meant a fresh project had no marker yet, so pruning was
|
|
119
|
+
// off for that entire first session - and a project switched to another
|
|
120
|
+
// profile kept pruning with the previous one's thresholds for one more
|
|
121
|
+
// session.
|
|
122
|
+
//
|
|
123
|
+
// The hook is therefore always registered, and the decision to prune is made
|
|
124
|
+
// per request. Pruning is opt-in per profile: with no active profile, or a
|
|
125
|
+
// profile that does not enable it, nothing is touched.
|
|
126
|
+
type Resolved = {
|
|
127
|
+
profile: any
|
|
128
|
+
keepCalls: number
|
|
129
|
+
keepTurns: number
|
|
130
|
+
keepUsers: boolean
|
|
131
|
+
threshold: number
|
|
97
132
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
133
|
+
const DISABLED: Resolved = {
|
|
134
|
+
profile: null,
|
|
135
|
+
keepCalls: 0,
|
|
136
|
+
keepTurns: 0,
|
|
137
|
+
keepUsers: true,
|
|
138
|
+
threshold: 0,
|
|
102
139
|
}
|
|
140
|
+
// Only a SUCCESSFUL lookup is cached. A lookup that found nothing is
|
|
141
|
+
// retried on the next request, so a project whose marker appears later
|
|
142
|
+
// still gets pruning.
|
|
143
|
+
let cached: Resolved | null = null
|
|
103
144
|
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
145
|
+
const resolve = (): Resolved => {
|
|
146
|
+
if (cached) return cached
|
|
147
|
+
let profile = null as ReturnType<typeof loadProfile> | null
|
|
148
|
+
try {
|
|
149
|
+
const id = activeProfileId(project)
|
|
150
|
+
if (id) profile = loadProfile(id)
|
|
151
|
+
} catch {
|
|
152
|
+
profile = null
|
|
153
|
+
}
|
|
154
|
+
if (!profile?.prune?.enabled) return DISABLED
|
|
155
|
+
cached = {
|
|
156
|
+
profile,
|
|
157
|
+
keepCalls: profile.prune.keepRecentToolCalls ?? 6,
|
|
158
|
+
keepTurns: profile.prune.keepRecentTurns ?? 3,
|
|
159
|
+
keepUsers: profile.prune.keepUserMessages ?? true,
|
|
160
|
+
threshold: pruneThreshold(profile),
|
|
161
|
+
}
|
|
162
|
+
return cached
|
|
163
|
+
}
|
|
108
164
|
|
|
109
165
|
return {
|
|
110
|
-
"experimental.chat.messages.transform": async (_input, output) => {
|
|
166
|
+
"experimental.chat.messages.transform": async (_input: any, output: any) => {
|
|
111
167
|
try {
|
|
168
|
+
const { profile, keepCalls, keepTurns, keepUsers, threshold } = resolve()
|
|
169
|
+
if (!profile) return
|
|
112
170
|
const messages = output.messages
|
|
113
171
|
if (!Array.isArray(messages) || messages.length === 0) return
|
|
114
172
|
|
|
@@ -165,11 +223,25 @@ export const PrunePlugin: Plugin = async ({ worktree, $ }) => {
|
|
|
165
223
|
stats.bytesAfter += size
|
|
166
224
|
stats.kept++
|
|
167
225
|
break
|
|
168
|
-
case "reasoning":
|
|
169
|
-
// Old reasoning is stale by definition and safe to drop
|
|
170
|
-
|
|
226
|
+
case "reasoning": {
|
|
227
|
+
// Old reasoning is stale by definition and safe to drop, and
|
|
228
|
+
// it IS dropped.
|
|
229
|
+
//
|
|
230
|
+
// The part used to keep its full text while the stats booked
|
|
231
|
+
// 200 bytes "after" for it. The reported saving therefore
|
|
232
|
+
// included bytes that were never removed - and reasoning is
|
|
233
|
+
// 15.6% of stored bytes, so the number was not a rounding
|
|
234
|
+
// error.
|
|
235
|
+
//
|
|
236
|
+
// The part keeps its shape: a reasoning part in the OpenCode
|
|
237
|
+
// part schema has no signature, so replacing its text is
|
|
238
|
+
// safe. A part that grew one would have to be removed rather
|
|
239
|
+
// than rewritten.
|
|
240
|
+
part.text = REASONING_STUB
|
|
241
|
+
stats.bytesAfter += sizeOf(part)
|
|
171
242
|
stats.stubbed++
|
|
172
243
|
break
|
|
244
|
+
}
|
|
173
245
|
case "tool": {
|
|
174
246
|
// Only completed calls carry an output worth pruning. Pending
|
|
175
247
|
// and running states are small, and error states are usually
|
|
@@ -207,7 +279,7 @@ export const PrunePlugin: Plugin = async ({ worktree, $ }) => {
|
|
|
207
279
|
|
|
208
280
|
if (stats.stubbed > 0) {
|
|
209
281
|
const saved = Math.max(0, stats.bytesBefore - stats.bytesAfter)
|
|
210
|
-
await logPrune(profile.id, {
|
|
282
|
+
await logPrune(project, profile.id, {
|
|
211
283
|
...stats,
|
|
212
284
|
saved,
|
|
213
285
|
threshold,
|
|
@@ -227,13 +299,20 @@ export const PrunePlugin: Plugin = async ({ worktree, $ }) => {
|
|
|
227
299
|
}
|
|
228
300
|
|
|
229
301
|
async function logPrune(
|
|
302
|
+
worktree: string,
|
|
230
303
|
profileId: string,
|
|
231
304
|
stats: StubStats & {
|
|
232
305
|
saved: number
|
|
233
306
|
threshold: number
|
|
234
307
|
},
|
|
235
308
|
) {
|
|
236
|
-
|
|
309
|
+
// The WORKTREE, not `process.cwd()`. Every other writer in this kit uses
|
|
310
|
+
// the worktree, and the plugin's worktree is the project. The process cwd
|
|
311
|
+
// is wherever opencode happened to be started from, which is often the
|
|
312
|
+
// user's home directory - so the log landed outside the project, in a
|
|
313
|
+
// directory no project-scoped tool looks at, and two projects sharing a cwd
|
|
314
|
+
// shared one log.
|
|
315
|
+
const dir = join(worktree, ".space")
|
|
237
316
|
try {
|
|
238
317
|
const { mkdir, appendFile } = await import("node:fs/promises")
|
|
239
318
|
await mkdir(dir, { recursive: true })
|
|
@@ -244,7 +323,7 @@ async function logPrune(
|
|
|
244
323
|
profile: profileId,
|
|
245
324
|
stubbed: stats.stubbed,
|
|
246
325
|
kept: stats.kept,
|
|
247
|
-
|
|
326
|
+
savedChars: stats.saved,
|
|
248
327
|
threshold: stats.threshold,
|
|
249
328
|
}) + "\n",
|
|
250
329
|
)
|