@quadiusai/dsh-greenharness 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GETTING-STARTED.md +90 -0
- package/README.md +149 -2
- package/bench/BREV.md +127 -0
- package/bench/lib/patch.mjs +60 -0
- package/bench/mock-openai-server.mjs +210 -0
- package/bench/presets/mock-chat-template.json +40 -0
- package/bench/presets/mock.json +32 -0
- package/bench/presets/nemo-brev.json +40 -0
- package/bench/run-case.mjs +242 -0
- package/bench/run-mock-suite.mjs +85 -0
- package/bench/scenarios/loop-stuck.json +11 -0
- package/bench/scenarios/subtract-fix.json +12 -0
- package/bench/tasks/subtract-bug/TASK.md +2 -0
- package/bench/tasks/subtract-bug/package.json +9 -0
- package/bench/tasks/subtract-bug/src/math.mjs +8 -0
- package/bench/tasks/subtract-bug/test/math.test.mjs +11 -0
- package/bench/tasks/subtract-bug/test/noise.test.mjs +9 -0
- package/cordis.patch.yml +4 -0
- package/package.json +42 -4
- package/src/config.mjs +100 -0
- package/src/control.mjs +261 -0
- package/src/core/ansi.mjs +15 -0
- package/src/core/cascade.mjs +81 -0
- package/src/core/distill.mjs +128 -0
- package/src/core/ledger.mjs +21 -0
- package/src/core/loop-guard.mjs +52 -0
- package/src/core/patterns.mjs +26 -0
- package/src/core/pricing.mjs +23 -0
- package/src/core/read-index.mjs +74 -0
- package/src/core/route.mjs +41 -0
- package/src/core/verify.mjs +89 -0
- package/src/index.mjs +233 -0
- package/src/runtime.mjs +297 -0
package/src/config.mjs
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { DEFAULT_VERIFIER_PATTERNS, compileVerifierPatterns } from './core/verify.mjs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Validate and complete the plugin configuration. Misconfiguration fails at load: a tier
|
|
5
|
+
* without a provider and model, a non-integer threshold, or an invalid verifier pattern throws
|
|
6
|
+
* here rather than silently changing routing later.
|
|
7
|
+
*
|
|
8
|
+
* @param {Record<string, any>} raw - the configuration DSH validated against the plugin schema.
|
|
9
|
+
* @returns {{
|
|
10
|
+
* tiers: { small?: Route, large?: Route },
|
|
11
|
+
* routing: { enabled: boolean },
|
|
12
|
+
* cascade: { escalateAfterFailures: number, deescalateAfterPasses: number, minStepsPerTier: number },
|
|
13
|
+
* verifier: { patterns: string[] },
|
|
14
|
+
* distill: { enabled: boolean, maxChars: number, contextLines: number },
|
|
15
|
+
* readDedupe: { enabled: boolean },
|
|
16
|
+
* loopBreaker: { enabled: boolean, threshold: number },
|
|
17
|
+
* ledger: { path: string, pricing: Record<string, Price> },
|
|
18
|
+
* }} complete settings with every default applied.
|
|
19
|
+
*/
|
|
20
|
+
export function normalizeSettings(raw) {
|
|
21
|
+
const tiers = raw?.tiers ?? {}
|
|
22
|
+
// Tiers are optional. Without a large tier, routing stays off and only the other mechanisms run.
|
|
23
|
+
const small = isSetRoute(tiers.small) ? normalizeRoute(tiers.small, 'tiers.small') : undefined
|
|
24
|
+
const large = isSetRoute(tiers.large) ? normalizeRoute(tiers.large, 'tiers.large') : undefined
|
|
25
|
+
|
|
26
|
+
const cascade = {
|
|
27
|
+
escalateAfterFailures: integerAtLeast(raw.cascade?.escalateAfterFailures ?? 1, 1, 'cascade.escalateAfterFailures'),
|
|
28
|
+
deescalateAfterPasses: integerAtLeast(raw.cascade?.deescalateAfterPasses ?? 2, 1, 'cascade.deescalateAfterPasses'),
|
|
29
|
+
minStepsPerTier: integerAtLeast(raw.cascade?.minStepsPerTier ?? 2, 0, 'cascade.minStepsPerTier'),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const patterns = raw.verifier?.patterns?.length ? [...raw.verifier.patterns] : [...DEFAULT_VERIFIER_PATTERNS]
|
|
33
|
+
try {
|
|
34
|
+
compileVerifierPatterns(patterns)
|
|
35
|
+
} catch (error) {
|
|
36
|
+
throw new Error(`greenharness: verifier.patterns has an invalid regular expression: ${String(error)}`)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
tiers: { small, large },
|
|
41
|
+
routing: { enabled: large !== undefined },
|
|
42
|
+
cascade,
|
|
43
|
+
verifier: { patterns },
|
|
44
|
+
distill: {
|
|
45
|
+
enabled: raw.distill?.enabled ?? true,
|
|
46
|
+
maxChars: integerAtLeast(raw.distill?.maxChars ?? 6000, 500, 'distill.maxChars'),
|
|
47
|
+
contextLines: integerAtLeast(raw.distill?.contextLines ?? 2, 0, 'distill.contextLines'),
|
|
48
|
+
},
|
|
49
|
+
readDedupe: { enabled: raw.readDedupe?.enabled ?? true },
|
|
50
|
+
loopBreaker: {
|
|
51
|
+
enabled: raw.loopBreaker?.enabled ?? true,
|
|
52
|
+
threshold: integerAtLeast(raw.loopBreaker?.threshold ?? 3, 2, 'loopBreaker.threshold'),
|
|
53
|
+
},
|
|
54
|
+
ledger: {
|
|
55
|
+
path: raw.ledger?.path ?? '.greenharness/ledger.jsonl',
|
|
56
|
+
pricing: raw.ledger?.pricing ?? {},
|
|
57
|
+
},
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @typedef {{ provider: string, model: string, reasoningEffort: string }} Route
|
|
63
|
+
* @typedef {{ inputPerMTok?: number, cachedInputPerMTok?: number, outputPerMTok?: number }} Price
|
|
64
|
+
*/
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* A tier is set when it names a provider or a model. An empty tier (from a schema default) is unset,
|
|
68
|
+
* and a half-written tier still fails in normalizeRoute.
|
|
69
|
+
* @param {{ provider?: string, model?: string } | undefined} route
|
|
70
|
+
*/
|
|
71
|
+
function isSetRoute(route) {
|
|
72
|
+
return Boolean(route && (route.provider || route.model))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Normalize one tier route.
|
|
77
|
+
* @param {{ provider?: string, model?: string, reasoningEffort?: string } | undefined} route - the configured route.
|
|
78
|
+
* @param {string} label - configuration path, for error messages.
|
|
79
|
+
* @returns {Route} the route; an empty reasoningEffort means the adapter default.
|
|
80
|
+
*/
|
|
81
|
+
function normalizeRoute(route, label) {
|
|
82
|
+
if (!route || !route.provider || !route.model) {
|
|
83
|
+
throw new Error(`greenharness: ${label} needs both \`provider\` and \`model\``)
|
|
84
|
+
}
|
|
85
|
+
return { provider: String(route.provider), model: String(route.model), reasoningEffort: String(route.reasoningEffort ?? '') }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Require an integer at or above a minimum.
|
|
90
|
+
* @param {unknown} value - the configured value.
|
|
91
|
+
* @param {number} min - the smallest accepted value.
|
|
92
|
+
* @param {string} label - configuration path, for error messages.
|
|
93
|
+
* @returns {number} the value.
|
|
94
|
+
*/
|
|
95
|
+
function integerAtLeast(value, min, label) {
|
|
96
|
+
if (!Number.isInteger(value) || value < min) {
|
|
97
|
+
throw new Error(`greenharness: ${label} must be an integer >= ${min} (got ${String(value)})`)
|
|
98
|
+
}
|
|
99
|
+
return value
|
|
100
|
+
}
|
package/src/control.mjs
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The /greenharness slash command: parsing, and the replies it gives. These are pure functions
|
|
3
|
+
* over a host object, so they can be tested without DSH.
|
|
4
|
+
*
|
|
5
|
+
* host = { control: { paused: boolean }, settings, runtime }
|
|
6
|
+
*
|
|
7
|
+
* @module @quadiusai/dsh-greenharness/control
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** Mechanisms that can be switched on or off by name. */
|
|
11
|
+
const TOGGLES = new Set(['distill', 'dedupe', 'loop', 'routing'])
|
|
12
|
+
|
|
13
|
+
export const HELP = [
|
|
14
|
+
'GREENHARNESS: HOW TO USE IT',
|
|
15
|
+
'',
|
|
16
|
+
'It starts saving tokens on its own. These commands let you steer it.',
|
|
17
|
+
'',
|
|
18
|
+
'SEE WHAT IT IS DOING',
|
|
19
|
+
' /greenharness summary: tokens saved, progress bar, models',
|
|
20
|
+
'',
|
|
21
|
+
'CHOOSE MODELS (the cheap one for easy steps, a strong one for hard steps)',
|
|
22
|
+
' /greenharness models list the models you have set up in DSH, numbered',
|
|
23
|
+
' /greenharness large 3 use model number 3 from that list as the strong model',
|
|
24
|
+
' /greenharness large nvidia Nemotron 3 Ultra',
|
|
25
|
+
' or type part of its name, the provider first is optional',
|
|
26
|
+
' /greenharness small 1 set the cheap model (default: whatever DSH already uses)',
|
|
27
|
+
' Add API keys first in DSH: Settings > Models.',
|
|
28
|
+
'',
|
|
29
|
+
'TURN THINGS ON OR OFF',
|
|
30
|
+
' /greenharness on | off resume or pause everything',
|
|
31
|
+
' /greenharness routing on|off switching between the cheap and strong model',
|
|
32
|
+
' /greenharness distill on|off shorten very long command output',
|
|
33
|
+
' /greenharness dedupe on|off replace a repeated file read with a short pointer',
|
|
34
|
+
' /greenharness loop on|off stop a command that keeps failing the same way',
|
|
35
|
+
'',
|
|
36
|
+
'Settings last until DSH restarts.',
|
|
37
|
+
].join('\n')
|
|
38
|
+
|
|
39
|
+
const EFFORTS = new Set(['minimal', 'low', 'medium', 'high', 'xhigh', 'max'])
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Parse the text after `/greenharness`.
|
|
43
|
+
* @param {string} rawInput
|
|
44
|
+
* @returns {{ action: 'status' | 'help' } | { action: 'pause', paused: boolean }
|
|
45
|
+
* | { action: 'toggle', mechanism: string, enabled: boolean }
|
|
46
|
+
* | { action: 'models' } | { action: 'tier', tier: 'small' | 'large', args: string[] }
|
|
47
|
+
* | { action: 'error', message: string }}
|
|
48
|
+
*/
|
|
49
|
+
export function parseCommand(rawInput = '') {
|
|
50
|
+
const [first = 'status', ...args] = String(rawInput).trim().split(/\s+/).filter(Boolean)
|
|
51
|
+
const verb = first.toLowerCase()
|
|
52
|
+
if (verb === 'status' || verb === 'help') {
|
|
53
|
+
return args.length === 0 ? { action: verb } : usage(`/greenharness ${verb}`)
|
|
54
|
+
}
|
|
55
|
+
if (verb === 'on' || verb === 'off') {
|
|
56
|
+
return args.length === 0 ? { action: 'pause', paused: verb === 'off' } : usage(`/greenharness ${verb}`)
|
|
57
|
+
}
|
|
58
|
+
if (TOGGLES.has(verb)) {
|
|
59
|
+
const value = (args[0] ?? '').toLowerCase()
|
|
60
|
+
if (args.length !== 1 || (value !== 'on' && value !== 'off')) return usage(`/greenharness ${verb} on|off`)
|
|
61
|
+
return { action: 'toggle', mechanism: verb, enabled: value === 'on' }
|
|
62
|
+
}
|
|
63
|
+
if (verb === 'models') return args.length === 0 ? { action: 'models' } : usage('/greenharness models')
|
|
64
|
+
if (verb === 'large' || verb === 'small') return { action: 'tier', tier: verb, args }
|
|
65
|
+
return { action: 'error', message: `Unknown command "${first}". Try /greenharness help.` }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Run one command and return the reply DSH shows.
|
|
70
|
+
* @param {string} rawInput
|
|
71
|
+
* @param {{ control: { paused: boolean }, settings: any, runtime: { stats(): Record<string, number> } }} host
|
|
72
|
+
* @returns {Promise<{ kind: 'success' | 'error', text: string }>}
|
|
73
|
+
*/
|
|
74
|
+
export async function runCommand(rawInput, host) {
|
|
75
|
+
const command = parseCommand(rawInput)
|
|
76
|
+
switch (command.action) {
|
|
77
|
+
case 'error': return fail(command.message)
|
|
78
|
+
case 'help': return ok(HELP)
|
|
79
|
+
case 'status': return ok(statusText(host))
|
|
80
|
+
case 'pause':
|
|
81
|
+
host.control.paused = command.paused
|
|
82
|
+
return ok(command.paused ? 'GreenHarness paused. Tool output and model routing are left alone.' : 'GreenHarness resumed.')
|
|
83
|
+
case 'toggle': return toggle(command, host)
|
|
84
|
+
case 'models': return showModels(host)
|
|
85
|
+
case 'tier': return setTier(command, host)
|
|
86
|
+
default: return fail('Unknown command. Try /greenharness help.')
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The status reply: switches, models, and what this process has done so far.
|
|
92
|
+
* @param {{ control: { paused: boolean }, settings: any, runtime: { stats(): Record<string, number> } }} host
|
|
93
|
+
*/
|
|
94
|
+
export function statusText({ control, settings, runtime }) {
|
|
95
|
+
const stats = runtime.stats()
|
|
96
|
+
const count = key => stats[key] ?? 0
|
|
97
|
+
const describe = route => (route ? `${route.provider}/${route.model}` : 'not set')
|
|
98
|
+
const small_ = settings.tiers.small
|
|
99
|
+
? describe(settings.tiers.small)
|
|
100
|
+
: 'the model DSH uses (recorded at the first routed call)'
|
|
101
|
+
const savedTokens = Math.round(count('charsSaved') / 4)
|
|
102
|
+
const smallTok = count('smallTokens')
|
|
103
|
+
const large = count('largeTokens')
|
|
104
|
+
const spent = smallTok + large
|
|
105
|
+
const cheapShare = spent === 0 ? 0 : smallTok / spent
|
|
106
|
+
const lines = [
|
|
107
|
+
`GreenHarness is ${control.paused ? 'PAUSED' : 'ON'}`,
|
|
108
|
+
'',
|
|
109
|
+
`Tokens saved so far: about ${savedTokens.toLocaleString('en-US')}`,
|
|
110
|
+
` (long outputs shortened: ${count('tool:distill')}, repeated reads replaced: ${count('tool:dedupe')}, loops stopped: ${count('loop-break')})`,
|
|
111
|
+
`Work done by the cheap model: ${bar(cheapShare)} ${Math.round(cheapShare * 100)}%`,
|
|
112
|
+
`Model calls: ${count('request')}. Switches between models: ${count('route')}.`,
|
|
113
|
+
]
|
|
114
|
+
if (count('costUsd') > 0) lines.push(`Cost so far: $${count('costUsd').toFixed(4)}`)
|
|
115
|
+
lines.push(
|
|
116
|
+
'',
|
|
117
|
+
`Models: small = ${small_}, large = ${describe(settings.tiers.large)}. Routing ${onOff(settings.routing.enabled)}.`,
|
|
118
|
+
`Features: shorten output ${onOff(settings.distill.enabled)}, repeated reads ${onOff(settings.readDedupe.enabled)}, loop stop ${onOff(settings.loopBreaker.enabled)}.`,
|
|
119
|
+
...(settings.tiers.large ? [] : ['Next step: type /greenharness models, then /greenharness large <number> to add a strong model.']),
|
|
120
|
+
'All commands: /greenharness help',
|
|
121
|
+
)
|
|
122
|
+
return lines.join('\n')
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** A 20-cell text progress bar. @param {number} fraction 0..1 */
|
|
126
|
+
export function bar(fraction) {
|
|
127
|
+
const filled = Math.max(0, Math.min(20, Math.round(fraction * 20)))
|
|
128
|
+
return '[' + '█'.repeat(filled) + '░'.repeat(20 - filled) + ']'
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** @param {{ mechanism: string, enabled: boolean }} command @param {{ settings: any }} host */
|
|
132
|
+
function toggle({ mechanism, enabled }, { settings }) {
|
|
133
|
+
if (mechanism === 'routing') {
|
|
134
|
+
if (enabled && settings.tiers.large === undefined) {
|
|
135
|
+
return fail('Routing needs a large model first: /greenharness large <provider> <model>')
|
|
136
|
+
}
|
|
137
|
+
settings.routing.enabled = enabled
|
|
138
|
+
return ok(`Routing ${onOff(enabled)}.`)
|
|
139
|
+
}
|
|
140
|
+
const owner = { distill: settings.distill, dedupe: settings.readDedupe, loop: settings.loopBreaker }[mechanism]
|
|
141
|
+
owner.enabled = enabled
|
|
142
|
+
return ok(`${mechanism === 'loop' ? 'Loop breaker' : mechanism[0].toUpperCase() + mechanism.slice(1)} ${onOff(enabled)}.`)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const norm = text => String(text).toLowerCase().replace(/[^a-z0-9]/g, '')
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Every model DSH knows about, numbered. Providers that cannot list models are skipped quietly.
|
|
149
|
+
* @param {{ llm?: any }} host
|
|
150
|
+
* @returns {Promise<{ n: number, provider: string, providerName: string, id: string, name: string }[]>}
|
|
151
|
+
*/
|
|
152
|
+
async function loadListing(host) {
|
|
153
|
+
const entries = []
|
|
154
|
+
if (host.llm === undefined) return entries
|
|
155
|
+
for (const provider of host.llm.listProviders()) {
|
|
156
|
+
let models = []
|
|
157
|
+
try {
|
|
158
|
+
models = await host.llm.listModels(provider.id)
|
|
159
|
+
} catch {
|
|
160
|
+
continue
|
|
161
|
+
}
|
|
162
|
+
for (const model of models) {
|
|
163
|
+
entries.push({ n: entries.length + 1, provider: provider.id, providerName: provider.name, id: model.id, name: model.name })
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
host.listing = entries
|
|
167
|
+
return entries
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const row = e => ` ${String(e.n).padStart(2)}. ${e.name} (${e.providerName}, id ${e.id})`
|
|
171
|
+
|
|
172
|
+
/** @param {{ llm?: any, listing?: any[] }} host */
|
|
173
|
+
async function showModels(host) {
|
|
174
|
+
const entries = await loadListing(host)
|
|
175
|
+
if (entries.length === 0) {
|
|
176
|
+
return ok([
|
|
177
|
+
'DSH has no models to list yet.',
|
|
178
|
+
'Add your API key and models first: open Settings > Models in DSH, then run /greenharness models again.',
|
|
179
|
+
].join('\n'))
|
|
180
|
+
}
|
|
181
|
+
return ok([
|
|
182
|
+
'MODELS YOU CAN CHOOSE FROM',
|
|
183
|
+
...entries.map(row),
|
|
184
|
+
'',
|
|
185
|
+
'Pick the strong one: /greenharness large <number> e.g. /greenharness large 3',
|
|
186
|
+
'Pick the cheap one: /greenharness small <number>',
|
|
187
|
+
].join('\n'))
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Work out which model the user meant: a list number, or words from its name with an optional
|
|
192
|
+
* provider first and an optional reasoning effort last.
|
|
193
|
+
* @param {string[]} args
|
|
194
|
+
* @param {{ llm?: any, listing?: any[] }} host
|
|
195
|
+
* @returns {Promise<{ route: { provider: string, model: string, reasoningEffort: string }, label: string } | { error: string }>}
|
|
196
|
+
*/
|
|
197
|
+
async function resolveModel(args, host) {
|
|
198
|
+
let effort = ''
|
|
199
|
+
if (args.length > 1 && EFFORTS.has(args[args.length - 1].toLowerCase())) effort = args[args.length - 1].toLowerCase()
|
|
200
|
+
const words = effort === '' ? args : args.slice(0, -1)
|
|
201
|
+
if (words.length === 0) return { error: 'Which model? Run /greenharness models to see the list, then use its number.' }
|
|
202
|
+
const entries = host.listing?.length ? host.listing : await loadListing(host)
|
|
203
|
+
const pick = e => ({ route: { provider: e.provider, model: e.id, reasoningEffort: effort }, label: `${e.name} (${e.providerName})` })
|
|
204
|
+
if (words.length === 1 && /^\d+$/.test(words[0])) {
|
|
205
|
+
const hit = entries.find(e => e.n === Number(words[0]))
|
|
206
|
+
return hit ? pick(hit) : { error: `There is no model number ${words[0]}. Run /greenharness models to see the list.` }
|
|
207
|
+
}
|
|
208
|
+
if (host.llm === undefined) {
|
|
209
|
+
if (words.length < 2) return { error: 'Use: /greenharness large <provider> <model>' }
|
|
210
|
+
return { route: { provider: words[0], model: words.slice(1).join(' '), reasoningEffort: effort }, label: `${words[0]}/${words.slice(1).join(' ')}` }
|
|
211
|
+
}
|
|
212
|
+
const providers = host.llm.listProviders()
|
|
213
|
+
const owner = providers.find(p => norm(p.id) === norm(words[0]) || norm(p.name) === norm(words[0]))
|
|
214
|
+
const rest = owner ? words.slice(1) : words
|
|
215
|
+
const pool = owner ? entries.filter(e => e.provider === owner.id) : entries
|
|
216
|
+
const query = norm(rest.join(''))
|
|
217
|
+
if (query === '') return { error: `Which model from ${owner.name}? Run /greenharness models to see the list.` }
|
|
218
|
+
const exact = pool.filter(e => norm(e.id) === query || norm(e.name) === query)
|
|
219
|
+
const near = exact.length > 0 ? exact : pool.filter(e => norm(e.id).includes(query) || norm(e.name).includes(query))
|
|
220
|
+
if (near.length === 1) return pick(near[0])
|
|
221
|
+
if (near.length > 1) {
|
|
222
|
+
return { error: ['That matches several models. Pick one by number:', ...near.map(row)].join('\n') }
|
|
223
|
+
}
|
|
224
|
+
if (owner && rest.length === 1) {
|
|
225
|
+
return { route: { provider: owner.id, model: rest[0], reasoningEffort: effort }, label: `${rest[0]} (${owner.name}, typed id)` }
|
|
226
|
+
}
|
|
227
|
+
return { error: `I could not find a model matching "${rest.join(' ')}". Run /greenharness models to see what DSH has. Add keys and models in Settings > Models.` }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** @param {{ tier: 'small' | 'large', args: string[] }} command @param {{ settings: any, llm?: any, listing?: any[] }} host */
|
|
231
|
+
async function setTier({ tier, args }, host) {
|
|
232
|
+
if (args.length === 0) return showModels(host)
|
|
233
|
+
const found = await resolveModel(args, host)
|
|
234
|
+
if ('error' in found) return fail(found.error)
|
|
235
|
+
host.settings.tiers[tier] = found.route
|
|
236
|
+
if (tier === 'large') {
|
|
237
|
+
host.settings.routing.enabled = true
|
|
238
|
+
return ok(`Strong model set to ${found.label}.\nSwitching is ON: GreenHarness will use it when the cheap model keeps failing tests, then go back.\nType /greenharness to watch it work.`)
|
|
239
|
+
}
|
|
240
|
+
return ok(`Cheap model set to ${found.label}.`)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** @param {string} message */
|
|
244
|
+
function usage(message) {
|
|
245
|
+
return { action: 'error', message: `Usage: ${message}` }
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** @param {boolean} value */
|
|
249
|
+
function onOff(value) {
|
|
250
|
+
return value ? 'on' : 'off'
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** @param {string} text */
|
|
254
|
+
function ok(text) {
|
|
255
|
+
return { kind: 'success', text }
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** @param {string} text */
|
|
259
|
+
function fail(text) {
|
|
260
|
+
return { kind: 'error', text }
|
|
261
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal control sequences: CSI (colors, cursor moves), OSC (titles, hyperlinks) and
|
|
3
|
+
* single-character escapes. Removing them makes line counts and fingerprints reflect
|
|
4
|
+
* visible text only.
|
|
5
|
+
*/
|
|
6
|
+
const ANSI_RE = /\u001b\[[0-?]*[ -/]*[@-~]|\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)|\u001b[@-Z\\-_]/g
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Remove terminal control sequences from text.
|
|
10
|
+
* @param {string} text - raw terminal output.
|
|
11
|
+
* @returns {string} the same text without control sequences.
|
|
12
|
+
*/
|
|
13
|
+
export function stripAnsi(text) {
|
|
14
|
+
return text.replace(ANSI_RE, '')
|
|
15
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-tier model cascade: a small model runs by default; a verifier failure moves the next
|
|
3
|
+
* step to the large model; verifier passes move back once they are stable. Pure state
|
|
4
|
+
* machine: callers read the returned change and apply it to the agent.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Default policy: escalate on the first failure, return after two passes, and stay at least two steps per tier. */
|
|
8
|
+
export const DEFAULT_CASCADE_POLICY = Object.freeze({
|
|
9
|
+
escalateAfterFailures: 1,
|
|
10
|
+
deescalateAfterPasses: 2,
|
|
11
|
+
minStepsPerTier: 2,
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Create cascade state for one session.
|
|
16
|
+
* @param {Partial<typeof DEFAULT_CASCADE_POLICY>} [policy] - overrides for the default policy.
|
|
17
|
+
* @returns {{ policy: typeof DEFAULT_CASCADE_POLICY, tier: 'small' | 'large', stepsInTier: number, consecutiveFailures: number, consecutivePasses: number }}
|
|
18
|
+
*/
|
|
19
|
+
export function createCascade(policy = {}) {
|
|
20
|
+
return {
|
|
21
|
+
policy: { ...DEFAULT_CASCADE_POLICY, ...policy },
|
|
22
|
+
tier: 'small',
|
|
23
|
+
stepsInTier: 0,
|
|
24
|
+
consecutiveFailures: 0,
|
|
25
|
+
consecutivePasses: 0,
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Count one completed model step in the current tier.
|
|
31
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
32
|
+
*/
|
|
33
|
+
export function recordStep(cascade) {
|
|
34
|
+
cascade.stepsInTier += 1
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Apply a verifier outcome to the cascade.
|
|
39
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
40
|
+
* @param {boolean} passed - whether the verifier command passed.
|
|
41
|
+
* @returns {{ from: string, to: string, reason: string } | null} the tier change, or null when the tier stays.
|
|
42
|
+
*/
|
|
43
|
+
export function recordVerification(cascade, passed) {
|
|
44
|
+
if (passed) {
|
|
45
|
+
cascade.consecutiveFailures = 0
|
|
46
|
+
cascade.consecutivePasses += 1
|
|
47
|
+
const stable = cascade.consecutivePasses >= cascade.policy.deescalateAfterPasses
|
|
48
|
+
&& cascade.stepsInTier >= cascade.policy.minStepsPerTier
|
|
49
|
+
return cascade.tier === 'large' && stable ? switchTier(cascade, 'small', 'verifier-passed') : null
|
|
50
|
+
}
|
|
51
|
+
cascade.consecutivePasses = 0
|
|
52
|
+
cascade.consecutiveFailures += 1
|
|
53
|
+
const escalate = cascade.consecutiveFailures >= cascade.policy.escalateAfterFailures
|
|
54
|
+
return cascade.tier === 'small' && escalate ? switchTier(cascade, 'large', 'verifier-failed') : null
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Force the large tier, for example when the loop breaker fires.
|
|
59
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
60
|
+
* @param {string} reason - audit reason recorded with the change.
|
|
61
|
+
* @returns {{ from: string, to: string, reason: string } | null} the tier change, or null when already large.
|
|
62
|
+
*/
|
|
63
|
+
export function escalate(cascade, reason) {
|
|
64
|
+
return cascade.tier === 'large' ? null : switchTier(cascade, 'large', reason)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Move to another tier and reset the counters that belong to the old tier.
|
|
69
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
70
|
+
* @param {'small' | 'large'} tier - the destination tier.
|
|
71
|
+
* @param {string} reason - audit reason.
|
|
72
|
+
* @returns {{ from: string, to: string, reason: string }} the change.
|
|
73
|
+
*/
|
|
74
|
+
function switchTier(cascade, tier, reason) {
|
|
75
|
+
const from = cascade.tier
|
|
76
|
+
cascade.tier = tier
|
|
77
|
+
cascade.stepsInTier = 0
|
|
78
|
+
cascade.consecutiveFailures = 0
|
|
79
|
+
cascade.consecutivePasses = 0
|
|
80
|
+
return { from, to: tier, reason }
|
|
81
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { stripAnsi } from './ansi.mjs'
|
|
2
|
+
import { FAIL_RE, FRAME_RE, LIB_FRAME_RE, PROGRESS_RE, SAFETY_RE, SUMMARY_RE, ZERO_RE } from './patterns.mjs'
|
|
3
|
+
|
|
4
|
+
/** Minimum size reduction (output at most this fraction of the input) before a distillation is used. */
|
|
5
|
+
const MIN_REDUCTION_RATIO = 0.9
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Classify one line for distillation.
|
|
9
|
+
* @param {string} line - one line with control sequences removed.
|
|
10
|
+
* @returns {'summary' | 'fail' | 'safety' | 'lib-frame' | 'frame' | 'plain'} the line's kind.
|
|
11
|
+
*/
|
|
12
|
+
function classify(line) {
|
|
13
|
+
if (ZERO_RE.test(line)) return 'summary'
|
|
14
|
+
if (FAIL_RE.test(line)) return 'fail'
|
|
15
|
+
if (SAFETY_RE.test(line)) return 'safety'
|
|
16
|
+
if (FRAME_RE.test(line)) return LIB_FRAME_RE.test(line) ? 'lib-frame' : 'frame'
|
|
17
|
+
if (SUMMARY_RE.test(line)) return 'summary'
|
|
18
|
+
return 'plain'
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Keep the signal in one shell or tool output and drop the rest, without changing any kept line.
|
|
23
|
+
*
|
|
24
|
+
* Kept, in order: failure, summary and safety lines; user-code stack frames and plain lines
|
|
25
|
+
* within `contextLines` of a failure; the first and last three lines. Dropped: progress
|
|
26
|
+
* frames, library stack frames, passing-test noise. Runs of three or more identical kept
|
|
27
|
+
* lines collapse to one line with a count. Output above `maxChars` keeps its head and tail.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} raw - tool output as the model would read it.
|
|
30
|
+
* @param {{ maxChars?: number, contextLines?: number }} [options] - size budget and window around failures.
|
|
31
|
+
* @returns {{ text: string, changed: boolean, stats: { inChars: number, outChars: number, droppedLines: number, droppedFrames: number, collapsedRepeats: number, failLines: number, elidedChars: number } }}
|
|
32
|
+
* `changed` is true only when the distilled text is at most 90% of the input and differs from it.
|
|
33
|
+
*/
|
|
34
|
+
export function distillOutput(raw, options = {}) {
|
|
35
|
+
const maxChars = options.maxChars ?? 6000
|
|
36
|
+
const contextLines = options.contextLines ?? 2
|
|
37
|
+
const input = String(raw ?? '')
|
|
38
|
+
const stats = {
|
|
39
|
+
inChars: input.length, outChars: 0, droppedLines: 0, droppedFrames: 0,
|
|
40
|
+
collapsedRepeats: 0, failLines: 0, elidedChars: 0,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let lines = stripAnsi(input).replace(/\r\n?/g, '\n').split('\n')
|
|
44
|
+
lines = lines.filter((line) => {
|
|
45
|
+
if (line.trim() !== '' && PROGRESS_RE.test(line)) {
|
|
46
|
+
stats.droppedLines += 1
|
|
47
|
+
return false
|
|
48
|
+
}
|
|
49
|
+
return true
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
const kinds = lines.map(classify)
|
|
53
|
+
stats.failLines = kinds.filter(kind => kind === 'fail').length
|
|
54
|
+
|
|
55
|
+
const nearFailure = new Array(lines.length).fill(false)
|
|
56
|
+
kinds.forEach((kind, index) => {
|
|
57
|
+
if (kind !== 'fail') return
|
|
58
|
+
const from = Math.max(0, index - contextLines)
|
|
59
|
+
const to = Math.min(lines.length - 1, index + contextLines)
|
|
60
|
+
for (let j = from; j <= to; j += 1) nearFailure[j] = true
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
const keep = lines.map((_, index) => {
|
|
64
|
+
const kind = kinds[index]
|
|
65
|
+
if (kind === 'fail' || kind === 'summary' || kind === 'safety') return true
|
|
66
|
+
if (kind === 'lib-frame') return false
|
|
67
|
+
if (index < 3 || index >= lines.length - 3) return true
|
|
68
|
+
return nearFailure[index]
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
const emitted = []
|
|
72
|
+
let gap = 0
|
|
73
|
+
let runText = null
|
|
74
|
+
let runCount = 0
|
|
75
|
+
const flushRun = () => {
|
|
76
|
+
if (runText === null) return
|
|
77
|
+
if (runCount >= 3) {
|
|
78
|
+
emitted.push(`${runText} (×${runCount})`)
|
|
79
|
+
stats.collapsedRepeats += runCount - 1
|
|
80
|
+
} else {
|
|
81
|
+
for (let k = 0; k < runCount; k += 1) emitted.push(runText)
|
|
82
|
+
}
|
|
83
|
+
runText = null
|
|
84
|
+
runCount = 0
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
88
|
+
if (!keep[index]) {
|
|
89
|
+
if (kinds[index] === 'lib-frame') stats.droppedFrames += 1
|
|
90
|
+
if (gap === 0) flushRun()
|
|
91
|
+
gap += 1
|
|
92
|
+
continue
|
|
93
|
+
}
|
|
94
|
+
if (gap > 0) {
|
|
95
|
+
emitted.push(`[… ${gap} line${gap === 1 ? '' : 's'} omitted]`)
|
|
96
|
+
gap = 0
|
|
97
|
+
}
|
|
98
|
+
const text = lines[index]
|
|
99
|
+
if (text === runText) {
|
|
100
|
+
runCount += 1
|
|
101
|
+
} else {
|
|
102
|
+
flushRun()
|
|
103
|
+
runText = text
|
|
104
|
+
runCount = 1
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
flushRun()
|
|
108
|
+
if (gap > 0) emitted.push(`[… ${gap} line${gap === 1 ? '' : 's'} omitted]`)
|
|
109
|
+
|
|
110
|
+
let text = emitted.join('\n').replace(/[ \t]+$/gm, '')
|
|
111
|
+
if (text.length > maxChars) {
|
|
112
|
+
const headBudget = Math.floor(maxChars * 0.4)
|
|
113
|
+
const tailBudget = Math.max(0, maxChars - headBudget - 80)
|
|
114
|
+
const headSlice = text.slice(0, headBudget)
|
|
115
|
+
const headCut = headSlice.lastIndexOf('\n')
|
|
116
|
+
const headText = headCut > 0 ? headSlice.slice(0, headCut) : headSlice
|
|
117
|
+
const tailSlice = text.slice(text.length - tailBudget)
|
|
118
|
+
const tailCut = tailSlice.indexOf('\n')
|
|
119
|
+
const tailText = tailCut >= 0 ? tailSlice.slice(tailCut + 1) : tailSlice
|
|
120
|
+
const omitted = text.length - headText.length - tailText.length
|
|
121
|
+
stats.elidedChars = omitted
|
|
122
|
+
text = `${headText}\n[… ${omitted} chars omitted …]\n${tailText}`
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
stats.outChars = text.length
|
|
126
|
+
const changed = text !== input && text.length <= input.length * MIN_REDUCTION_RATIO
|
|
127
|
+
return { text, changed, stats }
|
|
128
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync } from 'node:fs'
|
|
2
|
+
import { dirname } from 'node:path'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Cost and decision ledger: one JSON object per line, appended as events happen.
|
|
6
|
+
* Record types: `request` (provider usage per model step), `tool` (per tool result, with the
|
|
7
|
+
* distillation or dedupe action), `route` (tier change), `loop-break` (denied command).
|
|
8
|
+
*
|
|
9
|
+
* @param {string} path - JSONL file to append to; an empty value disables the ledger.
|
|
10
|
+
* @returns {{ path: string, write(record: Record<string, unknown>): void }}
|
|
11
|
+
*/
|
|
12
|
+
export function createLedger(path) {
|
|
13
|
+
if (!path) return { path: '', write() {} }
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
15
|
+
return {
|
|
16
|
+
path,
|
|
17
|
+
write(record) {
|
|
18
|
+
appendFileSync(path, `${JSON.stringify({ ts: new Date().toISOString(), ...record })}\n`)
|
|
19
|
+
},
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { normalizeCommand } from './verify.mjs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Loop guard: counts consecutive runs of the same command that fail with the same failure
|
|
5
|
+
* fingerprint. A run with a different fingerprint is progress and restarts the count; a
|
|
6
|
+
* passing run clears it; any workspace edit clears every count.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Create an empty loop-guard state for one session.
|
|
11
|
+
* @returns {{ streaks: Map<string, { count: number, fingerprint: string }> }}
|
|
12
|
+
*/
|
|
13
|
+
export function createLoopGuard() {
|
|
14
|
+
return { streaks: new Map() }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Record the outcome of an executed command.
|
|
19
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state (mutated).
|
|
20
|
+
* @param {string} command - the command text.
|
|
21
|
+
* @param {{ passed: boolean, fingerprint: string | null }} outcome - from `classifyOutcome`.
|
|
22
|
+
* @returns {number} the current consecutive count for this command (0 after a pass).
|
|
23
|
+
*/
|
|
24
|
+
export function recordCommandOutcome(guard, command, outcome) {
|
|
25
|
+
const key = normalizeCommand(command)
|
|
26
|
+
if (outcome.passed || outcome.fingerprint === null) {
|
|
27
|
+
guard.streaks.delete(key)
|
|
28
|
+
return 0
|
|
29
|
+
}
|
|
30
|
+
const previous = guard.streaks.get(key)
|
|
31
|
+
const count = previous !== undefined && previous.fingerprint === outcome.fingerprint ? previous.count + 1 : 1
|
|
32
|
+
guard.streaks.set(key, { count, fingerprint: outcome.fingerprint })
|
|
33
|
+
return count
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Consecutive same-fingerprint failures for one command.
|
|
38
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state.
|
|
39
|
+
* @param {string} command - the command text.
|
|
40
|
+
* @returns {number} the count, 0 when the command has no active streak.
|
|
41
|
+
*/
|
|
42
|
+
export function streakOf(guard, command) {
|
|
43
|
+
return guard.streaks.get(normalizeCommand(command))?.count ?? 0
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Clear every streak after a workspace change, because a failing command may pass now.
|
|
48
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state (mutated).
|
|
49
|
+
*/
|
|
50
|
+
export function noteWorkspaceChange(guard) {
|
|
51
|
+
guard.streaks.clear()
|
|
52
|
+
}
|