@quadiusai/dsh-greenharness 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GETTING-STARTED.md +76 -0
- package/README.md +147 -2
- package/bench/BREV.md +127 -0
- package/bench/lib/patch.mjs +60 -0
- package/bench/mock-openai-server.mjs +210 -0
- package/bench/presets/mock-chat-template.json +40 -0
- package/bench/presets/mock.json +32 -0
- package/bench/presets/nemo-brev.json +40 -0
- package/bench/run-case.mjs +242 -0
- package/bench/run-mock-suite.mjs +85 -0
- package/bench/scenarios/loop-stuck.json +11 -0
- package/bench/scenarios/subtract-fix.json +12 -0
- package/bench/tasks/subtract-bug/TASK.md +2 -0
- package/bench/tasks/subtract-bug/package.json +9 -0
- package/bench/tasks/subtract-bug/src/math.mjs +8 -0
- package/bench/tasks/subtract-bug/test/math.test.mjs +11 -0
- package/bench/tasks/subtract-bug/test/noise.test.mjs +9 -0
- package/cordis.patch.yml +4 -0
- package/package.json +42 -4
- package/src/config.mjs +100 -0
- package/src/control.mjs +160 -0
- package/src/core/ansi.mjs +15 -0
- package/src/core/cascade.mjs +81 -0
- package/src/core/distill.mjs +128 -0
- package/src/core/ledger.mjs +21 -0
- package/src/core/loop-guard.mjs +52 -0
- package/src/core/patterns.mjs +26 -0
- package/src/core/pricing.mjs +23 -0
- package/src/core/read-index.mjs +74 -0
- package/src/core/route.mjs +41 -0
- package/src/core/verify.mjs +89 -0
- package/src/index.mjs +232 -0
- package/src/runtime.mjs +297 -0
package/src/config.mjs
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { DEFAULT_VERIFIER_PATTERNS, compileVerifierPatterns } from './core/verify.mjs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Validate and complete the plugin configuration. Misconfiguration fails at load: a tier
|
|
5
|
+
* without a provider and model, a non-integer threshold, or an invalid verifier pattern throws
|
|
6
|
+
* here rather than silently changing routing later.
|
|
7
|
+
*
|
|
8
|
+
* @param {Record<string, any>} raw - the configuration DSH validated against the plugin schema.
|
|
9
|
+
* @returns {{
|
|
10
|
+
* tiers: { small?: Route, large?: Route },
|
|
11
|
+
* routing: { enabled: boolean },
|
|
12
|
+
* cascade: { escalateAfterFailures: number, deescalateAfterPasses: number, minStepsPerTier: number },
|
|
13
|
+
* verifier: { patterns: string[] },
|
|
14
|
+
* distill: { enabled: boolean, maxChars: number, contextLines: number },
|
|
15
|
+
* readDedupe: { enabled: boolean },
|
|
16
|
+
* loopBreaker: { enabled: boolean, threshold: number },
|
|
17
|
+
* ledger: { path: string, pricing: Record<string, Price> },
|
|
18
|
+
* }} complete settings with every default applied.
|
|
19
|
+
*/
|
|
20
|
+
export function normalizeSettings(raw) {
|
|
21
|
+
const tiers = raw?.tiers ?? {}
|
|
22
|
+
// Tiers are optional. Without a large tier, routing stays off and only the other mechanisms run.
|
|
23
|
+
const small = isSetRoute(tiers.small) ? normalizeRoute(tiers.small, 'tiers.small') : undefined
|
|
24
|
+
const large = isSetRoute(tiers.large) ? normalizeRoute(tiers.large, 'tiers.large') : undefined
|
|
25
|
+
|
|
26
|
+
const cascade = {
|
|
27
|
+
escalateAfterFailures: integerAtLeast(raw.cascade?.escalateAfterFailures ?? 1, 1, 'cascade.escalateAfterFailures'),
|
|
28
|
+
deescalateAfterPasses: integerAtLeast(raw.cascade?.deescalateAfterPasses ?? 2, 1, 'cascade.deescalateAfterPasses'),
|
|
29
|
+
minStepsPerTier: integerAtLeast(raw.cascade?.minStepsPerTier ?? 2, 0, 'cascade.minStepsPerTier'),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const patterns = raw.verifier?.patterns?.length ? [...raw.verifier.patterns] : [...DEFAULT_VERIFIER_PATTERNS]
|
|
33
|
+
try {
|
|
34
|
+
compileVerifierPatterns(patterns)
|
|
35
|
+
} catch (error) {
|
|
36
|
+
throw new Error(`greenharness: verifier.patterns has an invalid regular expression: ${String(error)}`)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
tiers: { small, large },
|
|
41
|
+
routing: { enabled: large !== undefined },
|
|
42
|
+
cascade,
|
|
43
|
+
verifier: { patterns },
|
|
44
|
+
distill: {
|
|
45
|
+
enabled: raw.distill?.enabled ?? true,
|
|
46
|
+
maxChars: integerAtLeast(raw.distill?.maxChars ?? 6000, 500, 'distill.maxChars'),
|
|
47
|
+
contextLines: integerAtLeast(raw.distill?.contextLines ?? 2, 0, 'distill.contextLines'),
|
|
48
|
+
},
|
|
49
|
+
readDedupe: { enabled: raw.readDedupe?.enabled ?? true },
|
|
50
|
+
loopBreaker: {
|
|
51
|
+
enabled: raw.loopBreaker?.enabled ?? true,
|
|
52
|
+
threshold: integerAtLeast(raw.loopBreaker?.threshold ?? 3, 2, 'loopBreaker.threshold'),
|
|
53
|
+
},
|
|
54
|
+
ledger: {
|
|
55
|
+
path: raw.ledger?.path ?? '.greenharness/ledger.jsonl',
|
|
56
|
+
pricing: raw.ledger?.pricing ?? {},
|
|
57
|
+
},
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @typedef {{ provider: string, model: string, reasoningEffort: string }} Route
|
|
63
|
+
* @typedef {{ inputPerMTok?: number, cachedInputPerMTok?: number, outputPerMTok?: number }} Price
|
|
64
|
+
*/
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* A tier is set when it names a provider or a model. An empty tier (from a schema default) is unset,
|
|
68
|
+
* and a half-written tier still fails in normalizeRoute.
|
|
69
|
+
* @param {{ provider?: string, model?: string } | undefined} route
|
|
70
|
+
*/
|
|
71
|
+
function isSetRoute(route) {
|
|
72
|
+
return Boolean(route && (route.provider || route.model))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Normalize one tier route.
|
|
77
|
+
* @param {{ provider?: string, model?: string, reasoningEffort?: string } | undefined} route - the configured route.
|
|
78
|
+
* @param {string} label - configuration path, for error messages.
|
|
79
|
+
* @returns {Route} the route; an empty reasoningEffort means the adapter default.
|
|
80
|
+
*/
|
|
81
|
+
function normalizeRoute(route, label) {
|
|
82
|
+
if (!route || !route.provider || !route.model) {
|
|
83
|
+
throw new Error(`greenharness: ${label} needs both \`provider\` and \`model\``)
|
|
84
|
+
}
|
|
85
|
+
return { provider: String(route.provider), model: String(route.model), reasoningEffort: String(route.reasoningEffort ?? '') }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Require an integer at or above a minimum.
|
|
90
|
+
* @param {unknown} value - the configured value.
|
|
91
|
+
* @param {number} min - the smallest accepted value.
|
|
92
|
+
* @param {string} label - configuration path, for error messages.
|
|
93
|
+
* @returns {number} the value.
|
|
94
|
+
*/
|
|
95
|
+
function integerAtLeast(value, min, label) {
|
|
96
|
+
if (!Number.isInteger(value) || value < min) {
|
|
97
|
+
throw new Error(`greenharness: ${label} must be an integer >= ${min} (got ${String(value)})`)
|
|
98
|
+
}
|
|
99
|
+
return value
|
|
100
|
+
}
|
package/src/control.mjs
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The /greenharness slash command: parsing, and the replies it gives. These are pure functions
|
|
3
|
+
* over a host object, so they can be tested without DSH.
|
|
4
|
+
*
|
|
5
|
+
* host = { control: { paused: boolean }, settings, runtime }
|
|
6
|
+
*
|
|
7
|
+
* @module @quadiusai/dsh-greenharness/control
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** Mechanisms that can be switched on or off by name. */
|
|
11
|
+
const TOGGLES = new Set(['distill', 'dedupe', 'loop', 'routing'])
|
|
12
|
+
|
|
13
|
+
export const HELP = [
|
|
14
|
+
'GreenHarness commands (settings last until DSH restarts):',
|
|
15
|
+
' /greenharness status: what is on, and what it has done so far',
|
|
16
|
+
' /greenharness on | off resume or pause every mechanism',
|
|
17
|
+
' /greenharness large <provider> <model> [effort]',
|
|
18
|
+
' turn on model routing with this large model',
|
|
19
|
+
' /greenharness small <provider> <model> [effort]',
|
|
20
|
+
' set the small model (default: the model DSH uses)',
|
|
21
|
+
' /greenharness routing on | off',
|
|
22
|
+
' /greenharness distill on | off shorten long command output',
|
|
23
|
+
' /greenharness dedupe on | off replace a repeated file read with a pointer',
|
|
24
|
+
' /greenharness loop on | off block a repeated failing command',
|
|
25
|
+
' /greenharness help',
|
|
26
|
+
].join('\n')
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Parse the text after `/greenharness`.
|
|
30
|
+
* @param {string} rawInput
|
|
31
|
+
* @returns {{ action: 'status' | 'help' } | { action: 'pause', paused: boolean }
|
|
32
|
+
* | { action: 'toggle', mechanism: string, enabled: boolean }
|
|
33
|
+
* | { action: 'tier', tier: 'small' | 'large', route: { provider: string, model: string, reasoningEffort: string } }
|
|
34
|
+
* | { action: 'error', message: string }}
|
|
35
|
+
*/
|
|
36
|
+
export function parseCommand(rawInput = '') {
|
|
37
|
+
const [first = 'status', ...args] = String(rawInput).trim().split(/\s+/).filter(Boolean)
|
|
38
|
+
const verb = first.toLowerCase()
|
|
39
|
+
if (verb === 'status' || verb === 'help') {
|
|
40
|
+
return args.length === 0 ? { action: verb } : usage(`/greenharness ${verb}`)
|
|
41
|
+
}
|
|
42
|
+
if (verb === 'on' || verb === 'off') {
|
|
43
|
+
return args.length === 0 ? { action: 'pause', paused: verb === 'off' } : usage(`/greenharness ${verb}`)
|
|
44
|
+
}
|
|
45
|
+
if (TOGGLES.has(verb)) {
|
|
46
|
+
const value = (args[0] ?? '').toLowerCase()
|
|
47
|
+
if (args.length !== 1 || (value !== 'on' && value !== 'off')) return usage(`/greenharness ${verb} on|off`)
|
|
48
|
+
return { action: 'toggle', mechanism: verb, enabled: value === 'on' }
|
|
49
|
+
}
|
|
50
|
+
if (verb === 'large' || verb === 'small') {
|
|
51
|
+
if (args.length < 2 || args.length > 3) return usage(`/greenharness ${verb} <provider> <model> [effort]`)
|
|
52
|
+
return { action: 'tier', tier: verb, route: { provider: args[0], model: args[1], reasoningEffort: args[2] ?? '' } }
|
|
53
|
+
}
|
|
54
|
+
return { action: 'error', message: `Unknown command "${first}". Try /greenharness help.` }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Run one command and return the reply DSH shows.
|
|
59
|
+
* @param {string} rawInput
|
|
60
|
+
* @param {{ control: { paused: boolean }, settings: any, runtime: { stats(): Record<string, number> } }} host
|
|
61
|
+
* @returns {{ kind: 'success' | 'error', text: string }}
|
|
62
|
+
*/
|
|
63
|
+
export function runCommand(rawInput, host) {
|
|
64
|
+
const command = parseCommand(rawInput)
|
|
65
|
+
switch (command.action) {
|
|
66
|
+
case 'error': return fail(command.message)
|
|
67
|
+
case 'help': return ok(HELP)
|
|
68
|
+
case 'status': return ok(statusText(host))
|
|
69
|
+
case 'pause':
|
|
70
|
+
host.control.paused = command.paused
|
|
71
|
+
return ok(command.paused ? 'GreenHarness paused. Tool output and model routing are left alone.' : 'GreenHarness resumed.')
|
|
72
|
+
case 'toggle': return toggle(command, host)
|
|
73
|
+
case 'tier': return setTier(command, host)
|
|
74
|
+
default: return fail('Unknown command. Try /greenharness help.')
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The status reply: switches, models, and what this process has done so far.
|
|
80
|
+
* @param {{ control: { paused: boolean }, settings: any, runtime: { stats(): Record<string, number> } }} host
|
|
81
|
+
*/
|
|
82
|
+
export function statusText({ control, settings, runtime }) {
|
|
83
|
+
const stats = runtime.stats()
|
|
84
|
+
const count = key => stats[key] ?? 0
|
|
85
|
+
const describe = route => (route ? `${route.provider}/${route.model}` : 'not set')
|
|
86
|
+
const small_ = settings.tiers.small
|
|
87
|
+
? describe(settings.tiers.small)
|
|
88
|
+
: 'the model DSH uses (recorded at the first routed call)'
|
|
89
|
+
const savedTokens = Math.round(count('charsSaved') / 4)
|
|
90
|
+
const smallTok = count('smallTokens')
|
|
91
|
+
const large = count('largeTokens')
|
|
92
|
+
const spent = smallTok + large
|
|
93
|
+
const cheapShare = spent === 0 ? 0 : smallTok / spent
|
|
94
|
+
const lines = [
|
|
95
|
+
`GreenHarness is ${control.paused ? 'PAUSED' : 'ON'}`,
|
|
96
|
+
'',
|
|
97
|
+
`Tokens saved so far: about ${savedTokens.toLocaleString('en-US')}`,
|
|
98
|
+
` (long outputs shortened: ${count('tool:distill')}, repeated reads replaced: ${count('tool:dedupe')}, loops stopped: ${count('loop-break')})`,
|
|
99
|
+
`Work done by the cheap model: ${bar(cheapShare)} ${Math.round(cheapShare * 100)}%`,
|
|
100
|
+
`Model calls: ${count('request')}. Switches between models: ${count('route')}.`,
|
|
101
|
+
]
|
|
102
|
+
if (count('costUsd') > 0) lines.push(`Cost so far: $${count('costUsd').toFixed(4)}`)
|
|
103
|
+
lines.push(
|
|
104
|
+
'',
|
|
105
|
+
`Models: small = ${small_}, large = ${describe(settings.tiers.large)}. Routing ${onOff(settings.routing.enabled)}.`,
|
|
106
|
+
`Features: shorten output ${onOff(settings.distill.enabled)}, repeated reads ${onOff(settings.readDedupe.enabled)}, loop stop ${onOff(settings.loopBreaker.enabled)}.`,
|
|
107
|
+
settings.tiers.large ? '' : 'Tip: pick a big model with /greenharness large <provider> <model> to save more.',
|
|
108
|
+
)
|
|
109
|
+
return lines.filter((l, i, a) => !(l === '' && (i === a.length - 1))).join('\n')
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** A 20-cell text progress bar. @param {number} fraction 0..1 */
|
|
113
|
+
export function bar(fraction) {
|
|
114
|
+
const filled = Math.max(0, Math.min(20, Math.round(fraction * 20)))
|
|
115
|
+
return '[' + '█'.repeat(filled) + '░'.repeat(20 - filled) + ']'
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** @param {{ mechanism: string, enabled: boolean }} command @param {{ settings: any }} host */
|
|
119
|
+
function toggle({ mechanism, enabled }, { settings }) {
|
|
120
|
+
if (mechanism === 'routing') {
|
|
121
|
+
if (enabled && settings.tiers.large === undefined) {
|
|
122
|
+
return fail('Routing needs a large model first: /greenharness large <provider> <model>')
|
|
123
|
+
}
|
|
124
|
+
settings.routing.enabled = enabled
|
|
125
|
+
return ok(`Routing ${onOff(enabled)}.`)
|
|
126
|
+
}
|
|
127
|
+
const owner = { distill: settings.distill, dedupe: settings.readDedupe, loop: settings.loopBreaker }[mechanism]
|
|
128
|
+
owner.enabled = enabled
|
|
129
|
+
return ok(`${mechanism === 'loop' ? 'Loop breaker' : mechanism[0].toUpperCase() + mechanism.slice(1)} ${onOff(enabled)}.`)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** @param {{ tier: 'small' | 'large', route: object }} command @param {{ settings: any }} host */
|
|
133
|
+
function setTier({ tier, route }, { settings }) {
|
|
134
|
+
settings.tiers[tier] = route
|
|
135
|
+
if (tier === 'large') {
|
|
136
|
+
settings.routing.enabled = true
|
|
137
|
+
return ok(`Large model set to ${route.provider}/${route.model}. Routing is on: calls escalate to it after failing tests.`)
|
|
138
|
+
}
|
|
139
|
+
return ok(`Small model set to ${route.provider}/${route.model}.`)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** @param {string} message */
|
|
143
|
+
function usage(message) {
|
|
144
|
+
return { action: 'error', message: `Usage: ${message}` }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** @param {boolean} value */
|
|
148
|
+
function onOff(value) {
|
|
149
|
+
return value ? 'on' : 'off'
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** @param {string} text */
|
|
153
|
+
function ok(text) {
|
|
154
|
+
return { kind: 'success', text }
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** @param {string} text */
|
|
158
|
+
function fail(text) {
|
|
159
|
+
return { kind: 'error', text }
|
|
160
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal control sequences: CSI (colors, cursor moves), OSC (titles, hyperlinks) and
|
|
3
|
+
* single-character escapes. Removing them makes line counts and fingerprints reflect
|
|
4
|
+
* visible text only.
|
|
5
|
+
*/
|
|
6
|
+
const ANSI_RE = /\u001b\[[0-?]*[ -/]*[@-~]|\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)|\u001b[@-Z\\-_]/g
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Remove terminal control sequences from text.
|
|
10
|
+
* @param {string} text - raw terminal output.
|
|
11
|
+
* @returns {string} the same text without control sequences.
|
|
12
|
+
*/
|
|
13
|
+
export function stripAnsi(text) {
|
|
14
|
+
return text.replace(ANSI_RE, '')
|
|
15
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-tier model cascade: a small model runs by default; a verifier failure moves the next
|
|
3
|
+
* step to the large model; verifier passes move back once they are stable. Pure state
|
|
4
|
+
* machine: callers read the returned change and apply it to the agent.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Default policy: escalate on the first failure, return after two passes, and stay at least two steps per tier. */
|
|
8
|
+
export const DEFAULT_CASCADE_POLICY = Object.freeze({
|
|
9
|
+
escalateAfterFailures: 1,
|
|
10
|
+
deescalateAfterPasses: 2,
|
|
11
|
+
minStepsPerTier: 2,
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Create cascade state for one session.
|
|
16
|
+
* @param {Partial<typeof DEFAULT_CASCADE_POLICY>} [policy] - overrides for the default policy.
|
|
17
|
+
* @returns {{ policy: typeof DEFAULT_CASCADE_POLICY, tier: 'small' | 'large', stepsInTier: number, consecutiveFailures: number, consecutivePasses: number }}
|
|
18
|
+
*/
|
|
19
|
+
export function createCascade(policy = {}) {
|
|
20
|
+
return {
|
|
21
|
+
policy: { ...DEFAULT_CASCADE_POLICY, ...policy },
|
|
22
|
+
tier: 'small',
|
|
23
|
+
stepsInTier: 0,
|
|
24
|
+
consecutiveFailures: 0,
|
|
25
|
+
consecutivePasses: 0,
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Count one completed model step in the current tier.
|
|
31
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
32
|
+
*/
|
|
33
|
+
export function recordStep(cascade) {
|
|
34
|
+
cascade.stepsInTier += 1
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Apply a verifier outcome to the cascade.
|
|
39
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
40
|
+
* @param {boolean} passed - whether the verifier command passed.
|
|
41
|
+
* @returns {{ from: string, to: string, reason: string } | null} the tier change, or null when the tier stays.
|
|
42
|
+
*/
|
|
43
|
+
export function recordVerification(cascade, passed) {
|
|
44
|
+
if (passed) {
|
|
45
|
+
cascade.consecutiveFailures = 0
|
|
46
|
+
cascade.consecutivePasses += 1
|
|
47
|
+
const stable = cascade.consecutivePasses >= cascade.policy.deescalateAfterPasses
|
|
48
|
+
&& cascade.stepsInTier >= cascade.policy.minStepsPerTier
|
|
49
|
+
return cascade.tier === 'large' && stable ? switchTier(cascade, 'small', 'verifier-passed') : null
|
|
50
|
+
}
|
|
51
|
+
cascade.consecutivePasses = 0
|
|
52
|
+
cascade.consecutiveFailures += 1
|
|
53
|
+
const escalate = cascade.consecutiveFailures >= cascade.policy.escalateAfterFailures
|
|
54
|
+
return cascade.tier === 'small' && escalate ? switchTier(cascade, 'large', 'verifier-failed') : null
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Force the large tier, for example when the loop breaker fires.
|
|
59
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
60
|
+
* @param {string} reason - audit reason recorded with the change.
|
|
61
|
+
* @returns {{ from: string, to: string, reason: string } | null} the tier change, or null when already large.
|
|
62
|
+
*/
|
|
63
|
+
export function escalate(cascade, reason) {
|
|
64
|
+
return cascade.tier === 'large' ? null : switchTier(cascade, 'large', reason)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Move to another tier and reset the counters that belong to the old tier.
|
|
69
|
+
* @param {ReturnType<typeof createCascade>} cascade - the session's cascade state (mutated).
|
|
70
|
+
* @param {'small' | 'large'} tier - the destination tier.
|
|
71
|
+
* @param {string} reason - audit reason.
|
|
72
|
+
* @returns {{ from: string, to: string, reason: string }} the change.
|
|
73
|
+
*/
|
|
74
|
+
function switchTier(cascade, tier, reason) {
|
|
75
|
+
const from = cascade.tier
|
|
76
|
+
cascade.tier = tier
|
|
77
|
+
cascade.stepsInTier = 0
|
|
78
|
+
cascade.consecutiveFailures = 0
|
|
79
|
+
cascade.consecutivePasses = 0
|
|
80
|
+
return { from, to: tier, reason }
|
|
81
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { stripAnsi } from './ansi.mjs'
|
|
2
|
+
import { FAIL_RE, FRAME_RE, LIB_FRAME_RE, PROGRESS_RE, SAFETY_RE, SUMMARY_RE, ZERO_RE } from './patterns.mjs'
|
|
3
|
+
|
|
4
|
+
/** Minimum size reduction (output at most this fraction of the input) before a distillation is used. */
|
|
5
|
+
const MIN_REDUCTION_RATIO = 0.9
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Classify one line for distillation.
|
|
9
|
+
* @param {string} line - one line with control sequences removed.
|
|
10
|
+
* @returns {'summary' | 'fail' | 'safety' | 'lib-frame' | 'frame' | 'plain'} the line's kind.
|
|
11
|
+
*/
|
|
12
|
+
function classify(line) {
|
|
13
|
+
if (ZERO_RE.test(line)) return 'summary'
|
|
14
|
+
if (FAIL_RE.test(line)) return 'fail'
|
|
15
|
+
if (SAFETY_RE.test(line)) return 'safety'
|
|
16
|
+
if (FRAME_RE.test(line)) return LIB_FRAME_RE.test(line) ? 'lib-frame' : 'frame'
|
|
17
|
+
if (SUMMARY_RE.test(line)) return 'summary'
|
|
18
|
+
return 'plain'
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Keep the signal in one shell or tool output and drop the rest, without changing any kept line.
|
|
23
|
+
*
|
|
24
|
+
* Kept, in order: failure, summary and safety lines; user-code stack frames and plain lines
|
|
25
|
+
* within `contextLines` of a failure; the first and last three lines. Dropped: progress
|
|
26
|
+
* frames, library stack frames, passing-test noise. Runs of three or more identical kept
|
|
27
|
+
* lines collapse to one line with a count. Output above `maxChars` keeps its head and tail.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} raw - tool output as the model would read it.
|
|
30
|
+
* @param {{ maxChars?: number, contextLines?: number }} [options] - size budget and window around failures.
|
|
31
|
+
* @returns {{ text: string, changed: boolean, stats: { inChars: number, outChars: number, droppedLines: number, droppedFrames: number, collapsedRepeats: number, failLines: number, elidedChars: number } }}
|
|
32
|
+
* `changed` is true only when the distilled text is at most 90% of the input and differs from it.
|
|
33
|
+
*/
|
|
34
|
+
export function distillOutput(raw, options = {}) {
|
|
35
|
+
const maxChars = options.maxChars ?? 6000
|
|
36
|
+
const contextLines = options.contextLines ?? 2
|
|
37
|
+
const input = String(raw ?? '')
|
|
38
|
+
const stats = {
|
|
39
|
+
inChars: input.length, outChars: 0, droppedLines: 0, droppedFrames: 0,
|
|
40
|
+
collapsedRepeats: 0, failLines: 0, elidedChars: 0,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let lines = stripAnsi(input).replace(/\r\n?/g, '\n').split('\n')
|
|
44
|
+
lines = lines.filter((line) => {
|
|
45
|
+
if (line.trim() !== '' && PROGRESS_RE.test(line)) {
|
|
46
|
+
stats.droppedLines += 1
|
|
47
|
+
return false
|
|
48
|
+
}
|
|
49
|
+
return true
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
const kinds = lines.map(classify)
|
|
53
|
+
stats.failLines = kinds.filter(kind => kind === 'fail').length
|
|
54
|
+
|
|
55
|
+
const nearFailure = new Array(lines.length).fill(false)
|
|
56
|
+
kinds.forEach((kind, index) => {
|
|
57
|
+
if (kind !== 'fail') return
|
|
58
|
+
const from = Math.max(0, index - contextLines)
|
|
59
|
+
const to = Math.min(lines.length - 1, index + contextLines)
|
|
60
|
+
for (let j = from; j <= to; j += 1) nearFailure[j] = true
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
const keep = lines.map((_, index) => {
|
|
64
|
+
const kind = kinds[index]
|
|
65
|
+
if (kind === 'fail' || kind === 'summary' || kind === 'safety') return true
|
|
66
|
+
if (kind === 'lib-frame') return false
|
|
67
|
+
if (index < 3 || index >= lines.length - 3) return true
|
|
68
|
+
return nearFailure[index]
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
const emitted = []
|
|
72
|
+
let gap = 0
|
|
73
|
+
let runText = null
|
|
74
|
+
let runCount = 0
|
|
75
|
+
const flushRun = () => {
|
|
76
|
+
if (runText === null) return
|
|
77
|
+
if (runCount >= 3) {
|
|
78
|
+
emitted.push(`${runText} (×${runCount})`)
|
|
79
|
+
stats.collapsedRepeats += runCount - 1
|
|
80
|
+
} else {
|
|
81
|
+
for (let k = 0; k < runCount; k += 1) emitted.push(runText)
|
|
82
|
+
}
|
|
83
|
+
runText = null
|
|
84
|
+
runCount = 0
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
88
|
+
if (!keep[index]) {
|
|
89
|
+
if (kinds[index] === 'lib-frame') stats.droppedFrames += 1
|
|
90
|
+
if (gap === 0) flushRun()
|
|
91
|
+
gap += 1
|
|
92
|
+
continue
|
|
93
|
+
}
|
|
94
|
+
if (gap > 0) {
|
|
95
|
+
emitted.push(`[… ${gap} line${gap === 1 ? '' : 's'} omitted]`)
|
|
96
|
+
gap = 0
|
|
97
|
+
}
|
|
98
|
+
const text = lines[index]
|
|
99
|
+
if (text === runText) {
|
|
100
|
+
runCount += 1
|
|
101
|
+
} else {
|
|
102
|
+
flushRun()
|
|
103
|
+
runText = text
|
|
104
|
+
runCount = 1
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
flushRun()
|
|
108
|
+
if (gap > 0) emitted.push(`[… ${gap} line${gap === 1 ? '' : 's'} omitted]`)
|
|
109
|
+
|
|
110
|
+
let text = emitted.join('\n').replace(/[ \t]+$/gm, '')
|
|
111
|
+
if (text.length > maxChars) {
|
|
112
|
+
const headBudget = Math.floor(maxChars * 0.4)
|
|
113
|
+
const tailBudget = Math.max(0, maxChars - headBudget - 80)
|
|
114
|
+
const headSlice = text.slice(0, headBudget)
|
|
115
|
+
const headCut = headSlice.lastIndexOf('\n')
|
|
116
|
+
const headText = headCut > 0 ? headSlice.slice(0, headCut) : headSlice
|
|
117
|
+
const tailSlice = text.slice(text.length - tailBudget)
|
|
118
|
+
const tailCut = tailSlice.indexOf('\n')
|
|
119
|
+
const tailText = tailCut >= 0 ? tailSlice.slice(tailCut + 1) : tailSlice
|
|
120
|
+
const omitted = text.length - headText.length - tailText.length
|
|
121
|
+
stats.elidedChars = omitted
|
|
122
|
+
text = `${headText}\n[… ${omitted} chars omitted …]\n${tailText}`
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
stats.outChars = text.length
|
|
126
|
+
const changed = text !== input && text.length <= input.length * MIN_REDUCTION_RATIO
|
|
127
|
+
return { text, changed, stats }
|
|
128
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync } from 'node:fs'
|
|
2
|
+
import { dirname } from 'node:path'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Cost and decision ledger: one JSON object per line, appended as events happen.
|
|
6
|
+
* Record types: `request` (provider usage per model step), `tool` (per tool result, with the
|
|
7
|
+
* distillation or dedupe action), `route` (tier change), `loop-break` (denied command).
|
|
8
|
+
*
|
|
9
|
+
* @param {string} path - JSONL file to append to; an empty value disables the ledger.
|
|
10
|
+
* @returns {{ path: string, write(record: Record<string, unknown>): void }}
|
|
11
|
+
*/
|
|
12
|
+
export function createLedger(path) {
|
|
13
|
+
if (!path) return { path: '', write() {} }
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
15
|
+
return {
|
|
16
|
+
path,
|
|
17
|
+
write(record) {
|
|
18
|
+
appendFileSync(path, `${JSON.stringify({ ts: new Date().toISOString(), ...record })}\n`)
|
|
19
|
+
},
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { normalizeCommand } from './verify.mjs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Loop guard: counts consecutive runs of the same command that fail with the same failure
|
|
5
|
+
* fingerprint. A run with a different fingerprint is progress and restarts the count; a
|
|
6
|
+
* passing run clears it; any workspace edit clears every count.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Create an empty loop-guard state for one session.
|
|
11
|
+
* @returns {{ streaks: Map<string, { count: number, fingerprint: string }> }}
|
|
12
|
+
*/
|
|
13
|
+
export function createLoopGuard() {
|
|
14
|
+
return { streaks: new Map() }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Record the outcome of an executed command.
|
|
19
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state (mutated).
|
|
20
|
+
* @param {string} command - the command text.
|
|
21
|
+
* @param {{ passed: boolean, fingerprint: string | null }} outcome - from `classifyOutcome`.
|
|
22
|
+
* @returns {number} the current consecutive count for this command (0 after a pass).
|
|
23
|
+
*/
|
|
24
|
+
export function recordCommandOutcome(guard, command, outcome) {
|
|
25
|
+
const key = normalizeCommand(command)
|
|
26
|
+
if (outcome.passed || outcome.fingerprint === null) {
|
|
27
|
+
guard.streaks.delete(key)
|
|
28
|
+
return 0
|
|
29
|
+
}
|
|
30
|
+
const previous = guard.streaks.get(key)
|
|
31
|
+
const count = previous !== undefined && previous.fingerprint === outcome.fingerprint ? previous.count + 1 : 1
|
|
32
|
+
guard.streaks.set(key, { count, fingerprint: outcome.fingerprint })
|
|
33
|
+
return count
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Consecutive same-fingerprint failures for one command.
|
|
38
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state.
|
|
39
|
+
* @param {string} command - the command text.
|
|
40
|
+
* @returns {number} the count, 0 when the command has no active streak.
|
|
41
|
+
*/
|
|
42
|
+
export function streakOf(guard, command) {
|
|
43
|
+
return guard.streaks.get(normalizeCommand(command))?.count ?? 0
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Clear every streak after a workspace change, because a failing command may pass now.
|
|
48
|
+
* @param {ReturnType<typeof createLoopGuard>} guard - the session's loop-guard state (mutated).
|
|
49
|
+
*/
|
|
50
|
+
export function noteWorkspaceChange(guard) {
|
|
51
|
+
guard.streaks.clear()
|
|
52
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Line classifiers shared by the distiller, the verifier fingerprint and the tests.
|
|
3
|
+
* Each pattern describes one kind of line seen in test runners, compilers, linters
|
|
4
|
+
* and shells. Patterns only decide what to keep; they never rewrite text.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Lines that carry a failure signal (failures, errors, assertion detail, stack-trace headers). */
|
|
8
|
+
export const FAIL_RE = /\b(?:fail(?:s|ed|ure|ures|ing)?|errors?|exceptions?|tracebacks?|panic(?:ked)?|assert(?:ion|ions|ed)?|expected|received|actual|not ok|typeerror|referenceerror|syntaxerror|rangeerror|cannot find|no such file|segmentation fault|timed? ?out|killed)\b|[✗✖×]|^\s*E\s{3,}/i
|
|
9
|
+
|
|
10
|
+
/** Summary lines that state zero failures; these are summaries, not failures. */
|
|
11
|
+
export const ZERO_RE = /\b(?:0\s+(?:failed|failures?|errors?|problems?)|(?:failed|failures?|errors?)\s*[:=]\s*0)\b/i
|
|
12
|
+
|
|
13
|
+
/** Result summaries and run metadata that must survive distillation. */
|
|
14
|
+
export const SUMMARY_RE = /\b\d+\s+(?:passed|failed|skipped|todo|errors?|failures?|tests?|suites?|problems?|examples?)\b|^\s*Tests?\s*:|^\s*Test Files\b|^\s*#\s*(?:tests|pass|fail|suites|duration_ms)\b|^\s*(?:ok|not ok)\b|\bexit code\b|\bran \d+\b|^={3,}.*={3,}\s*$|\bdone in\b|\bfinished in\b|\bduration\b/i
|
|
15
|
+
|
|
16
|
+
/** Lines a human must not miss even when they are not failures (security, deprecation, permissions). */
|
|
17
|
+
export const SAFETY_RE = /\b(?:warn(?:ing|ings)?|deprecat\w*|secur\w*|cve-\d{4}-\d+|permission|denied|unauthori[sz]ed|forbidden|credential\w*|secret\w*|vulnerab\w*|fatal)\b/i
|
|
18
|
+
|
|
19
|
+
/** Spinner frames and progress bars: rendering noise with no information. */
|
|
20
|
+
export const PROGRESS_RE = /^\s*[|/\\-]\s*$|^\s*\[[#=\- ]{4,}\]?\s*\d{0,3}%?\s*$|^\s*\d{1,3}%\s*\|/
|
|
21
|
+
|
|
22
|
+
/** Stack-trace frame headers in common runtimes. */
|
|
23
|
+
export const FRAME_RE = /^\s*(?:at\s+\S|File "[^"]+", line \d+|❯\s|-->\s)/
|
|
24
|
+
|
|
25
|
+
/** Frames inside installed libraries or the runtime itself; they rarely explain an application failure. */
|
|
26
|
+
export const LIB_FRAME_RE = /node_modules[\\/]|node:internal|\binternal\/|site-packages[\\/]|dist-packages[\\/]|\/usr\/lib|\/\.nvm\/|\/\.pyenv\/|<frozen |<internal\//
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-request API cost from provider-reported usage.
|
|
3
|
+
*
|
|
4
|
+
* Usage counts are disjoint, as DSH reports them: `inputTokens` is uncached input,
|
|
5
|
+
* `cacheReadTokens` is cached input, `cacheWriteTokens` is cache creation, and `outputTokens`
|
|
6
|
+
* includes reasoning tokens. Rates are USD per million tokens.
|
|
7
|
+
*
|
|
8
|
+
* @param {{ inputTokens?: number, outputTokens?: number, cacheReadTokens?: number, cacheWriteTokens?: number } | undefined} usage
|
|
9
|
+
* @param {{ inputPerMTok?: number, cachedInputPerMTok?: number, outputPerMTok?: number } | undefined} price
|
|
10
|
+
* @returns {number} estimated cost in USD; 0 when no price is configured for the route.
|
|
11
|
+
*/
|
|
12
|
+
export function usageCostUsd(usage, price) {
|
|
13
|
+
if (usage === undefined || price === undefined) return 0
|
|
14
|
+
const perToken = rate => (rate ?? 0) / 1_000_000
|
|
15
|
+
const input = usage.inputTokens ?? 0
|
|
16
|
+
const cacheWrite = usage.cacheWriteTokens ?? 0
|
|
17
|
+
const cacheRead = usage.cacheReadTokens ?? 0
|
|
18
|
+
const output = usage.outputTokens ?? 0
|
|
19
|
+
return input * perToken(price.inputPerMTok)
|
|
20
|
+
+ cacheWrite * perToken(price.inputPerMTok)
|
|
21
|
+
+ cacheRead * perToken(price.cachedInputPerMTok ?? price.inputPerMTok)
|
|
22
|
+
+ output * perToken(price.outputPerMTok)
|
|
23
|
+
}
|