@alexeiled/claude-router 1.10.2 → 1.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +36 -0
- package/README.md +2 -2
- package/docs/activity-routing.md +13 -4
- package/docs/architecture.md +11 -7
- package/docs/configuration.md +9 -7
- package/docs/evaluation.md +1 -1
- package/docs/native-router.md +1 -1
- package/docs/user-guide.md +43 -26
- package/hooks/native-router.mjs +328 -211
- package/lib/activity-stats.mjs +54 -20
- package/lib/activity.mjs +14 -12
- package/lib/config.mjs +9 -0
- package/lib/display.mjs +7 -3
- package/lib/facts.mjs +82 -0
- package/lib/panel.mjs +12 -2
- package/lib/policy.mjs +38 -14
- package/lib/records.mjs +14 -0
- package/lib/route.mjs +93 -7
- package/lib/savings.mjs +4 -8
- package/lib/subagents.mjs +3 -6
- package/package.json +1 -1
- package/scripts/migrate-config.mjs +23 -3
- package/types/index.d.ts +4 -1
package/lib/activity-stats.mjs
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducers feed the
|
|
2
|
-
// counts kept across sessions
|
|
3
|
-
//
|
|
2
|
+
// counts kept across sessions, per session under STORE_PREFIX and METRICS_PREFIX. Pure: state and a turn in, new state
|
|
3
|
+
// out.
|
|
4
4
|
|
|
5
5
|
import { acceptActivity } from './activity.mjs';
|
|
6
6
|
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
7
|
+
import { afterResetOf, earlierSince, isRecord } from './records.mjs';
|
|
7
8
|
|
|
8
9
|
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
10
|
+
// The one stats key every session shared before 1.10.3: it reads as another session's record until a reset deletes it.
|
|
9
11
|
export const STORE_KEY = 'activity:stats:v1';
|
|
12
|
+
// Each session keeps its own stats under this prefix and its session id, as the metrics do.
|
|
13
|
+
export const STORE_PREFIX = 'activity:stats:v1:';
|
|
10
14
|
// Each session keeps its own metrics under this prefix and its session id, as savings does (SAVINGS_PREFIX): sessions
|
|
11
15
|
// finishing turns at the same moment do not overwrite each other. The readout is their sum (mergeMetrics).
|
|
12
16
|
export const METRICS_PREFIX = 'activity:metrics:v1:';
|
|
@@ -117,8 +121,9 @@ export const LATENCY_BOUNDS = [
|
|
|
117
121
|
const LATENCY_BUCKETS = LATENCY_BOUNDS.length + 1;
|
|
118
122
|
const LATENCY_CLASSIFIERS = 8;
|
|
119
123
|
|
|
124
|
+
// `since` (ms) is the first count after a reset; null in a record from before 1.10.3.
|
|
120
125
|
export function emptyStore() {
|
|
121
|
-
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 } };
|
|
126
|
+
return { version: 1, since: null, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 } };
|
|
122
127
|
}
|
|
123
128
|
|
|
124
129
|
// The p95 of a latency histogram: the upper bound of its bucket in ms, `over` when it lies above the last bound, and
|
|
@@ -137,19 +142,19 @@ export function latencyP95(buckets) {
|
|
|
137
142
|
}
|
|
138
143
|
|
|
139
144
|
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
140
|
-
const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
141
145
|
const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
|
|
142
146
|
const countsOf = (v, keys) => onlyKeys(v, keys) && Object.values(v).every(isCount);
|
|
143
147
|
const exactCountsOf = (v, keys) =>
|
|
144
148
|
isRecord(v) && Object.keys(v).length === keys.length && keys.every((k) => isCount(v[k]));
|
|
145
149
|
|
|
146
150
|
// The stored value if it has a known shape, else an empty store: a hand-edited or unknown file must not break stats.
|
|
147
|
-
// Any other key is rejected: a new count goes under its own key, such as
|
|
148
|
-
// holds the shadow readout, which reads as dropped; a release before 1.10.1 resets a store saved without it.
|
|
151
|
+
// Any other key is rejected: a new count goes under its own key, such as METRICS_PREFIX. A store saved before 1.10.1
|
|
152
|
+
// also holds the shadow readout, which reads as dropped; a release before 1.10.1 resets a store saved without it.
|
|
149
153
|
export function readStore(value) {
|
|
150
154
|
const ok =
|
|
151
|
-
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
155
|
+
onlyKeys(value, ['version', 'since', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
152
156
|
value.version === 1 &&
|
|
157
|
+
(value.since === undefined || value.since === null || Number.isFinite(value.since)) &&
|
|
153
158
|
onlyKeys(value.confusion, CONFUSION_ROWS) &&
|
|
154
159
|
Object.values(value.confusion).every((row) => countsOf(row, OBSERVED)) &&
|
|
155
160
|
onlyKeys(value.runs, ACTIVITIES) &&
|
|
@@ -159,28 +164,60 @@ export function readStore(value) {
|
|
|
159
164
|
exactCountsOf(value.lateral, ['taken', 'refused']);
|
|
160
165
|
if (!ok) return emptyStore();
|
|
161
166
|
const { shadow: _retired, ...store } = value;
|
|
162
|
-
return store;
|
|
167
|
+
return { ...store, since: store.since ?? null };
|
|
163
168
|
}
|
|
164
169
|
|
|
165
|
-
// One finished turn
|
|
166
|
-
// `lateral` the outcome of a lateral switch decision. Unknown labels are dropped, so the store keeps a fixed
|
|
167
|
-
|
|
170
|
+
// One finished turn at `now`. `predicted` is the classifier's answer (null: none), `runEnded` a run that closed this
|
|
171
|
+
// turn and `lateral` the outcome of a lateral switch decision. Unknown labels are dropped, so the store keeps a fixed
|
|
172
|
+
// set of keys. A turn that adds nothing returns `store` itself.
|
|
173
|
+
export function recordStore(store, { predicted, observed, runEnded, lateral }, now = null) {
|
|
168
174
|
const row = predicted ?? 'none';
|
|
169
175
|
const next = {
|
|
170
176
|
version: 1,
|
|
177
|
+
since: store.since ?? now,
|
|
171
178
|
confusion: { ...store.confusion },
|
|
172
179
|
runs: { ...store.runs },
|
|
173
180
|
lateral: { ...store.lateral },
|
|
174
181
|
};
|
|
175
|
-
|
|
182
|
+
let counted = false;
|
|
183
|
+
if (CONFUSION_ROWS.includes(row) && OBSERVED.includes(observed)) {
|
|
176
184
|
next.confusion[row] = { ...next.confusion[row], [observed]: (next.confusion[row]?.[observed] ?? 0) + 1 };
|
|
185
|
+
counted = true;
|
|
186
|
+
}
|
|
177
187
|
if (runEnded && ACTIVITIES.includes(runEnded.activity) && runEnded.length >= 1) {
|
|
178
188
|
const lengths = [...(next.runs[runEnded.activity] ?? Array(RUN_BUCKETS).fill(0))];
|
|
179
189
|
lengths[Math.min(Math.floor(runEnded.length), RUN_BUCKETS) - 1] += 1;
|
|
180
190
|
next.runs[runEnded.activity] = lengths;
|
|
191
|
+
counted = true;
|
|
181
192
|
}
|
|
182
|
-
if (lateral === 'taken' || lateral === 'refused')
|
|
183
|
-
|
|
193
|
+
if (lateral === 'taken' || lateral === 'refused') {
|
|
194
|
+
next.lateral[lateral] += 1;
|
|
195
|
+
counted = true;
|
|
196
|
+
}
|
|
197
|
+
return counted ? next : store;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// A record whose first count came before the last reset at `resetAt` (ms) reads as empty.
|
|
201
|
+
export const storeAfterReset = (record, resetAt) => afterResetOf(record, resetAt, emptyStore);
|
|
202
|
+
|
|
203
|
+
// The stats kept across sessions: every session's record since the last reset added up, since the earliest.
|
|
204
|
+
export function mergeStores(values, resetAt = null) {
|
|
205
|
+
const merged = emptyStore();
|
|
206
|
+
for (const one of values.map((value) => storeAfterReset(readStore(value), resetAt))) {
|
|
207
|
+
merged.since = earlierSince(merged.since, one.since);
|
|
208
|
+
for (const [row, counts] of Object.entries(one.confusion)) {
|
|
209
|
+
const into = { ...merged.confusion[row] };
|
|
210
|
+
for (const [observed, n] of Object.entries(counts)) into[observed] = (into[observed] ?? 0) + n;
|
|
211
|
+
merged.confusion[row] = into;
|
|
212
|
+
}
|
|
213
|
+
for (const [activity, lengths] of Object.entries(one.runs))
|
|
214
|
+
merged.runs[activity] = lengths.map((n, i) => n + (merged.runs[activity]?.[i] ?? 0));
|
|
215
|
+
merged.lateral = {
|
|
216
|
+
taken: merged.lateral.taken + one.lateral.taken,
|
|
217
|
+
refused: merged.lateral.refused + one.lateral.refused,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
return merged;
|
|
184
221
|
}
|
|
185
222
|
|
|
186
223
|
// The readings kept across sessions beside the stats store, one session's or their sum: the classifier wait per
|
|
@@ -237,18 +274,15 @@ export function recordMetrics(metrics, { classifier = null, adviceMs = null, dow
|
|
|
237
274
|
return next;
|
|
238
275
|
}
|
|
239
276
|
|
|
240
|
-
// A record whose first reading came before the last reset at `resetAt` (ms) reads as empty
|
|
241
|
-
export
|
|
242
|
-
return Number.isFinite(resetAt) && record.since !== null && record.since < resetAt ? emptyMetrics() : record;
|
|
243
|
-
}
|
|
277
|
+
// A record whose first reading came before the last reset at `resetAt` (ms) reads as empty.
|
|
278
|
+
export const metricsAfterReset = (record, resetAt) => afterResetOf(record, resetAt, emptyMetrics);
|
|
244
279
|
|
|
245
280
|
// The metrics kept across sessions: every session's record since the last reset added up, since the earliest. Latency
|
|
246
281
|
// keeps the first LATENCY_CLASSIFIERS classifier ids in the order given, so the sum still reads back (readMetrics).
|
|
247
282
|
export function mergeMetrics(values, resetAt = null) {
|
|
248
283
|
const merged = emptyMetrics();
|
|
249
284
|
for (const one of values.map((value) => metricsAfterReset(readMetrics(value), resetAt))) {
|
|
250
|
-
|
|
251
|
-
merged.since = since.length ? Math.min(...since) : null;
|
|
285
|
+
merged.since = earlierSince(merged.since, one.since);
|
|
252
286
|
for (const [id, buckets] of Object.entries(one.latency)) {
|
|
253
287
|
// Own keys only: a classifier id such as `toString` must not read Object.prototype.
|
|
254
288
|
const known = Object.hasOwn(merged.latency, id) ? merged.latency[id] : null;
|
package/lib/activity.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// compare, and when a route change inside one tier is worth it. Pure.
|
|
3
3
|
import { ACTIVITIES, EFFORTS, resolveRoute, routeSpec } from './config.mjs';
|
|
4
4
|
import { routeEffort } from './cost.mjs';
|
|
5
|
-
import { cashGate } from './policy.mjs';
|
|
5
|
+
import { cashGate, sameTaskOf } from './policy.mjs';
|
|
6
6
|
|
|
7
7
|
// The classifier's activity when it names one with at least `policy.activityMass`, else null: uncertain, a weak
|
|
8
8
|
// label, or no answer resolve to the tier's base route.
|
|
@@ -34,18 +34,20 @@ export const sameRequest = (config, a, b, sentEffort) =>
|
|
|
34
34
|
|
|
35
35
|
// The activity a turn runs with, given the incumbent cell `{ tier, activity }` and its resolved `route`. Without
|
|
36
36
|
// advice, or without an activity answer (a skipped, failed or malformed activity step), the incumbent's activity
|
|
37
|
-
// stays: a flaky step must not move the route every turn.
|
|
38
|
-
//
|
|
39
|
-
|
|
37
|
+
// stays: a flaky step must not move the route every turn. On the same task (sameTaskOf: a review inside a turn, or a
|
|
38
|
+
// continuation prompt) a stronger label is taken at once (explore, then code), and a weaker one only with at least
|
|
39
|
+
// `downgradeMass` probability once facts.downAllowed holds: the dwell has passed and a verification passed since the
|
|
40
|
+
// last model change. An unclear label keeps the running activity.
|
|
41
|
+
export function chooseActivity(config, advice, incumbent, sentEffort, facts = {}) {
|
|
40
42
|
if (!advice?.activity) return incumbent.activity;
|
|
41
|
-
if (advice
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
return
|
|
43
|
+
if (!sameTaskOf(config, advice, facts)) return acceptActivity(config, advice);
|
|
44
|
+
const label = advice.activity.choice;
|
|
45
|
+
if (!ACTIVITIES.includes(label)) return incumbent.activity;
|
|
46
|
+
const order = compareRoutes(config, resolveRoute(config, incumbent.tier, label), incumbent.route, sentEffort);
|
|
47
|
+
if (order > 0) return label;
|
|
48
|
+
const mass = advice.activity.probabilities?.[label] ?? 0;
|
|
49
|
+
if (order < 0 && facts.downAllowed && mass >= config.policy.downgradeMass) return label;
|
|
50
|
+
return incumbent.activity;
|
|
49
51
|
}
|
|
50
52
|
|
|
51
53
|
// A route change inside the incumbent's tier, `from` → `to`. Up on evidence: `mass`, the label's probability (1 for
|
package/lib/config.mjs
CHANGED
|
@@ -115,6 +115,12 @@ export const DEFAULTS = {
|
|
|
115
115
|
downgradeHorizonTurns: 5, // turns whose output and read savings offset a downgrade's cache write
|
|
116
116
|
continuationMass: 0.7,
|
|
117
117
|
escalationHoldTurns: 2,
|
|
118
|
+
// Inside a turn the classifier is asked again every this many requests, in the background; 0 turns it off. The
|
|
119
|
+
// answer applies at a later step, so a step never waits for it. Not tuned yet.
|
|
120
|
+
reviewEveryRequests: 4,
|
|
121
|
+
// A move to a cheaper route on the same task waits this many requests after the last model change, and for a
|
|
122
|
+
// verification command (tests, build, lint) that passed since then. Not tuned yet.
|
|
123
|
+
minDwellRequests: 8,
|
|
118
124
|
// Ceiling on a cold cache write to a `credits` model. No default model bills credits, so this is
|
|
119
125
|
// inert until a user adds one in router.json; a Claude Code turn starts near 100k tokens.
|
|
120
126
|
cashCapUsd: 2,
|
|
@@ -529,6 +535,9 @@ function validate(config) {
|
|
|
529
535
|
for (const field of ['upgradeVotes', 'downgradeVotes', 'downgradeHorizonTurns', 'escalationHoldTurns']) {
|
|
530
536
|
if (!isNumber(p[field], 1, { integer: true })) throw new Error(`policy.${field} must be a positive integer`);
|
|
531
537
|
}
|
|
538
|
+
for (const field of ['reviewEveryRequests', 'minDwellRequests']) {
|
|
539
|
+
if (!isNumber(p[field], 0, { integer: true })) throw new Error(`policy.${field} must be a non-negative integer`);
|
|
540
|
+
}
|
|
532
541
|
// The pivot divides: tax / (tax + pivot). Zero makes a free switch NaN and silently stops that switch.
|
|
533
542
|
for (const side of ['upgrade', 'downgrade']) {
|
|
534
543
|
if (!isNumber(p[`${side}Slope`], 0)) throw new Error(`policy.${side}Slope must be a non-negative number`);
|
package/lib/display.mjs
CHANGED
|
@@ -59,13 +59,13 @@ export const REASONS = {
|
|
|
59
59
|
ready: 'ready for the next turn',
|
|
60
60
|
'same-tier': 'the task fits the current tier',
|
|
61
61
|
'no-advice': 'keeping the current model without classifier advice',
|
|
62
|
-
continuation: 'continuing the previous task',
|
|
63
62
|
uncertain: 'the classifier could not justify a change',
|
|
64
63
|
upgrade: 'enough support for a stronger model',
|
|
65
64
|
jump: 'clear need for a stronger model',
|
|
66
65
|
downgrade: 'enough support for a cheaper model',
|
|
67
66
|
'upgrade-pending': 'waiting before upgrading',
|
|
68
67
|
'downgrade-pending': 'switching is not justified yet',
|
|
68
|
+
'downgrade-held': 'a cheaper model waits for a passed check on this task',
|
|
69
69
|
hold: 'staying after an escalation',
|
|
70
70
|
escalation: 'repeated tool failures need a stronger model',
|
|
71
71
|
'cash-gate': 'estimated cold cache write exceeds the cap',
|
|
@@ -73,6 +73,10 @@ export const REASONS = {
|
|
|
73
73
|
'context-unknown': 'context is not measured reliably',
|
|
74
74
|
'model-unavailable': 'the requested model is not available',
|
|
75
75
|
pinned: 'one-turn model pin',
|
|
76
|
+
manual: 'routing did not choose this turn, so Claude’s model is kept',
|
|
77
|
+
'routing off': 'routing is off, so Claude’s model is kept',
|
|
78
|
+
'native-fallback': 'Claude Code switched to another model, so the router kept it',
|
|
79
|
+
interrupted: 'the turn ended before a model was chosen',
|
|
76
80
|
'activity-up': 'the activity needs a stronger route',
|
|
77
81
|
'activity-down': 'a cheaper route fits the activity and is estimated to pay back its cache write',
|
|
78
82
|
'activity-pending': 'a route for the activity does not pay back yet',
|
|
@@ -125,13 +129,13 @@ export const SHORT_REASONS = {
|
|
|
125
129
|
ready: 'ready',
|
|
126
130
|
'same-tier': '= fits',
|
|
127
131
|
'no-advice': '= no advice',
|
|
128
|
-
continuation: '→ continuing',
|
|
129
132
|
uncertain: '= uncertain',
|
|
130
133
|
upgrade: '↑ upgrade',
|
|
131
134
|
jump: '↑ jump',
|
|
132
135
|
downgrade: '↓ downgrade',
|
|
133
136
|
'upgrade-pending': '… waiting to go up',
|
|
134
137
|
'downgrade-pending': '… waiting to go down',
|
|
138
|
+
'downgrade-held': '… down after a passed check',
|
|
135
139
|
hold: '= holding',
|
|
136
140
|
escalation: '↑ tool errors',
|
|
137
141
|
'cash-gate': '= cash cap',
|
|
@@ -198,7 +202,7 @@ function activityRouteText(config, view) {
|
|
|
198
202
|
}
|
|
199
203
|
|
|
200
204
|
// Reasons that only say the route stayed: on an override cell they leave out why it is not the tier's base route.
|
|
201
|
-
const STAYED = ['ready', 'same-tier', 'no-advice', '
|
|
205
|
+
const STAYED = ['ready', 'same-tier', 'no-advice', 'uncertain', 'downgrade-held'];
|
|
202
206
|
|
|
203
207
|
// The Now tab's Why. In `on`, an activity move, or a stay on an override cell, says the cell plainly: "coding at low
|
|
204
208
|
// runs on Sonnet 5.5 · high", or for a move to no activity "low runs on its base route, Haiku 5.5 · high".
|
package/lib/facts.mjs
CHANGED
|
@@ -147,3 +147,85 @@ function readsOnly(command) {
|
|
|
147
147
|
return name === 'git' ? READ_GIT.has(sub) : READ_COMMANDS.has(name);
|
|
148
148
|
});
|
|
149
149
|
}
|
|
150
|
+
|
|
151
|
+
// A command whose success shows that a check passed: a checker run, or a launcher whose subcommand names a check
|
|
152
|
+
// (`npm test`, `npm run lint`, `go test`, `make lint`, `uv run pytest`, `python -m pytest`, `node --test`), alone or in an
|
|
153
|
+
// `&&` list, so the command's status is the check's. A pipeline, `;`, `||` or background job can hide a failed check,
|
|
154
|
+
// so none of them counts. Neither do package names, paths and file names: `npm install -D vitest`, `git diff test/x.mjs`.
|
|
155
|
+
const CHECKERS = new Set(['pytest', 'jest', 'vitest', 'mocha', 'tsc', 'eslint', 'biome', 'ruff', 'mypy', 'pyright']);
|
|
156
|
+
const LAUNCHERS = new Set([
|
|
157
|
+
'npm',
|
|
158
|
+
'pnpm',
|
|
159
|
+
'yarn',
|
|
160
|
+
'bun',
|
|
161
|
+
'npx',
|
|
162
|
+
'bunx',
|
|
163
|
+
'uv',
|
|
164
|
+
'uvx',
|
|
165
|
+
'poetry',
|
|
166
|
+
'cargo',
|
|
167
|
+
'go',
|
|
168
|
+
'make',
|
|
169
|
+
'just',
|
|
170
|
+
'gradle',
|
|
171
|
+
'gradlew',
|
|
172
|
+
'mvn',
|
|
173
|
+
'dotnet',
|
|
174
|
+
'swift',
|
|
175
|
+
'deno',
|
|
176
|
+
'node',
|
|
177
|
+
'python',
|
|
178
|
+
'python3',
|
|
179
|
+
'tox',
|
|
180
|
+
]);
|
|
181
|
+
const RUNS = new Set(['run', 'exec', 'x', 'dlx']);
|
|
182
|
+
const CHECK_WORD =
|
|
183
|
+
/\b(?:test|tests|pytest|jest|vitest|mocha|tsc|typecheck|lint|eslint|biome|ruff|mypy|pyright|clippy|build|check|validate)\b/;
|
|
184
|
+
|
|
185
|
+
function verifies(command) {
|
|
186
|
+
if (typeof command !== 'string') return false;
|
|
187
|
+
const bare = command.replace(/'[^']*'|"[^"]*"/g, '""').replace(/\d?>\s*&\d|\d?>\s*\/dev\/null/g, '');
|
|
188
|
+
const steps = bare.split('&&');
|
|
189
|
+
if (steps.some((step) => /\|\||[;|&\n]/.test(step))) return false;
|
|
190
|
+
return steps.some((step) => runsCheck(step.trim().split(/\s+/)));
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// A version or help request runs no check.
|
|
194
|
+
const INFO = /^(?:--version|-V|--help|-h|help|version)$/;
|
|
195
|
+
|
|
196
|
+
// An option before the subcommand may take a value (`npm --prefix test install`), so the subcommand must come first;
|
|
197
|
+
// `python -m` is the one option form read. Anything else unrecognised does not count.
|
|
198
|
+
function runsCheck(words) {
|
|
199
|
+
while (words.length > 1 && /^\w+=/.test(words[0])) words.shift();
|
|
200
|
+
const [first, ...rest] = words;
|
|
201
|
+
const name = first.split('/').pop();
|
|
202
|
+
if (rest.some((w) => INFO.test(w))) return false;
|
|
203
|
+
if (CHECKERS.has(name)) return true;
|
|
204
|
+
if (!LAUNCHERS.has(name)) return false;
|
|
205
|
+
if (rest.some((w) => /^--test\b/.test(w))) return true;
|
|
206
|
+
let args = (name === 'python' || name === 'python3') && rest[0] === '-m' ? rest.slice(1) : rest;
|
|
207
|
+
if (RUNS.has(args[0])) args = args.slice(1);
|
|
208
|
+
const sub = args[0] ?? '';
|
|
209
|
+
return !sub.startsWith('-') && !sub.includes('/') && !/\.\w+$/.test(sub) && CHECK_WORD.test(sub);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Whether the last verification command at or after message `fromIndex` passed: the progress checkpoint a move to a
|
|
213
|
+
// cheaper route on the same task waits for. False when none ran since, or the last one failed (an unresolved failure),
|
|
214
|
+
// or it went to the background, by request or after a timeout: its result reports the launch, not the check.
|
|
215
|
+
export function verifiedSince(messages, fromIndex) {
|
|
216
|
+
let last = null;
|
|
217
|
+
for (let i = Math.max(0, fromIndex); i < messages.length; i += 1)
|
|
218
|
+
if (messages[i]?.role === 'assistant')
|
|
219
|
+
for (const block of blocks(messages[i].content))
|
|
220
|
+
if (block.type === 'tool_use' && block.name === 'Bash' && verifies(block.input?.command))
|
|
221
|
+
last = { id: block.id, index: i, background: block.input?.run_in_background === true };
|
|
222
|
+
if (!last || last.background) return false;
|
|
223
|
+
for (let i = last.index + 1; i < messages.length; i += 1)
|
|
224
|
+
for (const block of blocks(messages[i]?.content))
|
|
225
|
+
if (block.type === 'tool_result' && block.tool_use_id === last.id)
|
|
226
|
+
return !block.is_error && !BACKGROUNDED.test(textOf(blocks(block.content)));
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// What Claude Code's Bash tool returns for a command it moved to the background.
|
|
231
|
+
const BACKGROUNDED = /running in background|moved to the background/i;
|
package/lib/panel.mjs
CHANGED
|
@@ -108,8 +108,18 @@ export function routeDraftOf(config, view) {
|
|
|
108
108
|
|
|
109
109
|
const dollars = (value) => (Number.isFinite(value) ? `$${value.toFixed(3)}` : 'not reported');
|
|
110
110
|
const difference = (value) => `${value < 0 ? '−' : '+'}$${Math.abs(value).toFixed(3)}`;
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
// The host and port as the request sees them: user info before an `@` is not the host. An endpoint that does not
|
|
112
|
+
// parse, such as one with a placeholder in its host, shows as written up to its path, and is not this machine.
|
|
113
|
+
function urlOf(endpoint) {
|
|
114
|
+
try {
|
|
115
|
+
return new URL(endpoint);
|
|
116
|
+
} catch {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const hostOf = (endpoint) => urlOf(endpoint)?.host ?? endpoint.replace(/^https?:\/\//, '').split('/')[0];
|
|
121
|
+
const LOCAL_HOSTS = ['127.0.0.1', 'localhost', '[::1]'];
|
|
122
|
+
const isLocal = (endpoint) => LOCAL_HOSTS.includes(urlOf(endpoint)?.hostname);
|
|
113
123
|
const windowSize = (tokens) => (tokens >= 1e6 ? `${tokens / 1e6}M` : `${Math.round(tokens / 1e3)}K`);
|
|
114
124
|
const share = (value) => (value === null ? '—' : `${value}%`);
|
|
115
125
|
const plural = (n, word, many = `${word}s`) => `${n} ${n === 1 ? word : many}`;
|
package/lib/policy.mjs
CHANGED
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
import { rank, resolveRoute, routeSpec, TIERS } from './config.mjs';
|
|
3
3
|
import * as nativeCosts from './cost.mjs';
|
|
4
4
|
|
|
5
|
+
// Whether an answer is about the task already running: a review inside a turn (facts.midTurn), or a new prompt the
|
|
6
|
+
// classifier scores as a continuation. On the same task either way is open, but a cheaper route waits for
|
|
7
|
+
// facts.downAllowed: the dwell since the last model change and a passed verification.
|
|
8
|
+
export function sameTaskOf(config, advice, facts) {
|
|
9
|
+
return Boolean(facts.midTurn) || (advice?.continuation ?? 0) >= config.policy.continuationMass;
|
|
10
|
+
}
|
|
11
|
+
|
|
5
12
|
export function initialState() {
|
|
6
13
|
return { turn: 0, votes: [], holdUntilTurn: 0, escalatedSignature: null };
|
|
7
14
|
}
|
|
@@ -37,7 +44,6 @@ export function decide({
|
|
|
37
44
|
}
|
|
38
45
|
if (next.turn <= next.holdUntilTurn) return stay('hold');
|
|
39
46
|
if (!advice) return stay('no-advice');
|
|
40
|
-
if (advice.continuation >= config.policy.continuationMass) return stay('continuation');
|
|
41
47
|
if (advice.choice === 'uncertain') return stay('uncertain');
|
|
42
48
|
|
|
43
49
|
const choice = advice.choice;
|
|
@@ -68,19 +74,33 @@ export function decide({
|
|
|
68
74
|
return stay('upgrade-pending', estimate);
|
|
69
75
|
}
|
|
70
76
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
//
|
|
74
|
-
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const
|
|
81
|
-
|
|
77
|
+
// The move lands on the cheapest tier the answer supports as a whole: from the choice up to just below the
|
|
78
|
+
// incumbent, the first tier whose mass at or below it clears its bar. A split answer (micro 0.8, low 0.16) on high
|
|
79
|
+
// moves to low instead of waiting for micro alone to reach the bar. A pending stay reports the choice's estimate.
|
|
80
|
+
let pending = null;
|
|
81
|
+
for (const target of TIERS.filter((t) => rank(t) >= rank(choice) && rank(t) < rank(incumbent))) {
|
|
82
|
+
const support = massAtOrBelow(advice.probabilities, target);
|
|
83
|
+
const streak = trailing(next.votes, (v) => rank(v.tier) <= rank(target));
|
|
84
|
+
// A downgrade is not always cheaper this turn: a cold candidate next to a warm incumbent can pay a bigger cache
|
|
85
|
+
// write than it saves. The bar rises with that cost, net of what the next turns save, the way the upgrade bar does.
|
|
86
|
+
const tax = costs.downgradeTaxUsd(
|
|
87
|
+
config,
|
|
88
|
+
routeFor(target),
|
|
89
|
+
current,
|
|
90
|
+
facts,
|
|
91
|
+
now,
|
|
92
|
+
config.policy.downgradeHorizonTurns,
|
|
93
|
+
);
|
|
94
|
+
const threshold =
|
|
95
|
+
config.policy.downgradeMass + config.policy.downgradeSlope * (tax / (tax + config.policy.downgradePivotUsd));
|
|
96
|
+
const estimate = { downgradeMass: support, streak, taxUsd: tax, threshold };
|
|
97
|
+
pending ??= estimate;
|
|
98
|
+
if (support < threshold || streak < downgradeVotes) continue;
|
|
99
|
+
if (sameTaskOf(config, advice, facts) && !facts.downAllowed) return stay('downgrade-held', estimate);
|
|
100
|
+
const gated = cashGate(config, routeFor(target), facts, now, costs, routeFor);
|
|
101
|
+
return gated.blocked ? gatedResult(gated, incumbent, next) : result(target, 'downgrade', next, estimate);
|
|
82
102
|
}
|
|
83
|
-
return stay('downgrade-pending',
|
|
103
|
+
return stay('downgrade-pending', pending);
|
|
84
104
|
}
|
|
85
105
|
|
|
86
106
|
// Share of a model's window the next request may fill. The estimate leaves out the new prompt and tool
|
|
@@ -110,7 +130,11 @@ export function cashGate(config, route, facts, now, costs, routeFor = (tier) =>
|
|
|
110
130
|
const cold = costs.coldWriteUsd(config, route.model, costs.nextContextTokens(facts));
|
|
111
131
|
if (cold <= config.policy.cashCapUsd) return { blocked: false };
|
|
112
132
|
const fallback = [...TIERS].reverse().find((t) => routeSpec(config, routeFor(t)).billing === 'plan');
|
|
113
|
-
const estimate = {
|
|
133
|
+
const estimate = {
|
|
134
|
+
coldUsd: cold,
|
|
135
|
+
cap: config.policy.cashCapUsd,
|
|
136
|
+
cache: cacheOf(config, route, facts, now, costs),
|
|
137
|
+
};
|
|
114
138
|
return { blocked: true, fallback, estimate };
|
|
115
139
|
}
|
|
116
140
|
|
package/lib/records.mjs
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// What the records kept across sessions share: savings, activity stats and metrics, and subagents. Each session keeps
|
|
2
|
+
// its own record of a kind, with `since` (ms), its first count after a reset; the readout sums them.
|
|
3
|
+
|
|
4
|
+
export const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
5
|
+
|
|
6
|
+
// A record whose first count came before the last reset at `resetAt` (ms) reads as `empty()`.
|
|
7
|
+
export const afterResetOf = (record, resetAt, empty) =>
|
|
8
|
+
Number.isFinite(resetAt) && record.since !== null && record.since < resetAt ? empty() : record;
|
|
9
|
+
|
|
10
|
+
// The earlier of two `since` values, or null when neither is set.
|
|
11
|
+
export function earlierSince(a, b) {
|
|
12
|
+
const since = [a, b].filter(Number.isFinite);
|
|
13
|
+
return since.length ? Math.min(...since) : null;
|
|
14
|
+
}
|
package/lib/route.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { chooseActivity, compareRoutes, lateralMove, sameRequest } from './activity.mjs';
|
|
2
2
|
import { ACTIVITIES, rank, resolveRoute, routeSpec, TIERS } from './config.mjs';
|
|
3
3
|
import * as costs from './cost.mjs';
|
|
4
|
-
import { clip, extractFacts } from './facts.mjs';
|
|
4
|
+
import { clip, extractFacts, verifiedSince } from './facts.mjs';
|
|
5
5
|
import { CONTEXT_FILL, decide, fitTier, initialState } from './policy.mjs';
|
|
6
6
|
|
|
7
7
|
// Exact observed API snapshot, not a rule that merges different dated versions.
|
|
@@ -52,6 +52,15 @@ export function emptyLoop(config, model) {
|
|
|
52
52
|
turnId: null,
|
|
53
53
|
decision: null,
|
|
54
54
|
ineligible: [],
|
|
55
|
+
// Routed main requests so far, the one that started on the running route (`switchedAt`: requests and message
|
|
56
|
+
// count then), and the request count at the last classifier answer (`reviewedAt`). Dwell and reviews count these.
|
|
57
|
+
requests: 0,
|
|
58
|
+
servedRoute: null,
|
|
59
|
+
switchedAt: null,
|
|
60
|
+
reviewedAt: 0,
|
|
61
|
+
// The message count at the last review applied inside a turn: tool calls before it were already judged, so only
|
|
62
|
+
// later ones can move the route up again (observedMove).
|
|
63
|
+
observedFrom: 0,
|
|
55
64
|
// Each configured model's simulated cache, by models key, for the routing-vs-your-model readout (compareModels in
|
|
56
65
|
// savings.mjs).
|
|
57
66
|
yoursBy: {},
|
|
@@ -75,15 +84,36 @@ export function resetHistory(loop) {
|
|
|
75
84
|
}
|
|
76
85
|
|
|
77
86
|
export function prepareLoop(loop, messageCount) {
|
|
78
|
-
|
|
79
|
-
|
|
87
|
+
if (loop.lastMessageCount === null || messageCount >= loop.lastMessageCount)
|
|
88
|
+
return { ...loop, lastMessageCount: messageCount };
|
|
89
|
+
// Compaction or a rewind shortened the history: message cursors past its end move to it, so checks and edits made
|
|
90
|
+
// from here on count. The request-based dwell is kept.
|
|
91
|
+
const cursor = (count) => Math.min(count ?? 0, messageCount);
|
|
92
|
+
return {
|
|
93
|
+
...resetHistory(loop),
|
|
94
|
+
lastMessageCount: messageCount,
|
|
95
|
+
switchedAt: loop.switchedAt && { ...loop.switchedAt, messageCount: cursor(loop.switchedAt.messageCount) },
|
|
96
|
+
observedFrom: cursor(loop.observedFrom),
|
|
97
|
+
};
|
|
80
98
|
}
|
|
81
99
|
|
|
82
100
|
export function nativeFacts(config, loop, { messages, prompt, effort, turnId, contextTokens }) {
|
|
83
101
|
const facts = extractFacts({ messages, output_config: { effort } }, loop, config.context);
|
|
84
102
|
if (typeof prompt === 'string') facts.prompt = clip(prompt, config.context.maxTextChars);
|
|
103
|
+
// A move down on the same task waits out the dwell and for a passed verification since the last model change.
|
|
104
|
+
const since = loop.switchedAt ?? { requests: 0, messageCount: 0 };
|
|
105
|
+
const downAllowed =
|
|
106
|
+
(loop.requests ?? 0) - since.requests >= config.policy.minDwellRequests &&
|
|
107
|
+
verifiedSince(messages, since.messageCount);
|
|
85
108
|
// A loop saved without the field counts as measured, keeping the vote streak.
|
|
86
|
-
return {
|
|
109
|
+
return {
|
|
110
|
+
...facts,
|
|
111
|
+
downAllowed,
|
|
112
|
+
pin: null,
|
|
113
|
+
contextTokens,
|
|
114
|
+
turnKey: turnId,
|
|
115
|
+
historyMeasured: loop.historyMeasured !== false,
|
|
116
|
+
};
|
|
87
117
|
}
|
|
88
118
|
|
|
89
119
|
export function isModelAllowed(id, available, nativeModel) {
|
|
@@ -119,6 +149,7 @@ export function chooseRoute(config, loop, input) {
|
|
|
119
149
|
turnId: facts.turnKey,
|
|
120
150
|
decision: applied,
|
|
121
151
|
suspended: false,
|
|
152
|
+
reviewedAt: loop.requests ?? 0,
|
|
122
153
|
};
|
|
123
154
|
}
|
|
124
155
|
|
|
@@ -135,7 +166,7 @@ function routeTurn(config, loop, { facts, advice, pin, nativeModel, contextKnown
|
|
|
135
166
|
const tier0 = TIERS.includes(facts.lastRoute) ? facts.lastRoute : config.baselineTier;
|
|
136
167
|
const incumbent = { tier: tier0, activity: last.activity, route: resolveRoute(config, tier0, last.activity) };
|
|
137
168
|
const activity =
|
|
138
|
-
pin || config.activityRouting !== 'on' ? null : chooseActivity(config, advice, incumbent, facts.effort);
|
|
169
|
+
pin || config.activityRouting !== 'on' ? null : chooseActivity(config, advice, incumbent, facts.effort, facts);
|
|
139
170
|
let result =
|
|
140
171
|
pin && TIERS.includes(pin)
|
|
141
172
|
? {
|
|
@@ -246,7 +277,52 @@ function decideCell({ config, projected, facts, advice, state, now, incumbent, a
|
|
|
246
277
|
};
|
|
247
278
|
}
|
|
248
279
|
|
|
249
|
-
//
|
|
280
|
+
// Whether some activity's route at the decision's tier is stronger than the one running: only then can the turn's own
|
|
281
|
+
// tool calls move it (observedMove), so a step on the strongest route skips reading the transcript.
|
|
282
|
+
export function canMoveUp(config, decision, effort) {
|
|
283
|
+
if (config.activityRouting !== 'on' || !decision?.tier || decision.pinned) return false;
|
|
284
|
+
const from = resolveRoute(config, decision.tier, decision.activity ?? null);
|
|
285
|
+
return ACTIVITIES.some((a) => compareRoutes(config, resolveRoute(config, decision.tier, a), from, effort) > 0);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// A move up inside a turn: the classifier labels a turn from its prompt, and a prompt that reads as ops or explore can
|
|
289
|
+
// still end up editing code. Once the turn's tool calls show an activity (observedActivity in facts.mjs) whose route at the
|
|
290
|
+
// tier is stronger, the remaining steps take that route. The tools are the evidence, so the move needs no classifier
|
|
291
|
+
// probability. At probability 1 the switching tax never holds it back (the upgrade bar tops out near 0.9): only the
|
|
292
|
+
// cash gate, priced at the context-fitted route, can. Only up: a move down comes from a classifier review, past its
|
|
293
|
+
// checks. Returns the loop, unchanged when nothing moves.
|
|
294
|
+
export function observedMove(config, loop, { observed, facts, contextKnown, now }) {
|
|
295
|
+
const decision = loop.decision;
|
|
296
|
+
if (!canMoveUp(config, decision, facts.effort) || !ACTIVITIES.includes(observed)) return loop;
|
|
297
|
+
const from = resolveRoute(config, decision.tier, decision.activity ?? null);
|
|
298
|
+
// The gates price the route that will run: the one the context fits, not the tier's first choice.
|
|
299
|
+
const routeFor = (tier) => resolveRoute(config, tier, observed);
|
|
300
|
+
const tier = fitTier(config, decision.tier, costs.nextContextTokens(facts), routeFor);
|
|
301
|
+
const to = routeFor(tier);
|
|
302
|
+
if (!(compareRoutes(config, to, from, facts.effort) > 0)) return loop;
|
|
303
|
+
// Without a reliable reading of the context, a smaller window might overflow, as in routeTurn.
|
|
304
|
+
const window = (route) => routeSpec(config, route).contextWindow;
|
|
305
|
+
if ((!contextKnown || !loop.historyMeasured) && window(to) < window(from)) return loop;
|
|
306
|
+
const projected = economicConfig(config, loop);
|
|
307
|
+
const lateral = lateralMove({ config: projected, from, to, mass: 1, hold: false, facts, now, costs });
|
|
308
|
+
if (!lateral.move) return loop;
|
|
309
|
+
return {
|
|
310
|
+
...loop,
|
|
311
|
+
lastActivity: observed,
|
|
312
|
+
decision: {
|
|
313
|
+
...decision,
|
|
314
|
+
tier,
|
|
315
|
+
activity: observed,
|
|
316
|
+
model: routeSpec(config, to).id,
|
|
317
|
+
effort: costs.routeEffort(config, to, facts.effort),
|
|
318
|
+
reason: lateral.reason,
|
|
319
|
+
estimate: lateral.estimate,
|
|
320
|
+
},
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// A tool continuation keeps the turn's route and activity; only the context, the model list, or the turn's own tool
|
|
325
|
+
// calls (observedMove) can move it.
|
|
250
326
|
export function continueRoute(config, loop, { nativeModel, contextTokens, contextKnown, availableModels, effort }) {
|
|
251
327
|
if (!loop.decision) return loop;
|
|
252
328
|
let decision = loop.decision;
|
|
@@ -269,7 +345,17 @@ export function continueRoute(config, loop, { nativeModel, contextTokens, contex
|
|
|
269
345
|
}
|
|
270
346
|
|
|
271
347
|
export function observeResponse(config, loop, { usage, requestedModel, effort, stopReason, now }) {
|
|
272
|
-
|
|
348
|
+
// Every request counts toward dwell and reviews; a request on another (model, effort) than the last starts a dwell,
|
|
349
|
+
// and so does the first one this loop serves: checks from before it ran on a route the router did not choose.
|
|
350
|
+
const served = `${requestedModel}@${effort ?? 'session'}`;
|
|
351
|
+
const requests = (loop.requests ?? 0) + 1;
|
|
352
|
+
const switched = loop.servedRoute !== served;
|
|
353
|
+
let next = {
|
|
354
|
+
...loop,
|
|
355
|
+
requests,
|
|
356
|
+
servedRoute: served,
|
|
357
|
+
switchedAt: switched ? { requests, messageCount: loop.lastMessageCount ?? 0 } : (loop.switchedAt ?? null),
|
|
358
|
+
};
|
|
273
359
|
if (stopReason === 'model_context_window_exceeded') {
|
|
274
360
|
next = { ...next, ineligible: [...new Set([...next.ineligible, requestedModel])] };
|
|
275
361
|
}
|