@alexeiled/claude-router 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +61 -0
- package/README.md +4 -1
- package/docs/architecture.md +17 -3
- package/docs/configuration.md +6 -2
- package/docs/evaluation.md +15 -11
- package/docs/native-router.md +2 -0
- package/docs/router-band.svg +30 -8
- package/docs/router-pane-now.svg +133 -131
- package/docs/router-pane-policy.svg +47 -43
- package/docs/routing-modes.md +143 -0
- package/docs/user-guide.md +17 -2
- package/hooks/native-router.mjs +428 -36
- package/lib/activity-stats.mjs +33 -11
- package/lib/activity.mjs +12 -6
- package/lib/band.mjs +8 -1
- package/lib/config.mjs +72 -17
- package/lib/cost.mjs +3 -0
- package/lib/decisions.mjs +252 -0
- package/lib/display.mjs +46 -5
- package/lib/facts.mjs +125 -4
- package/lib/modes.mjs +98 -0
- package/lib/panel.mjs +41 -3
- package/lib/policy.mjs +7 -4
- package/lib/route.mjs +84 -26
- package/lib/savings.mjs +17 -0
- package/lib/usage.mjs +31 -19
- package/package.json +2 -1
- package/types/index.d.ts +15 -2
package/lib/activity-stats.mjs
CHANGED
|
@@ -6,7 +6,9 @@ import { acceptActivity } from './activity.mjs';
|
|
|
6
6
|
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
7
7
|
import { afterResetOf, earlierSince, isRecord } from './records.mjs';
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
// A turn's bucket from its tools (observedActivity in facts.mjs). `shell` and `check` came in 1.16: a version before
|
|
10
|
+
// reads a store that holds them as empty.
|
|
11
|
+
export const OBSERVED = ['code', 'docs', 'shell', 'ops', 'check', 'read', 'talk'];
|
|
10
12
|
// The one stats key every session shared before 1.10.3: it reads as another session's record until a reset deletes it.
|
|
11
13
|
export const STORE_KEY = 'activity:stats:v1';
|
|
12
14
|
// Each session keeps its own stats under this prefix and its session id, as the metrics do.
|
|
@@ -16,16 +18,17 @@ export const STORE_PREFIX = 'activity:stats:v1:';
|
|
|
16
18
|
export const METRICS_PREFIX = 'activity:metrics:v1:';
|
|
17
19
|
|
|
18
20
|
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
19
|
-
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
21
|
+
// separate plan from review. That is enough to catch a cheap route on a coding turn. A `shell` write is a candidate
|
|
22
|
+
// for code, docs or routine work, so each of them allows it; a `check` run fits a review, a debug session or ops.
|
|
20
23
|
export const ALLOWED = {
|
|
21
|
-
code: ['code'],
|
|
22
|
-
debug: ['code', 'ops', 'read'],
|
|
24
|
+
code: ['code', 'shell'],
|
|
25
|
+
debug: ['code', 'shell', 'ops', 'check', 'read'],
|
|
23
26
|
explore: ['read', 'talk'],
|
|
24
27
|
plan: ['talk', 'read', 'docs'],
|
|
25
|
-
review: ['read', 'talk'],
|
|
28
|
+
review: ['read', 'talk', 'check'],
|
|
26
29
|
// git status or a test listing is still running commands: a read-only Bash turn is ops too.
|
|
27
|
-
ops: ['ops', 'read'],
|
|
28
|
-
docs: ['docs'],
|
|
30
|
+
ops: ['ops', 'shell', 'check', 'read'],
|
|
31
|
+
docs: ['docs', 'shell'],
|
|
29
32
|
};
|
|
30
33
|
|
|
31
34
|
// An answer without an allowed set (uncertain, none) never agrees; callers count agreement only for activities.
|
|
@@ -224,9 +227,15 @@ export function mergeStores(values, resetAt = null) {
|
|
|
224
227
|
// classifier id, and the cheaper activity moves taken in `on` with the escalations that followed one, since `since`
|
|
225
228
|
// (ms), the first reading after a reset.
|
|
226
229
|
export function emptyMetrics() {
|
|
227
|
-
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 } };
|
|
230
|
+
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 }, unpriced: emptyUnpriced() };
|
|
228
231
|
}
|
|
229
232
|
|
|
233
|
+
// Why a routed main reply was left out of the usage record (unpricedCause in savings.mjs): its usage did not read
|
|
234
|
+
// (`usage`), a model in the comparison had no price (`model`), the cache lifetime could not be read (`ttl`), or the
|
|
235
|
+
// comparison failed (`error`). Kept beside the stats from 1.16; a record from before reads as none.
|
|
236
|
+
export const UNPRICED_CAUSES = ['usage', 'model', 'ttl', 'error'];
|
|
237
|
+
const emptyUnpriced = () => Object.fromEntries(UNPRICED_CAUSES.map((cause) => [cause, 0]));
|
|
238
|
+
|
|
230
239
|
const isLatency = (v) =>
|
|
231
240
|
isRecord(v) &&
|
|
232
241
|
Object.keys(v).length <= LATENCY_CLASSIFIERS &&
|
|
@@ -242,19 +251,31 @@ export function readMetrics(value) {
|
|
|
242
251
|
since: Number.isFinite(value.since) ? value.since : null,
|
|
243
252
|
latency: isLatency(value.latency) ? value.latency : empty.latency,
|
|
244
253
|
downMoves: exactCountsOf(value.downMoves, ['moves', 'escalations']) ? value.downMoves : empty.downMoves,
|
|
254
|
+
unpriced: exactCountsOf(value.unpriced, UNPRICED_CAUSES) ? value.unpriced : empty.unpriced,
|
|
245
255
|
};
|
|
246
256
|
}
|
|
247
257
|
|
|
248
|
-
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed),
|
|
249
|
-
// 'escalated' (advanceDown). A turn that adds nothing
|
|
250
|
-
|
|
258
|
+
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed), `downMove` 'moved' or
|
|
259
|
+
// 'escalated' (advanceDown), and `unpriced` the turn's routed replies left unpriced, by cause. A turn that adds nothing
|
|
260
|
+
// returns `metrics` itself.
|
|
261
|
+
export function recordMetrics(
|
|
262
|
+
metrics,
|
|
263
|
+
{ classifier = null, adviceMs = null, downMove = null, unpriced = {}, now = null },
|
|
264
|
+
) {
|
|
251
265
|
const next = {
|
|
252
266
|
version: 1,
|
|
253
267
|
since: metrics.since ?? now,
|
|
254
268
|
latency: { ...metrics.latency },
|
|
255
269
|
downMoves: { ...metrics.downMoves },
|
|
270
|
+
unpriced: { ...emptyUnpriced(), ...metrics.unpriced },
|
|
256
271
|
};
|
|
257
272
|
let counted = downMove === 'moved' || downMove === 'escalated';
|
|
273
|
+
for (const cause of UNPRICED_CAUSES) {
|
|
274
|
+
const n = count(unpriced?.[cause]);
|
|
275
|
+
if (!n) continue;
|
|
276
|
+
next.unpriced[cause] += n;
|
|
277
|
+
counted = true;
|
|
278
|
+
}
|
|
258
279
|
const known = Object.hasOwn(next.latency, classifier);
|
|
259
280
|
if (
|
|
260
281
|
typeof classifier === 'string' &&
|
|
@@ -293,6 +314,7 @@ export function mergeMetrics(values, resetAt = null) {
|
|
|
293
314
|
moves: merged.downMoves.moves + one.downMoves.moves,
|
|
294
315
|
escalations: merged.downMoves.escalations + one.downMoves.escalations,
|
|
295
316
|
};
|
|
317
|
+
for (const cause of UNPRICED_CAUSES) merged.unpriced[cause] += one.unpriced[cause];
|
|
296
318
|
}
|
|
297
319
|
return merged;
|
|
298
320
|
}
|
package/lib/activity.mjs
CHANGED
|
@@ -34,10 +34,10 @@ export const sameRequest = (config, a, b, sentEffort) =>
|
|
|
34
34
|
|
|
35
35
|
// The activity a turn runs with, given the incumbent cell `{ tier, activity }` and its resolved `route`. Without
|
|
36
36
|
// advice, or without an activity answer (a skipped, failed or malformed activity step), the incumbent's activity
|
|
37
|
-
// stays: a flaky step must not move the route every turn. On the same task (sameTaskOf: a review inside a turn,
|
|
38
|
-
// continuation prompt) a stronger label is taken at once (explore, then code), and a weaker
|
|
39
|
-
// `downgradeMass` probability once facts.downAllowed holds: the dwell has passed and a
|
|
40
|
-
// last model change. An unclear label keeps the running activity.
|
|
37
|
+
// stays: a flaky step must not move the route every turn. On the same task (sameTaskOf: a review inside a turn, a turn
|
|
38
|
+
// with a progress origin, or a continuation prompt) a stronger label is taken at once (explore, then code), and a weaker
|
|
39
|
+
// one only with at least `downgradeMass` probability once facts.downAllowed holds: the dwell has passed and a checkpoint
|
|
40
|
+
// (completedSince) since the last model change. An unclear label keeps the running activity.
|
|
41
41
|
export function chooseActivity(config, advice, incumbent, sentEffort, facts = {}) {
|
|
42
42
|
if (!advice?.activity) return incumbent.activity;
|
|
43
43
|
if (!sameTaskOf(config, advice, facts)) return acceptActivity(config, advice);
|
|
@@ -54,15 +54,21 @@ export function chooseActivity(config, advice, incumbent, sentEffort, facts = {}
|
|
|
54
54
|
// the base route), must clear the tier upgrade bar for this switching tax. Down on economics: the move must pay back
|
|
55
55
|
// its cache write within the downgrade horizon. A pair that does not order (an unpriced model, a numeric effort, two
|
|
56
56
|
// models at one price) counts as up: it needs evidence, and the base route is always reachable. Labels alone never
|
|
57
|
-
// switch, so one vote is enough. During an escalation hold only a stronger route may be taken.
|
|
57
|
+
// switch, so one vote is enough. During an escalation hold only a stronger route may be taken. A cheaper route on the
|
|
58
|
+
// same task (`sameTask`) waits for facts.downAllowed, as a tier downgrade does: a routing mode changes the route
|
|
59
|
+
// without a new label, and a free switch must still wait out the dwell. A model with a lower input price also counts
|
|
60
|
+
// as cheaper there, whatever its output price: that is the order the cost mode picks by. Returns
|
|
58
61
|
// { move, reason, estimate }.
|
|
59
|
-
export function lateralMove({ config, from, to, mass, hold, facts, now, costs }) {
|
|
62
|
+
export function lateralMove({ config, from, to, mass, hold, sameTask = false, facts, now, costs }) {
|
|
60
63
|
const order = compareRoutes(config, to, from, facts.effort);
|
|
61
64
|
if (hold && !(order > 0)) return { move: false, reason: 'hold', estimate: null };
|
|
62
65
|
const p = config.policy;
|
|
63
66
|
let estimate;
|
|
67
|
+
if (order !== -1 && sameTask && !facts.downAllowed && routeSpec(config, to).input < routeSpec(config, from).input)
|
|
68
|
+
return { move: false, reason: 'downgrade-held', estimate: null };
|
|
64
69
|
if (order === -1) {
|
|
65
70
|
estimate = { taxUsd: costs.downgradeTaxUsd(config, to, from, facts, now, p.downgradeHorizonTurns) };
|
|
71
|
+
if (sameTask && !facts.downAllowed) return { move: false, reason: 'downgrade-held', estimate };
|
|
66
72
|
// Before the first measured reply no cache of this history exists, though the tax prices an unknown one as warm:
|
|
67
73
|
// staying protects nothing.
|
|
68
74
|
const fresh = facts.historyMeasured === false;
|
package/lib/band.mjs
CHANGED
|
@@ -5,12 +5,14 @@ import {
|
|
|
5
5
|
classifierStatus,
|
|
6
6
|
classifierWarns,
|
|
7
7
|
differenceShare,
|
|
8
|
+
modeLever,
|
|
8
9
|
modelColor,
|
|
9
10
|
modelName,
|
|
10
11
|
NOT_A_TIER_REASON,
|
|
11
12
|
percentPair,
|
|
12
13
|
reasonActivity,
|
|
13
14
|
routeLabel,
|
|
15
|
+
routingModeText,
|
|
14
16
|
SAVINGS_COLOR,
|
|
15
17
|
shortReason,
|
|
16
18
|
TIER_COLOR,
|
|
@@ -162,7 +164,11 @@ export function bandSegments(config, view, usage, actions) {
|
|
|
162
164
|
)
|
|
163
165
|
: null;
|
|
164
166
|
if (!view.tier && view.phase === 'ready' && !view.actualModel)
|
|
165
|
-
return [
|
|
167
|
+
return [
|
|
168
|
+
segment(0, meter(null, true), part(' Routing on · ready', { dimColor: true })),
|
|
169
|
+
pin,
|
|
170
|
+
segment(5, part(routingModeText(view), { dimColor: true })),
|
|
171
|
+
].filter(Boolean);
|
|
166
172
|
const main = segment(
|
|
167
173
|
0,
|
|
168
174
|
// The space after the pill keeps its background off the next word.
|
|
@@ -193,6 +199,7 @@ export function bandSegments(config, view, usage, actions) {
|
|
|
193
199
|
main,
|
|
194
200
|
pin,
|
|
195
201
|
segment(1, part(shortReason(view.reason, reasonActivity(config, view)), { dimColor: true })),
|
|
202
|
+
segment(5, part([routingModeText(view), modeLever(config, view)].filter(Boolean).join(' · '), { dimColor: true })),
|
|
196
203
|
support ? segment(4, part(support, { dimColor: true })) : null,
|
|
197
204
|
Number.isFinite(switchUsd) && switchUsd >= 0.005
|
|
198
205
|
? segment(4, part(`switch cost ≈ ${usd(switchUsd)}`, { dimColor: true }))
|
package/lib/config.mjs
CHANGED
|
@@ -16,6 +16,10 @@ const withoutShadow = (file) =>
|
|
|
16
16
|
Object.fromEntries(Object.entries(file).filter(([key, value]) => !(value === 'shadow' && MODE_KEYS.includes(key))));
|
|
17
17
|
// A `subagents.types` value that asks the classifier instead of naming a model.
|
|
18
18
|
export const CLASSIFY = 'classify';
|
|
19
|
+
// The routing mode trades cost against answer quality (lib/modes.mjs). Not the session's auto|manual mode.
|
|
20
|
+
export const ROUTING_MODES = ['cost', 'balanced', 'quality'];
|
|
21
|
+
// The modes that patch the policy; `balanced` is the base policy itself.
|
|
22
|
+
const PATCHED_MODES = ['cost', 'quality'];
|
|
19
23
|
const SONNET_HIGH = { model: 'sonnet', effort: 'high' };
|
|
20
24
|
export const DEFAULTS = {
|
|
21
25
|
baselineTier: 'low',
|
|
@@ -128,6 +132,37 @@ export const DEFAULTS = {
|
|
|
128
132
|
// classifier yet: the probe results record whether the label was right, not its probability.
|
|
129
133
|
activityMass: 0.6,
|
|
130
134
|
},
|
|
135
|
+
// `cost` uses a cheaper route where the turn allows it, `quality` a stronger one; `balanced` routes as 1.14. The
|
|
136
|
+
// routes stay as configured: the mode moves a tier's route one step (lib/modes.mjs). Each mode's `policy` replaces
|
|
137
|
+
// those fields of `policy` while it is active. The values started as codex-router's presets. 1.16 sets cost's upgrade
|
|
138
|
+
// bar to balanced's (2 votes from 0.75): from 3 votes at 0.85 the bar, raised by the cache write of a cold stronger
|
|
139
|
+
// model, sat at 0.92 from 60k tokens and 0.97 at 254k, so a cost turn on Haiku kept a task the classifier scored
|
|
140
|
+
// hard at 90%. Cost still moves down sooner. Not tuned on outcomes yet.
|
|
141
|
+
mode: 'balanced',
|
|
142
|
+
modes: {
|
|
143
|
+
cost: {
|
|
144
|
+
policy: {
|
|
145
|
+
upgradeVotes: 2,
|
|
146
|
+
upgradeBase: 0.75,
|
|
147
|
+
jumpConfidence: 0.98,
|
|
148
|
+
downgradeVotes: 1,
|
|
149
|
+
downgradeMass: 0.8,
|
|
150
|
+
minDwellRequests: 4,
|
|
151
|
+
escalationHoldTurns: 1,
|
|
152
|
+
},
|
|
153
|
+
},
|
|
154
|
+
quality: {
|
|
155
|
+
policy: {
|
|
156
|
+
upgradeVotes: 1,
|
|
157
|
+
upgradeBase: 0.6,
|
|
158
|
+
jumpConfidence: 0.9,
|
|
159
|
+
downgradeVotes: 3,
|
|
160
|
+
downgradeMass: 0.95,
|
|
161
|
+
minDwellRequests: 16,
|
|
162
|
+
escalationHoldTurns: 4,
|
|
163
|
+
},
|
|
164
|
+
},
|
|
165
|
+
},
|
|
131
166
|
// The active entry of `classifiers`. `api` names the wire protocol (CLASSIFIER_APIS); a user entry without one speaks
|
|
132
167
|
// `system-one`. `keyOption` names the plugin option that holds the bearer key, or null for no key (a local server);
|
|
133
168
|
// a `{name}` in `endpoint` is filled from the plugin option of that name. Both must be in CLASSIFIER_OPTIONS.
|
|
@@ -372,6 +407,12 @@ export function withActivities(file, draft) {
|
|
|
372
407
|
};
|
|
373
408
|
}
|
|
374
409
|
|
|
410
|
+
// The router.json `file` with the profile's routing mode; the default is written as no key.
|
|
411
|
+
export function withMode(file, mode) {
|
|
412
|
+
const { mode: _, ...rest } = file;
|
|
413
|
+
return mode === DEFAULTS.mode ? rest : { ...rest, mode };
|
|
414
|
+
}
|
|
415
|
+
|
|
375
416
|
// The pane's policy controls and the router.json fields they stand for. The classifier deadline saves on its own.
|
|
376
417
|
const TUNING_FIELDS = {
|
|
377
418
|
downgradeVotes: ['policy', 'downgradeVotes'],
|
|
@@ -462,7 +503,7 @@ function checkShape(file) {
|
|
|
462
503
|
if (file && typeof file === 'object' && Object.hasOwn(file, 'jev'))
|
|
463
504
|
throw new Error(`router.json uses the 1.1 jev section; ${MIGRATION_HINT}`);
|
|
464
505
|
checkObject(file, 'router.json', Object.keys(DEFAULTS));
|
|
465
|
-
const { routes, activities, models, cache, policy, classifiers, context, subagents } = file;
|
|
506
|
+
const { routes, activities, models, cache, policy, modes, classifiers, context, subagents } = file;
|
|
466
507
|
if (routes !== undefined) {
|
|
467
508
|
checkObject(routes, 'routes', TIERS);
|
|
468
509
|
for (const [tier, route] of Object.entries(routes)) checkObject(route, `routes.${tier}`, ROUTE_KEYS);
|
|
@@ -490,6 +531,13 @@ function checkShape(file) {
|
|
|
490
531
|
if (cache[map] !== undefined) checkObject(cache[map], `cache.${map}`);
|
|
491
532
|
}
|
|
492
533
|
if (policy !== undefined) checkObject(policy, 'policy', Object.keys(DEFAULTS.policy));
|
|
534
|
+
if (modes !== undefined) {
|
|
535
|
+
checkObject(modes, 'modes', PATCHED_MODES);
|
|
536
|
+
for (const [mode, entry] of Object.entries(modes)) {
|
|
537
|
+
checkObject(entry, `modes.${mode}`, ['policy']);
|
|
538
|
+
if (entry.policy !== undefined) checkObject(entry.policy, `modes.${mode}.policy`, Object.keys(DEFAULTS.policy));
|
|
539
|
+
}
|
|
540
|
+
}
|
|
493
541
|
if (classifiers !== undefined) {
|
|
494
542
|
checkObject(classifiers, 'classifiers');
|
|
495
543
|
for (const [id, entry] of Object.entries(classifiers)) checkObject(entry, `classifiers.${id}`, CLASSIFIER_KEYS);
|
|
@@ -536,22 +584,11 @@ function validate(config) {
|
|
|
536
584
|
}
|
|
537
585
|
for (const [alias, model] of Object.entries(config.models)) validateModel(`models.${alias}`, model);
|
|
538
586
|
if (!TIERS.includes(config.baselineTier)) throw new Error(`baselineTier must be one of ${TIERS.join(', ')}`);
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
if (!isNumber(p[field], 1, { integer: true })) throw new Error(`policy.${field} must be a positive integer`);
|
|
545
|
-
}
|
|
546
|
-
for (const field of ['reviewEveryRequests', 'minDwellRequests']) {
|
|
547
|
-
if (!isNumber(p[field], 0, { integer: true })) throw new Error(`policy.${field} must be a non-negative integer`);
|
|
548
|
-
}
|
|
549
|
-
// The pivot divides: tax / (tax + pivot). Zero makes a free switch NaN and silently stops that switch.
|
|
550
|
-
for (const side of ['upgrade', 'downgrade']) {
|
|
551
|
-
if (!isNumber(p[`${side}Slope`], 0)) throw new Error(`policy.${side}Slope must be a non-negative number`);
|
|
552
|
-
if (!isNumber(p[`${side}PivotUsd`], 0, { above: true })) throw new Error(`policy.${side}PivotUsd must be positive`);
|
|
553
|
-
}
|
|
554
|
-
if (!isNumber(p.cashCapUsd, 0)) throw new Error('policy.cashCapUsd must be a non-negative number');
|
|
587
|
+
validatePolicy(config.policy, 'policy');
|
|
588
|
+
if (!ROUTING_MODES.includes(config.mode)) throw new Error(`mode must be one of ${ROUTING_MODES.join(', ')}`);
|
|
589
|
+
// A patch is checked as the policy it makes; the base passed already, so an error is the patch's field.
|
|
590
|
+
for (const mode of PATCHED_MODES)
|
|
591
|
+
validatePolicy({ ...config.policy, ...config.modes[mode]?.policy }, `modes.${mode}.policy`);
|
|
555
592
|
const { cache, context } = config;
|
|
556
593
|
for (const map of ['writeMultiplier', 'ttlMs']) {
|
|
557
594
|
for (const [ttl, value] of Object.entries(cache[map]))
|
|
@@ -579,6 +616,24 @@ function validate(config) {
|
|
|
579
616
|
}
|
|
580
617
|
}
|
|
581
618
|
|
|
619
|
+
function validatePolicy(p, at) {
|
|
620
|
+
for (const field of ['upgradeBase', 'jumpConfidence', 'downgradeMass', 'continuationMass', 'activityMass']) {
|
|
621
|
+
if (!(isNumber(p[field], 0) && p[field] <= 1)) throw new Error(`${at}.${field} must be between 0 and 1`);
|
|
622
|
+
}
|
|
623
|
+
for (const field of ['upgradeVotes', 'downgradeVotes', 'downgradeHorizonTurns', 'escalationHoldTurns']) {
|
|
624
|
+
if (!isNumber(p[field], 1, { integer: true })) throw new Error(`${at}.${field} must be a positive integer`);
|
|
625
|
+
}
|
|
626
|
+
for (const field of ['reviewEveryRequests', 'minDwellRequests']) {
|
|
627
|
+
if (!isNumber(p[field], 0, { integer: true })) throw new Error(`${at}.${field} must be a non-negative integer`);
|
|
628
|
+
}
|
|
629
|
+
// The pivot divides: tax / (tax + pivot). Zero makes a free switch NaN and silently stops that switch.
|
|
630
|
+
for (const side of ['upgrade', 'downgrade']) {
|
|
631
|
+
if (!isNumber(p[`${side}Slope`], 0)) throw new Error(`${at}.${side}Slope must be a non-negative number`);
|
|
632
|
+
if (!isNumber(p[`${side}PivotUsd`], 0, { above: true })) throw new Error(`${at}.${side}PivotUsd must be positive`);
|
|
633
|
+
}
|
|
634
|
+
if (!isNumber(p.cashCapUsd, 0)) throw new Error(`${at}.cashCapUsd must be a non-negative number`);
|
|
635
|
+
}
|
|
636
|
+
|
|
582
637
|
function validateModel(at, model) {
|
|
583
638
|
for (const field of ['input', 'cacheRead', 'contextWindow']) {
|
|
584
639
|
if (!Number.isFinite(model[field]) || model[field] < 0)
|
package/lib/cost.mjs
CHANGED
|
@@ -82,7 +82,10 @@ export function coldWriteUsd(config, alias, tokens) {
|
|
|
82
82
|
);
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
// Routes that share a cache key (one model at two efforts, where effort keeps the cache) write the same new tokens
|
|
86
|
+
// to the same cache: switching between them costs no write. Bounding them apart would charge the new tokens as a tax.
|
|
85
87
|
export function switchingTaxUsd(config, candidate, incumbent, facts, now) {
|
|
88
|
+
if (routeCacheKey(config, candidate, facts.effort) === routeCacheKey(config, incumbent, facts.effort)) return 0;
|
|
86
89
|
const tokens = nextContextTokens(facts);
|
|
87
90
|
return (
|
|
88
91
|
inputBounds(config, candidate, tokens, facts, now).max - inputBounds(config, incumbent, tokens, facts, now).min
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// Routing decisions kept per session, so a later look can tell why a turn ran where it did: the prompt's origin, the
|
|
2
|
+
// classifier's answer, the tier and activity chosen, the reason, the route, and at the turn's end what its tools did and
|
|
3
|
+
// whether its replies were priced. Counts, labels and routes only, never text. Pure: the hook collects a turn's rows and
|
|
4
|
+
// writes them at turn.complete to the session's own $.store key under DECISIONS_PREFIX.
|
|
5
|
+
|
|
6
|
+
import { afterResetOf, isRecord } from './records.mjs';
|
|
7
|
+
|
|
8
|
+
export const DECISIONS_PREFIX = 'decisions:v1:';
|
|
9
|
+
// The newest rows a session keeps, and the sessions kept: $.store holds 4 MiB for every key together, and a row is about
|
|
10
|
+
// 300 bytes, so the records stay under 400 KB. Upgrade trigger: a review needs more history than this, or the store nears its limit; then roll rows into
|
|
11
|
+
// per-day counts.
|
|
12
|
+
export const MAX_ROWS = 60;
|
|
13
|
+
export const MAX_RECORDS = 20;
|
|
14
|
+
export const RETAIN_MS = 30 * 24 * 60 * 60 * 1000;
|
|
15
|
+
export const ROW_KINDS = ['start', 'review', 'observed', 'end'];
|
|
16
|
+
|
|
17
|
+
// Where a prompt came from, as the engine stamps it (prompt.submit's PromptOrigin.kind). Progress origins report on the
|
|
18
|
+
// running work, so their turn is the same task: the classifier judges the active goal with the report as dialogue, and a
|
|
19
|
+
// cheaper route still waits for the checkpoint. Goal origins are a person's or another agent's request, which may start
|
|
20
|
+
// a new task. Anything else (unclassified, a kind a later engine adds, or none seen) routes as before and leaves the
|
|
21
|
+
// goal as it was.
|
|
22
|
+
export const PROGRESS_ORIGINS = ['task-notification', 'observer', 'observer-activity', 'auto-continuation'];
|
|
23
|
+
export const GOAL_ORIGINS = [
|
|
24
|
+
'composer',
|
|
25
|
+
'bridge',
|
|
26
|
+
'sdk',
|
|
27
|
+
'slack-ping',
|
|
28
|
+
'channel',
|
|
29
|
+
'scheduled-trigger',
|
|
30
|
+
'plugin',
|
|
31
|
+
'peer',
|
|
32
|
+
'peer-send-message',
|
|
33
|
+
'projects-relay',
|
|
34
|
+
'coordinator',
|
|
35
|
+
];
|
|
36
|
+
const ORIGIN = /^[a-z][a-z-]{0,39}$/;
|
|
37
|
+
export const originKind = (origin) =>
|
|
38
|
+
typeof origin?.kind === 'string' && ORIGIN.test(origin.kind) ? origin.kind : 'unknown';
|
|
39
|
+
export const isProgressOrigin = (kind) => PROGRESS_ORIGINS.includes(kind);
|
|
40
|
+
export const isGoalOrigin = (kind) => GOAL_ORIGINS.includes(kind);
|
|
41
|
+
|
|
42
|
+
// Submissions waiting for their turn: a prompt typed or delivered while a turn runs starts its own turn later, or never
|
|
43
|
+
// (one delivered into the running turn). Each waits at most QUEUE_MS, and at most QUEUE_MAX wait.
|
|
44
|
+
export const QUEUE_MS = 10 * 60 * 1000;
|
|
45
|
+
export const QUEUE_MAX = 16;
|
|
46
|
+
const sameText = (a, b) => typeof a === 'string' && typeof b === 'string' && a.trim() === b.trim();
|
|
47
|
+
|
|
48
|
+
// The queue with one more submission, the expired and the oldest past the bound dropped.
|
|
49
|
+
export function queueOrigin(queue, { kind, text, at }) {
|
|
50
|
+
return [...queue.filter((entry) => at - entry.at < QUEUE_MS), { kind, text, at }].slice(-QUEUE_MAX);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// The origin of a turn that starts with `text` at `at`, taken from the queue: the oldest waiting submission with that
|
|
54
|
+
// text. Returns { kind, queue }; kind 'unknown' when none matches.
|
|
55
|
+
export function takeOrigin(queue, text, at) {
|
|
56
|
+
const live = queue.filter((entry) => at - entry.at < QUEUE_MS);
|
|
57
|
+
const i = live.findIndex((entry) => sameText(entry.text, text));
|
|
58
|
+
if (i === -1) return { kind: 'unknown', queue: live };
|
|
59
|
+
return { kind: live[i].kind, queue: [...live.slice(0, i), ...live.slice(i + 1)] };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const round = (n, digits = 2) => (Number.isFinite(n) ? Math.round(n * 10 ** digits) / 10 ** digits : null);
|
|
63
|
+
const label = (value) => (typeof value === 'string' && value.length > 0 && value.length <= 80 ? value : null);
|
|
64
|
+
const count = (n) => (Number.isSafeInteger(n) && n > 0 ? n : 0);
|
|
65
|
+
|
|
66
|
+
// One routing decision as a row: `kind` start (the turn's first step), review (a classifier review applied inside the
|
|
67
|
+
// turn) or observed (a move on the turn's own tool calls). `advice` is the classifier's answer or null, `error` why
|
|
68
|
+
// there is none, `decision` the route chosen (chooseRoute's), `facts` what it was chosen on, `sameTask` sameTaskOf.
|
|
69
|
+
export function decisionRow({
|
|
70
|
+
at,
|
|
71
|
+
kind,
|
|
72
|
+
origin = null,
|
|
73
|
+
mode = null,
|
|
74
|
+
advice = null,
|
|
75
|
+
error = null,
|
|
76
|
+
decision,
|
|
77
|
+
facts = {},
|
|
78
|
+
sameTask = null,
|
|
79
|
+
}) {
|
|
80
|
+
const activity = advice?.activity ?? null;
|
|
81
|
+
const { downAllowed, contextTokens } = /** @type {{ downAllowed?: boolean, contextTokens?: number }} */ (facts);
|
|
82
|
+
return {
|
|
83
|
+
at,
|
|
84
|
+
kind,
|
|
85
|
+
origin: label(origin),
|
|
86
|
+
mode: label(mode),
|
|
87
|
+
cls: label(advice?.choice),
|
|
88
|
+
p: round(advice?.probabilities?.[advice?.choice]),
|
|
89
|
+
cont: round(advice?.continuation),
|
|
90
|
+
act: label(activity?.choice),
|
|
91
|
+
actP: round(activity?.probabilities?.[activity?.choice]),
|
|
92
|
+
err: label(error),
|
|
93
|
+
tier: label(decision?.tier),
|
|
94
|
+
activity: label(decision?.activity),
|
|
95
|
+
reason: label(decision?.reason),
|
|
96
|
+
route: decision?.model ? label(`${decision.model}@${decision.effort ?? 'session'}`) : null,
|
|
97
|
+
lateral: label(decision?.lateral),
|
|
98
|
+
pinned: Boolean(decision?.pinned),
|
|
99
|
+
same: typeof sameTask === 'boolean' ? sameTask : null,
|
|
100
|
+
down: typeof downAllowed === 'boolean' ? downAllowed : null,
|
|
101
|
+
ctx: Number.isFinite(contextTokens) ? Math.round(contextTokens / 1000) : null,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// The turn's end: the bucket its tools fall in (observedActivity), its routed requests, how many were priced, and the
|
|
106
|
+
// unpriced ones by cause (UNPRICED_CAUSES in activity-stats.mjs).
|
|
107
|
+
export function endRow({ at, observed = null, requests = 0, priced = 0, unpriced = {} }) {
|
|
108
|
+
const causes = Object.fromEntries(
|
|
109
|
+
Object.entries(unpriced ?? {}).filter(([cause, n]) => label(cause) && Number.isSafeInteger(n) && n > 0),
|
|
110
|
+
);
|
|
111
|
+
return {
|
|
112
|
+
at,
|
|
113
|
+
kind: 'end',
|
|
114
|
+
observed: label(observed),
|
|
115
|
+
requests: count(requests),
|
|
116
|
+
priced: count(priced),
|
|
117
|
+
unpriced: causes,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// A turn keeps its first row (the start) and its newest others up to MAX_TURN_ROWS: a long turn reviewed every few
|
|
122
|
+
// requests must not push the session's earlier turns out of MAX_ROWS.
|
|
123
|
+
export const MAX_TURN_ROWS = 10;
|
|
124
|
+
export function addTurnRow(rows, row) {
|
|
125
|
+
const next = [...rows, row];
|
|
126
|
+
return next.length > MAX_TURN_ROWS ? [next[0], ...next.slice(-(MAX_TURN_ROWS - 1))] : next;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// What the classifier reads on a progress turn: the active goal as the request, and the report as the newest line of
|
|
130
|
+
// the dialogue, so it judges the work still to do rather than the report's wording. Without a goal it reads the turn as
|
|
131
|
+
// before. `recentTurns` bounds the dialogue as extractFacts does.
|
|
132
|
+
export function progressInput({ prompt, turns }, { goal = null, note = null }, recentTurns) {
|
|
133
|
+
if (!goal || !note) return { prompt, turns };
|
|
134
|
+
return { prompt: goal, turns: [...turns, { role: 'user', text: note }].slice(-recentTurns) };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function emptyDecisions() {
|
|
138
|
+
return { version: 1, since: null, rows: [] };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const nullable = (check) => (v) => v === null || Boolean(check(v));
|
|
142
|
+
const ROW_FIELDS = {
|
|
143
|
+
at: Number.isFinite,
|
|
144
|
+
kind: (v) => ROW_KINDS.includes(v),
|
|
145
|
+
origin: nullable(label),
|
|
146
|
+
mode: nullable(label),
|
|
147
|
+
cls: nullable(label),
|
|
148
|
+
p: nullable(Number.isFinite),
|
|
149
|
+
cont: nullable(Number.isFinite),
|
|
150
|
+
act: nullable(label),
|
|
151
|
+
actP: nullable(Number.isFinite),
|
|
152
|
+
err: nullable(label),
|
|
153
|
+
tier: nullable(label),
|
|
154
|
+
activity: nullable(label),
|
|
155
|
+
reason: nullable(label),
|
|
156
|
+
route: nullable(label),
|
|
157
|
+
lateral: nullable(label),
|
|
158
|
+
pinned: (v) => typeof v === 'boolean',
|
|
159
|
+
same: nullable((v) => typeof v === 'boolean'),
|
|
160
|
+
down: nullable((v) => typeof v === 'boolean'),
|
|
161
|
+
ctx: nullable(Number.isFinite),
|
|
162
|
+
observed: nullable(label),
|
|
163
|
+
requests: Number.isSafeInteger,
|
|
164
|
+
priced: Number.isSafeInteger,
|
|
165
|
+
unpriced: (v) => isRecord(v) && Object.values(v).every((n) => Number.isSafeInteger(n) && n >= 0),
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
// A row with only its known fields of the right type, or null when its time or kind does not read. A row a later
|
|
169
|
+
// version wrote keeps what reads.
|
|
170
|
+
function readRow(row) {
|
|
171
|
+
if (!isRecord(row) || !ROW_FIELDS.at(row.at) || !ROW_FIELDS.kind(row.kind)) return null;
|
|
172
|
+
const out = {};
|
|
173
|
+
for (const [field, check] of Object.entries(ROW_FIELDS))
|
|
174
|
+
if (Object.hasOwn(row, field) && check(row[field])) out[field] = row[field];
|
|
175
|
+
return out;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// The stored value with the rows that read, else an empty record: a hand edit never breaks a turn.
|
|
179
|
+
export function readDecisions(value) {
|
|
180
|
+
if (!isRecord(value) || value.version !== 1 || !Array.isArray(value.rows)) return emptyDecisions();
|
|
181
|
+
if (!(value.since === null || Number.isFinite(value.since))) return emptyDecisions();
|
|
182
|
+
return { version: 1, since: value.since, rows: value.rows.map(readRow).filter(Boolean).slice(-MAX_ROWS) };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// A record whose first row came before the last reset at `resetAt` (ms) reads as empty.
|
|
186
|
+
export const decisionsAfterReset = (record, resetAt) => afterResetOf(record, resetAt, emptyDecisions);
|
|
187
|
+
|
|
188
|
+
// The record with a finished turn's rows added at `now`, the oldest dropped past MAX_ROWS. No rows: the record itself.
|
|
189
|
+
export function appendDecisions(record, rows, now) {
|
|
190
|
+
if (!rows.length) return record;
|
|
191
|
+
return { version: 1, since: record.since ?? now, rows: [...record.rows, ...rows].slice(-MAX_ROWS) };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Which of the stored `[key, value]` records the store can let go at `now`: those whose newest row is older than
|
|
195
|
+
// RETAIN_MS, then past `max` records the ones whose newest row is oldest. A value that does not read as version 1 keeps
|
|
196
|
+
// its key: a later version may have written it.
|
|
197
|
+
export function prunedDecisions(entries, now, max = MAX_RECORDS) {
|
|
198
|
+
const dated = entries
|
|
199
|
+
.filter(([, value]) => isRecord(value) && value.version === 1)
|
|
200
|
+
.map(([key, value]) => {
|
|
201
|
+
const record = readDecisions(value);
|
|
202
|
+
return [key, record.rows.at(-1)?.at ?? record.since ?? 0];
|
|
203
|
+
});
|
|
204
|
+
const gone = new Set(dated.filter(([, last]) => now - last > RETAIN_MS).map(([key]) => key));
|
|
205
|
+
const kept = dated.filter(([key]) => !gone.has(key)).sort((a, b) => b[1] - a[1]);
|
|
206
|
+
for (const [key] of kept.slice(max)) gone.add(key);
|
|
207
|
+
return gone;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const pad2 = (n) => String(n).padStart(2, '0');
|
|
211
|
+
const clock = (ms) => {
|
|
212
|
+
const d = new Date(ms);
|
|
213
|
+
return `${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}`;
|
|
214
|
+
};
|
|
215
|
+
const share = (p) => (Number.isFinite(p) ? ` ${Math.round(p * 100)}%` : '');
|
|
216
|
+
|
|
217
|
+
function endLine(row) {
|
|
218
|
+
const unpriced = Object.entries(row.unpriced ?? {}).map(([cause, k]) => `${cause} ${k}`);
|
|
219
|
+
return [
|
|
220
|
+
clock(row.at),
|
|
221
|
+
'end ',
|
|
222
|
+
`tools ${row.observed ?? '?'}`,
|
|
223
|
+
`${row.requests} request${row.requests === 1 ? '' : 's'}`,
|
|
224
|
+
`${row.priced} priced`,
|
|
225
|
+
unpriced.length ? `unpriced ${unpriced.join(', ')}` : null,
|
|
226
|
+
]
|
|
227
|
+
.filter(Boolean)
|
|
228
|
+
.join(' ');
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// The newest `n` rows as lines for `/router decisions`, oldest first.
|
|
232
|
+
export function decisionLines(rows, n = 20) {
|
|
233
|
+
if (!rows.length) return ['No routing decisions recorded in this session yet.'];
|
|
234
|
+
return rows.slice(-n).map((row) => {
|
|
235
|
+
if (row.kind === 'end') return endLine(row);
|
|
236
|
+
const activity = row.act ? ` ${row.act}${share(row.actP)}` : '';
|
|
237
|
+
const asked = row.cls ? `${row.cls}${share(row.p)}${activity}` : (row.err ?? 'no answer');
|
|
238
|
+
const task = row.same === true ? 'same task' : row.same === false ? 'new task' : null;
|
|
239
|
+
return [
|
|
240
|
+
clock(row.at),
|
|
241
|
+
row.kind.padEnd(8),
|
|
242
|
+
row.origin ?? 'unknown',
|
|
243
|
+
`asked ${asked}`,
|
|
244
|
+
task,
|
|
245
|
+
`-> ${row.tier ?? '-'}${row.activity ? ` ${row.activity}` : ''} ${row.route ?? ''}`.trimEnd(),
|
|
246
|
+
`(${row.reason ?? '?'})`,
|
|
247
|
+
row.pinned ? 'pinned' : null,
|
|
248
|
+
]
|
|
249
|
+
.filter(Boolean)
|
|
250
|
+
.join(' ');
|
|
251
|
+
});
|
|
252
|
+
}
|