@alexeiled/claude-router 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +2 -2
- package/CHANGELOG.md +26 -0
- package/README.md +12 -10
- package/docs/architecture.md +59 -17
- package/docs/configuration.md +86 -32
- package/docs/evaluation.md +52 -1
- package/docs/native-router.md +37 -28
- package/docs/router-band.svg +83 -42
- package/docs/router-pane-now.svg +97 -71
- package/docs/router-pane-routing.svg +103 -32
- package/docs/user-guide.md +73 -27
- package/hooks/native-router.mjs +196 -32
- package/lib/activity-stats.mjs +179 -0
- package/lib/activity.mjs +77 -0
- package/lib/band.mjs +32 -11
- package/lib/classifier-apis.mjs +133 -69
- package/lib/classifier-client.mjs +63 -28
- package/lib/classifier-contract.mjs +48 -0
- package/lib/config.mjs +138 -4
- package/lib/cost.mjs +11 -11
- package/lib/display.mjs +89 -4
- package/lib/facts.mjs +48 -0
- package/lib/panel.mjs +287 -29
- package/lib/policy.mjs +45 -26
- package/lib/route.mjs +156 -51
- package/lib/view.mjs +34 -4
- package/package.json +1 -1
- package/types/index.d.ts +72 -1
package/hooks/native-router.mjs
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
|
+
import {
|
|
2
|
+
advanceRun,
|
|
3
|
+
emptySession,
|
|
4
|
+
emptyStore,
|
|
5
|
+
readStore,
|
|
6
|
+
recordStore,
|
|
7
|
+
recordTurn,
|
|
8
|
+
STORE_KEY,
|
|
9
|
+
turnActivity,
|
|
10
|
+
} from '../lib/activity-stats.mjs';
|
|
1
11
|
import { renderBand, switchToast } from '../lib/band.mjs';
|
|
2
12
|
import { ClassifierClient } from '../lib/classifier-client.mjs';
|
|
3
13
|
import { resolveCredentials } from '../lib/classifier-contract.mjs';
|
|
4
14
|
import {
|
|
15
|
+
ACTIVITY_MODES,
|
|
5
16
|
activeClassifier,
|
|
6
17
|
DEFAULTS,
|
|
18
|
+
editActivity,
|
|
19
|
+
effectiveActivities,
|
|
7
20
|
effectiveRoutes,
|
|
8
21
|
loadConfig,
|
|
9
22
|
MIGRATION_HINT,
|
|
10
23
|
supportedVersion,
|
|
11
24
|
TIERS,
|
|
12
25
|
tuningOf,
|
|
26
|
+
withActivities,
|
|
13
27
|
withClassifier,
|
|
14
28
|
withClassifierTimeout,
|
|
15
29
|
withRoutes,
|
|
@@ -17,6 +31,7 @@ import {
|
|
|
17
31
|
} from '../lib/config.mjs';
|
|
18
32
|
import { changedLeaves, notSaved, restored, rewrittenConfig } from '../lib/config-file.mjs';
|
|
19
33
|
import {
|
|
34
|
+
activityDetailLines,
|
|
20
35
|
classifierStatus,
|
|
21
36
|
classifierTimed,
|
|
22
37
|
GATEWAY_CLEANUP,
|
|
@@ -24,9 +39,10 @@ import {
|
|
|
24
39
|
missingCredentials,
|
|
25
40
|
NOT_A_TIER_REASON,
|
|
26
41
|
} from '../lib/display.mjs';
|
|
27
|
-
import { clip } from '../lib/facts.mjs';
|
|
42
|
+
import { clip, observedActivity, promptIndex } from '../lib/facts.mjs';
|
|
28
43
|
import { renderPanel, routeDraftOf, routingChanges } from '../lib/panel.mjs';
|
|
29
44
|
import {
|
|
45
|
+
cellForModel,
|
|
30
46
|
chooseRoute,
|
|
31
47
|
continueRoute,
|
|
32
48
|
emptyLoop,
|
|
@@ -37,9 +53,8 @@ import {
|
|
|
37
53
|
observeResponse,
|
|
38
54
|
prepareLoop,
|
|
39
55
|
resetHistory,
|
|
40
|
-
tierForModel,
|
|
41
56
|
} from '../lib/route.mjs';
|
|
42
|
-
import { CLEARED_READINGS, healthOf, initialView, responseMetrics } from '../lib/view.mjs';
|
|
57
|
+
import { CLEARED_READINGS, continuationView, healthOf, initialView, responseMetrics } from '../lib/view.mjs';
|
|
43
58
|
|
|
44
59
|
// The engine follows $ only into functions declared in this file, never across an import: every helper that takes $
|
|
45
60
|
// lives here, and the pure parts live in lib/.
|
|
@@ -68,6 +83,12 @@ function createRouter(options) {
|
|
|
68
83
|
turnControllers: new Map(),
|
|
69
84
|
view: null,
|
|
70
85
|
modes: new Map(),
|
|
86
|
+
// Each routed turn's activity record until turn.complete adds it to the stats, and the session's open run of
|
|
87
|
+
// one activity, flushed to the store when it ends.
|
|
88
|
+
turnActivities: new Map(),
|
|
89
|
+
run: null,
|
|
90
|
+
// Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
|
|
91
|
+
statsResets: 0,
|
|
71
92
|
};
|
|
72
93
|
}
|
|
73
94
|
|
|
@@ -98,7 +119,7 @@ async function modeOf($, router, fallback = 'auto') {
|
|
|
98
119
|
|
|
99
120
|
// A saved preference wins; a fresh session starts with routing on for a model some tier routes to, and off for any other.
|
|
100
121
|
async function startMode($, router, model) {
|
|
101
|
-
const mode = await modeOf($, router,
|
|
122
|
+
const mode = await modeOf($, router, cellForModel(router.config, model) === null ? 'manual' : 'auto');
|
|
102
123
|
router.modes.set(await $.session.id(), mode);
|
|
103
124
|
return mode;
|
|
104
125
|
}
|
|
@@ -155,6 +176,37 @@ async function setPin($, router, tier) {
|
|
|
155
176
|
return `${tier} pinned for the next turn and its tool continuations.`;
|
|
156
177
|
}
|
|
157
178
|
|
|
179
|
+
// `/router [auto|off|pin <tier>|activities <mode>]`; with no action, the pane, or the status text without a surface.
|
|
180
|
+
async function routerCommand($, router, args) {
|
|
181
|
+
const [action, value] = args.trim().split(/\s+/);
|
|
182
|
+
if (action === 'auto' || action === 'off') {
|
|
183
|
+
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
184
|
+
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
185
|
+
}
|
|
186
|
+
if (action === 'pin')
|
|
187
|
+
return { text: TIERS.includes(value) ? await setPin($, router, value) : `Choose ${TIERS.join(', ')}.` };
|
|
188
|
+
if (action === 'activities')
|
|
189
|
+
return {
|
|
190
|
+
text: ACTIVITY_MODES.includes(value)
|
|
191
|
+
? await setActivityRouting($, router, value)
|
|
192
|
+
: `Choose activities ${ACTIVITY_MODES.join(', ')}.`,
|
|
193
|
+
};
|
|
194
|
+
const storedView = await readView($, router);
|
|
195
|
+
const view = { ...storedView, mode: await modeOf($, router, storedView.mode) };
|
|
196
|
+
if (!(await $.session.surfaces()).length) return { text: detailText(router.config, view) };
|
|
197
|
+
await openPane($, router);
|
|
198
|
+
return {};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// `/router activities <mode>`: the pane's write, so the file keeps the rest and Undo in the pane restores it.
|
|
202
|
+
async function setActivityRouting($, router, mode) {
|
|
203
|
+
const view = router.view ?? (await readView($, router));
|
|
204
|
+
if (view.phase === 'unavailable') return `Routing unavailable: ${view.error}.`;
|
|
205
|
+
if (mode === router.config.activityRouting) return `Activity routing is already ${mode}.`;
|
|
206
|
+
await paneActions($, router, view).activityRouting(mode);
|
|
207
|
+
return router.view.notice;
|
|
208
|
+
}
|
|
209
|
+
|
|
158
210
|
// The environment variables that stand in for a classifier option: its upper-case name, then the one Claude Code
|
|
159
211
|
// exports for a plugin option. Spelled out because $.env.get takes literal names only. Cases: CLASSIFIER_OPTIONS.
|
|
160
212
|
async function envSettingOf($, name) {
|
|
@@ -236,6 +288,7 @@ function detailText(config, view) {
|
|
|
236
288
|
`Selected: ${view.selectedModel ?? 'not selected'}`,
|
|
237
289
|
`Observed: ${view.actualModel ?? 'no response yet'}`,
|
|
238
290
|
`Reason: ${view.reason}`,
|
|
291
|
+
...activityDetailLines(config, view),
|
|
239
292
|
view.error || missing ? classifierStatus(config, view.error ?? missing) : `${activeClassifier(config).label} ready`,
|
|
240
293
|
`Context: ${view.contextKnown ? `${view.contextTokens} tokens (estimate)` : 'unknown'}`,
|
|
241
294
|
`Observed cache: ${view.cacheRead ?? 'unknown'} read, ${view.cacheWrite ?? 'unknown'} written tokens`,
|
|
@@ -348,6 +401,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
348
401
|
if (!written.isSet || controller.signal.aborted) return null;
|
|
349
402
|
if (previous?.model && previous.model !== selected.decision.model && !selected.decision.pinned)
|
|
350
403
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
404
|
+
const activity = turnActivity(cfg, result.advice, previous, selected.decision);
|
|
405
|
+
router.turnActivities.set(e.turnId, { ...activity, sessionId, requests: 0, inputTokens: 0, outputTokens: 0 });
|
|
351
406
|
await updateView($, router, {
|
|
352
407
|
phase: 'routed',
|
|
353
408
|
selectedModel: selected.decision.model,
|
|
@@ -355,6 +410,7 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
355
410
|
tier: selected.decision.tier,
|
|
356
411
|
effort: selected.decision.effort,
|
|
357
412
|
reason: selected.decision.reason,
|
|
413
|
+
activity: activity.label,
|
|
358
414
|
contextTokens: context.tokens,
|
|
359
415
|
contextKnown: context.known,
|
|
360
416
|
comparison: selected.decision.comparison,
|
|
@@ -368,6 +424,9 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
368
424
|
adviceChoice: result.advice?.choice ?? null,
|
|
369
425
|
probabilities: result.advice?.probabilities ?? null,
|
|
370
426
|
estimate: selected.decision.estimate ?? null,
|
|
427
|
+
activityChoice: activity.predicted,
|
|
428
|
+
activityProbabilities: result.advice?.activity?.probabilities ?? null,
|
|
429
|
+
wouldRoute: selected.decision.wouldRoute ?? null,
|
|
371
430
|
}
|
|
372
431
|
: {}),
|
|
373
432
|
});
|
|
@@ -379,6 +438,76 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
379
438
|
}
|
|
380
439
|
}
|
|
381
440
|
|
|
441
|
+
// A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
|
|
442
|
+
// turn's activity record.
|
|
443
|
+
async function publishReply($, router, turnId, response, tier) {
|
|
444
|
+
const turn = router.turnActivities.get(turnId);
|
|
445
|
+
const metrics = responseMetrics(router.view, response, tier, tier ? (turn?.label ?? null) : null);
|
|
446
|
+
await updateView($, router, metrics);
|
|
447
|
+
if (turn)
|
|
448
|
+
router.turnActivities.set(turnId, {
|
|
449
|
+
...turn,
|
|
450
|
+
requests: turn.requests + 1,
|
|
451
|
+
inputTokens: turn.inputTokens + (metrics.inputTokens ?? 0),
|
|
452
|
+
outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
457
|
+
// break a turn: a failed read or write loses this turn's counts only.
|
|
458
|
+
async function recordActivity($, router, turn) {
|
|
459
|
+
try {
|
|
460
|
+
if ((await $.session.id()) !== turn.sessionId) return;
|
|
461
|
+
const messages = await $.session.messages({ as: 'api' });
|
|
462
|
+
const start = promptIndex(messages);
|
|
463
|
+
if (start < 0) return;
|
|
464
|
+
const observed = observedActivity(messages, start);
|
|
465
|
+
const view = router.view ?? (await readView($, router));
|
|
466
|
+
// The session may have changed during the reads; this turn is not the new session's.
|
|
467
|
+
if ((await $.session.id()) !== turn.sessionId) return;
|
|
468
|
+
// Advanced before any further await, so a session change meanwhile cannot carry this run into the new session.
|
|
469
|
+
const { run, ended } = advanceRun(router.run, turn.answer);
|
|
470
|
+
router.run = run;
|
|
471
|
+
await updateView($, router, {
|
|
472
|
+
activityStats: recordTurn(view.activityStats ?? emptySession(), {
|
|
473
|
+
activity: turn.label,
|
|
474
|
+
answer: turn.answer,
|
|
475
|
+
observed,
|
|
476
|
+
requests: turn.requests,
|
|
477
|
+
inputTokens: turn.inputTokens,
|
|
478
|
+
outputTokens: turn.outputTokens,
|
|
479
|
+
tierSwitches: turn.switched === 'tier' ? 1 : 0,
|
|
480
|
+
activitySwitches: turn.switched === 'activity' ? 1 : 0,
|
|
481
|
+
wouldDiffer: turn.wouldDiffer,
|
|
482
|
+
shadow: turn.mode === 'shadow',
|
|
483
|
+
}),
|
|
484
|
+
});
|
|
485
|
+
const resets = router.statsResets;
|
|
486
|
+
const store = readStore(await $.store.get(STORE_KEY));
|
|
487
|
+
// Writing counts read before a reset would bring them back; this turn's are lost instead.
|
|
488
|
+
if (router.statsResets !== resets) return;
|
|
489
|
+
await $.store.set(
|
|
490
|
+
STORE_KEY,
|
|
491
|
+
recordStore(store, {
|
|
492
|
+
predicted: turn.predicted,
|
|
493
|
+
observed,
|
|
494
|
+
runEnded: ended,
|
|
495
|
+
lateral: turn.lateral,
|
|
496
|
+
wouldDiffer: turn.wouldDiffer,
|
|
497
|
+
}),
|
|
498
|
+
);
|
|
499
|
+
} catch {}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// The session's open run ends with it; without this flush it is never counted.
|
|
503
|
+
async function flushRun($, run) {
|
|
504
|
+
try {
|
|
505
|
+
const store = readStore(await $.store.get(STORE_KEY));
|
|
506
|
+
const ended = { predicted: null, observed: null, runEnded: run, lateral: null, wouldDiffer: null };
|
|
507
|
+
await $.store.set(STORE_KEY, recordStore(store, ended));
|
|
508
|
+
} catch {}
|
|
509
|
+
}
|
|
510
|
+
|
|
382
511
|
// The pane's handlers. `view` is the view the pane was drawn from; a handler reads `router.view` first, which holds
|
|
383
512
|
// any press since.
|
|
384
513
|
function paneActions($, router, view) {
|
|
@@ -390,6 +519,18 @@ function paneActions($, router, view) {
|
|
|
390
519
|
notice: null,
|
|
391
520
|
});
|
|
392
521
|
};
|
|
522
|
+
const editCell = (activity, tier, kind, value) =>
|
|
523
|
+
updateView($, router, {
|
|
524
|
+
routeDraft: editActivity(
|
|
525
|
+
routeDraftOf(router.config, router.view ?? view),
|
|
526
|
+
router.config,
|
|
527
|
+
activity,
|
|
528
|
+
tier,
|
|
529
|
+
kind,
|
|
530
|
+
value,
|
|
531
|
+
),
|
|
532
|
+
notice: null,
|
|
533
|
+
});
|
|
393
534
|
// Every pane save adopts the file as written, which may name another classifier than before: a hand edit
|
|
394
535
|
// meanwhile, or a row press. Then the new classifier starts clean and the old one's readings leave the view;
|
|
395
536
|
// an unavailable reason stays.
|
|
@@ -410,7 +551,10 @@ function paneActions($, router, view) {
|
|
|
410
551
|
// follows the file, and the file becomes what the draft compares against, so picking a value the write replaced
|
|
411
552
|
// counts as a change again.
|
|
412
553
|
const rebased = ({ routeDraft, tuning, tuningBase }) => {
|
|
413
|
-
const routes = routeDraft &&
|
|
554
|
+
const routes = routeDraft && {
|
|
555
|
+
...effectiveRoutes(routeDraft, router.config),
|
|
556
|
+
...effectiveActivities(routeDraft, router.config),
|
|
557
|
+
};
|
|
414
558
|
const edited = Object.entries(tuning ?? {}).filter(([key, value]) => value !== tuningBase?.[key]);
|
|
415
559
|
return {
|
|
416
560
|
routeDraft: routes ? { ...routes, base: routeDraftOf(router.config, {}).base } : null,
|
|
@@ -489,17 +633,49 @@ function paneActions($, router, view) {
|
|
|
489
633
|
return write(
|
|
490
634
|
`${changes} routing change${changes === 1 ? '' : 's'}`,
|
|
491
635
|
(file) => {
|
|
492
|
-
|
|
636
|
+
// withActivities reads the routes withRoutes wrote: a removed built-in override is saved as its tier's route.
|
|
637
|
+
const routed = routeDraft ? withActivities(withRoutes(file, routeDraft), routeDraft) : file;
|
|
493
638
|
return edited.length ? withTuning(routed, { ...base, ...after }, base) : routed;
|
|
494
639
|
},
|
|
495
640
|
{ routeDraft: null, tuning: null, tuningBase: null },
|
|
496
641
|
);
|
|
497
642
|
},
|
|
643
|
+
activityMode: (mode) =>
|
|
644
|
+
updateView($, router, {
|
|
645
|
+
routeDraft: { ...routeDraftOf(router.config, router.view ?? view), activityRouting: mode },
|
|
646
|
+
notice: null,
|
|
647
|
+
}),
|
|
648
|
+
activityModel: (activity, tier, alias) => editCell(activity, tier, 'model', alias),
|
|
649
|
+
activityEffort: (activity, tier, effort) => editCell(activity, tier, 'effort', effort),
|
|
650
|
+
addActivity: (activity, tier) => editCell(activity, tier, 'add'),
|
|
651
|
+
removeActivity: (activity, tier) => editCell(activity, tier, 'remove'),
|
|
652
|
+
// `/router activities <mode>`: the mode alone, saved at once like a classifier press, with Undo.
|
|
653
|
+
activityRouting: (mode) => {
|
|
654
|
+
const from = router.config.activityRouting;
|
|
655
|
+
if (mode === from) return undefined;
|
|
656
|
+
return write(`activity routing ${from} → ${mode}`, (file) =>
|
|
657
|
+
withActivities(file, { base: { activityRouting: from }, activityRouting: mode }),
|
|
658
|
+
);
|
|
659
|
+
},
|
|
660
|
+
resetActivityStats: async () => {
|
|
661
|
+
router.statsResets += 1;
|
|
662
|
+
router.run = null;
|
|
663
|
+
const cleared = await $.store
|
|
664
|
+
.set(STORE_KEY, emptyStore())
|
|
665
|
+
.then(() => true)
|
|
666
|
+
.catch(() => false);
|
|
667
|
+
await updateView($, router, {
|
|
668
|
+
activityStats: null,
|
|
669
|
+
notice: cleared ? 'Activity stats reset.' : 'Session activity stats reset. Saved stats could not be cleared.',
|
|
670
|
+
});
|
|
671
|
+
},
|
|
498
672
|
discardRouting: () => updateView($, router, { routeDraft: null, tuning: null, tuningBase: null, notice: null }),
|
|
673
|
+
// Routes only: pending activity edits and the mode stay in the draft.
|
|
499
674
|
resetRoutes: () =>
|
|
500
675
|
loadDefaults(
|
|
501
676
|
{
|
|
502
677
|
routeDraft: {
|
|
678
|
+
...routeDraftOf(router.config, router.view ?? view),
|
|
503
679
|
...structuredClone({ routes: DEFAULTS.routes, baselineTier: DEFAULTS.baselineTier }),
|
|
504
680
|
base: routeDraftOf(router.config, {}).base,
|
|
505
681
|
},
|
|
@@ -550,8 +726,8 @@ export function register(on, options) {
|
|
|
550
726
|
on('session.start', async ($, e, next) => {
|
|
551
727
|
await $.command.register({
|
|
552
728
|
name: 'router',
|
|
553
|
-
description: 'Open the Router pane, or switch routing: auto, off, pin <tier>.',
|
|
554
|
-
argumentHint: '[auto|off|pin <tier
|
|
729
|
+
description: 'Open the Router pane, or switch routing: auto, off, pin <tier>, activities <mode>.',
|
|
730
|
+
argumentHint: '[auto|off|pin <tier>|activities off|shadow|on]',
|
|
555
731
|
immediate: true,
|
|
556
732
|
});
|
|
557
733
|
const model = await $.session.model();
|
|
@@ -612,20 +788,7 @@ export function register(on, options) {
|
|
|
612
788
|
return result;
|
|
613
789
|
});
|
|
614
790
|
|
|
615
|
-
on('command.run', { command: 'router' },
|
|
616
|
-
const [action, tier] = e.args.trim().split(/\s+/);
|
|
617
|
-
if (action === 'auto' || action === 'off') {
|
|
618
|
-
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
619
|
-
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
620
|
-
}
|
|
621
|
-
if (action === 'pin')
|
|
622
|
-
return { text: TIERS.includes(tier) ? await setPin($, router, tier) : `Choose ${TIERS.join(', ')}.` };
|
|
623
|
-
const storedView = await readView($, router);
|
|
624
|
-
const view = { ...storedView, mode: await modeOf($, router, storedView.mode) };
|
|
625
|
-
if (!(await $.session.surfaces()).length) return { text: detailText(router.config, view) };
|
|
626
|
-
await openPane($, router);
|
|
627
|
-
return {};
|
|
628
|
-
});
|
|
791
|
+
on('command.run', { command: 'router' }, ($, e) => routerCommand($, router, e.args));
|
|
629
792
|
|
|
630
793
|
on('turn.start', (_$, e, next) => {
|
|
631
794
|
router.turnConfigs.set(e.turnId, router.config);
|
|
@@ -694,14 +857,7 @@ export function register(on, options) {
|
|
|
694
857
|
availableModels,
|
|
695
858
|
effort: e.effort,
|
|
696
859
|
});
|
|
697
|
-
await updateView($, router,
|
|
698
|
-
selectedModel: loop.decision.model,
|
|
699
|
-
effort: loop.decision.effort,
|
|
700
|
-
tier: loop.decision.tier,
|
|
701
|
-
reason: loop.decision.reason,
|
|
702
|
-
contextTokens: context.tokens,
|
|
703
|
-
contextKnown: context.known,
|
|
704
|
-
});
|
|
860
|
+
await updateView($, router, continuationView(cfg, loop.decision, context));
|
|
705
861
|
}
|
|
706
862
|
const request = { ...e, model: loop.decision.model };
|
|
707
863
|
if (loop.decision.effort === null) delete request.effort;
|
|
@@ -719,7 +875,7 @@ export function register(on, options) {
|
|
|
719
875
|
const written = await $.state.set(ref, observed, { ifVersion: loopVersion });
|
|
720
876
|
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
721
877
|
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
722
|
-
if (written.isSet) await
|
|
878
|
+
if (written.isSet) await publishReply($, router, e.turnId, response, tier);
|
|
723
879
|
}
|
|
724
880
|
return response;
|
|
725
881
|
});
|
|
@@ -735,6 +891,10 @@ export function register(on, options) {
|
|
|
735
891
|
}
|
|
736
892
|
router.prompts.delete(e.turnId);
|
|
737
893
|
for (const key of router.decisions.keys()) if (key.includes(`:${e.turnId}:`)) router.decisions.delete(key);
|
|
894
|
+
// `off` asks nothing, so it records nothing.
|
|
895
|
+
const turn = router.turnActivities.get(e.turnId);
|
|
896
|
+
router.turnActivities.delete(e.turnId);
|
|
897
|
+
if (turn && turn.mode !== 'off') await recordActivity($, router, turn);
|
|
738
898
|
return next(e);
|
|
739
899
|
});
|
|
740
900
|
|
|
@@ -750,11 +910,15 @@ export function register(on, options) {
|
|
|
750
910
|
return result;
|
|
751
911
|
});
|
|
752
912
|
|
|
753
|
-
on('session.end', (
|
|
913
|
+
on('session.end', async ($, e, next) => {
|
|
754
914
|
for (const controller of router.controllers) controller.abort();
|
|
755
915
|
router.decisions.clear();
|
|
756
916
|
router.prompts.clear();
|
|
757
917
|
router.turnConfigs.clear();
|
|
918
|
+
router.turnActivities.clear();
|
|
919
|
+
const run = router.run;
|
|
920
|
+
router.run = null;
|
|
921
|
+
if (run) await flushRun($, run);
|
|
758
922
|
router.view = {
|
|
759
923
|
...initialView(router.view?.nativeModel ?? ''),
|
|
760
924
|
mode: 'auto',
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducer feeds the
|
|
2
|
+
// counts kept across sessions under `activity:stats:v1`. Pure: state and a turn in, new state out.
|
|
3
|
+
|
|
4
|
+
import { acceptActivity } from './activity.mjs';
|
|
5
|
+
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
6
|
+
|
|
7
|
+
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
8
|
+
export const STORE_KEY = 'activity:stats:v1';
|
|
9
|
+
|
|
10
|
+
// The observed buckets each predicted activity allows. A weak oracle: it tells ops from code from explore, and cannot
|
|
11
|
+
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
12
|
+
export const ALLOWED = {
|
|
13
|
+
code: ['code'],
|
|
14
|
+
debug: ['code', 'ops', 'read'],
|
|
15
|
+
explore: ['read', 'talk'],
|
|
16
|
+
plan: ['talk', 'read', 'docs'],
|
|
17
|
+
review: ['read', 'talk'],
|
|
18
|
+
ops: ['ops'],
|
|
19
|
+
docs: ['docs'],
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
// An answer without an allowed set (uncertain, none) never agrees; callers count agreement only for activities.
|
|
23
|
+
export function agrees(predicted, observed) {
|
|
24
|
+
return Boolean(ALLOWED[predicted]?.includes(observed));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const RUN_BUCKETS = 5;
|
|
28
|
+
const CONFUSION_ROWS = [...ACTIVITY_VALUES, 'none'];
|
|
29
|
+
|
|
30
|
+
const count = (n) => (Number.isFinite(n) && n > 0 ? Math.floor(n) : 0);
|
|
31
|
+
|
|
32
|
+
export function emptySession() {
|
|
33
|
+
return {
|
|
34
|
+
byActivity: {},
|
|
35
|
+
switches: { tier: 0, activity: 0 },
|
|
36
|
+
agreement: { matched: 0, total: 0 },
|
|
37
|
+
shadow: { differs: 0, turns: 0 },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
|
|
42
|
+
// agreement compares with the tools (null: not counted), `shadow` marks a turn in shadow mode and `wouldDiffer` says
|
|
43
|
+
// whether 'on' would have routed it differently (null: no would-route).
|
|
44
|
+
export function recordTurn(
|
|
45
|
+
session,
|
|
46
|
+
{
|
|
47
|
+
activity,
|
|
48
|
+
answer,
|
|
49
|
+
observed,
|
|
50
|
+
requests,
|
|
51
|
+
inputTokens,
|
|
52
|
+
outputTokens,
|
|
53
|
+
tierSwitches,
|
|
54
|
+
activitySwitches,
|
|
55
|
+
wouldDiffer,
|
|
56
|
+
shadow,
|
|
57
|
+
},
|
|
58
|
+
) {
|
|
59
|
+
const key = activity ?? 'none';
|
|
60
|
+
const before = session.byActivity[key] ?? { turns: 0, requests: 0, inputTokens: 0, outputTokens: 0 };
|
|
61
|
+
return {
|
|
62
|
+
byActivity: {
|
|
63
|
+
...session.byActivity,
|
|
64
|
+
[key]: {
|
|
65
|
+
turns: before.turns + 1,
|
|
66
|
+
requests: before.requests + count(requests),
|
|
67
|
+
inputTokens: before.inputTokens + count(inputTokens),
|
|
68
|
+
outputTokens: before.outputTokens + count(outputTokens),
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
switches: {
|
|
72
|
+
tier: session.switches.tier + count(tierSwitches),
|
|
73
|
+
activity: session.switches.activity + count(activitySwitches),
|
|
74
|
+
},
|
|
75
|
+
agreement: answer
|
|
76
|
+
? {
|
|
77
|
+
matched: session.agreement.matched + (agrees(answer, observed) ? 1 : 0),
|
|
78
|
+
total: session.agreement.total + 1,
|
|
79
|
+
}
|
|
80
|
+
: session.agreement,
|
|
81
|
+
shadow: {
|
|
82
|
+
differs: session.shadow.differs + (shadow && wouldDiffer === true ? 1 : 0),
|
|
83
|
+
turns: session.shadow.turns + (shadow ? 1 : 0),
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function emptyStore() {
|
|
89
|
+
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: { differs: 0, turns: 0 } };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
93
|
+
const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
94
|
+
const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
|
|
95
|
+
const countsOf = (v, keys) => onlyKeys(v, keys) && Object.values(v).every(isCount);
|
|
96
|
+
const exactCountsOf = (v, keys) =>
|
|
97
|
+
isRecord(v) && Object.keys(v).length === keys.length && keys.every((k) => isCount(v[k]));
|
|
98
|
+
|
|
99
|
+
// The stored value if it has the exact shape, else an empty store: a hand-edited or older file must not break stats.
|
|
100
|
+
export function readStore(value) {
|
|
101
|
+
const ok =
|
|
102
|
+
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
103
|
+
value.version === 1 &&
|
|
104
|
+
onlyKeys(value.confusion, CONFUSION_ROWS) &&
|
|
105
|
+
Object.values(value.confusion).every((row) => countsOf(row, OBSERVED)) &&
|
|
106
|
+
onlyKeys(value.runs, ACTIVITIES) &&
|
|
107
|
+
Object.values(value.runs).every(
|
|
108
|
+
(lengths) => Array.isArray(lengths) && lengths.length === RUN_BUCKETS && lengths.every(isCount),
|
|
109
|
+
) &&
|
|
110
|
+
exactCountsOf(value.lateral, ['taken', 'refused']) &&
|
|
111
|
+
exactCountsOf(value.shadow, ['differs', 'turns']);
|
|
112
|
+
return ok ? value : emptyStore();
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// One finished turn. `predicted` is the classifier's answer (null: none), `runEnded` a run that closed this turn,
|
|
116
|
+
// `lateral` the outcome of a lateral switch decision and `wouldDiffer` the shadow readout (null: not in shadow).
|
|
117
|
+
// Unknown labels are dropped, so the store keeps a fixed set of keys.
|
|
118
|
+
export function recordStore(store, { predicted, observed, runEnded, lateral, wouldDiffer }) {
|
|
119
|
+
const row = predicted ?? 'none';
|
|
120
|
+
const next = {
|
|
121
|
+
version: 1,
|
|
122
|
+
confusion: { ...store.confusion },
|
|
123
|
+
runs: { ...store.runs },
|
|
124
|
+
lateral: { ...store.lateral },
|
|
125
|
+
shadow: { ...store.shadow },
|
|
126
|
+
};
|
|
127
|
+
if (CONFUSION_ROWS.includes(row) && OBSERVED.includes(observed))
|
|
128
|
+
next.confusion[row] = { ...next.confusion[row], [observed]: (next.confusion[row]?.[observed] ?? 0) + 1 };
|
|
129
|
+
if (runEnded && ACTIVITIES.includes(runEnded.activity) && runEnded.length >= 1) {
|
|
130
|
+
const lengths = [...(next.runs[runEnded.activity] ?? Array(RUN_BUCKETS).fill(0))];
|
|
131
|
+
lengths[Math.min(Math.floor(runEnded.length), RUN_BUCKETS) - 1] += 1;
|
|
132
|
+
next.runs[runEnded.activity] = lengths;
|
|
133
|
+
}
|
|
134
|
+
if (lateral === 'taken' || lateral === 'refused') next.lateral[lateral] += 1;
|
|
135
|
+
if (typeof wouldDiffer === 'boolean') {
|
|
136
|
+
next.shadow.turns += 1;
|
|
137
|
+
if (wouldDiffer) next.shadow.differs += 1;
|
|
138
|
+
}
|
|
139
|
+
return next;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// A run is consecutive turns with the same predicted activity. Given the run so far ({ activity, length } or null)
|
|
143
|
+
// and this turn's predicted activity (null or 'uncertain' end it), returns the run now and the run that just ended.
|
|
144
|
+
export function advanceRun(run, predicted) {
|
|
145
|
+
const activity = ACTIVITIES.includes(predicted) ? predicted : null;
|
|
146
|
+
if (run && run.activity === activity) return { run: { activity, length: run.length + 1 }, ended: null };
|
|
147
|
+
return { run: activity ? { activity, length: 1 } : null, ended: run };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const LATERAL = { 'activity-up': 'taken', 'activity-down': 'taken', 'activity-pending': 'refused' };
|
|
151
|
+
const differs = (a, b) => a.model !== b.model || (a.effort ?? null) !== (b.effort ?? null);
|
|
152
|
+
|
|
153
|
+
// What one routed turn adds to the stats, from its config, the classifier's advice, and the decisions before and
|
|
154
|
+
// after it. `label` is the turn's activity: the applied one in `on`, the accepted answer in `shadow`; `answer` is the
|
|
155
|
+
// accepted answer in both, which the tools are checked against and runs follow. `switched` names
|
|
156
|
+
// what moved the route from the previous turn's: 'activity' for a lateral move, 'tier' for any other, null for none
|
|
157
|
+
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`; a lateral move refused
|
|
158
|
+
// by a hold or the cash gate shares its reason with tier decisions and is not counted.
|
|
159
|
+
export function turnActivity(config, advice, previous, decision) {
|
|
160
|
+
const mode = config.activityRouting;
|
|
161
|
+
const would = decision.wouldRoute ?? null;
|
|
162
|
+
const switched =
|
|
163
|
+
previous?.model && !decision.pinned && differs(previous, decision)
|
|
164
|
+
? LATERAL[decision.reason] === 'taken'
|
|
165
|
+
? 'activity'
|
|
166
|
+
: 'tier'
|
|
167
|
+
: null;
|
|
168
|
+
const answer = mode === 'off' ? null : acceptActivity(config, advice);
|
|
169
|
+
return {
|
|
170
|
+
mode,
|
|
171
|
+
answer,
|
|
172
|
+
label: mode === 'on' ? (decision.activity ?? null) : mode === 'shadow' ? answer : null,
|
|
173
|
+
predicted: advice?.activity?.choice ?? null,
|
|
174
|
+
switched,
|
|
175
|
+
lateral: LATERAL[(mode === 'shadow' ? would : mode === 'on' ? decision : null)?.reason] ?? null,
|
|
176
|
+
// A shadow turn always counts, so the session and store denominators agree.
|
|
177
|
+
wouldDiffer: mode === 'shadow' ? Boolean(would && differs(would, decision)) : null,
|
|
178
|
+
};
|
|
179
|
+
}
|
package/lib/activity.mjs
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// Activity routing (docs/plans/2026-10-09-activity-routing-plan.md §5): which activity a turn applies, how two routes
|
|
2
|
+
// compare, and when a route change inside one tier is worth it. Pure.
|
|
3
|
+
import { ACTIVITIES, EFFORTS, resolveRoute, routeSpec } from './config.mjs';
|
|
4
|
+
import { routeEffort } from './cost.mjs';
|
|
5
|
+
import { cashGate } from './policy.mjs';
|
|
6
|
+
|
|
7
|
+
// The classifier's activity when it names one with at least `policy.activityMass`, else null: uncertain, a weak
|
|
8
|
+
// label, or no answer resolve to the tier's base route.
|
|
9
|
+
export function acceptActivity(config, advice) {
|
|
10
|
+
const answer = advice?.activity;
|
|
11
|
+
if (!answer || !ACTIVITIES.includes(answer.choice)) return null;
|
|
12
|
+
return answer.probabilities?.[answer.choice] >= config.policy.activityMass ? answer.choice : null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// 1 when route `a` is stronger than `b`, -1 when cheaper, 0 when equal in strength, null when they cannot be ordered.
|
|
16
|
+
// Models order by configured output price, then effort. A null effort is `sentEffort`, the one Claude Code sends, as
|
|
17
|
+
// the request would carry it; without a named effort on both sides (none sent, or a numeric budget) two different
|
|
18
|
+
// efforts do not order. A missing output price orders only against the same model.
|
|
19
|
+
export function compareRoutes(config, a, b, sentEffort) {
|
|
20
|
+
if (a.model !== b.model) {
|
|
21
|
+
const [pa, pb] = [routeSpec(config, a).output, routeSpec(config, b).output];
|
|
22
|
+
if (!Number.isFinite(pa) || !Number.isFinite(pb)) return null;
|
|
23
|
+
if (pa !== pb) return Math.sign(pa - pb);
|
|
24
|
+
}
|
|
25
|
+
const [ea, eb] = [routeEffort(config, a, sentEffort), routeEffort(config, b, sentEffort)];
|
|
26
|
+
if (ea === eb) return 0;
|
|
27
|
+
if (!EFFORTS.includes(ea) || !EFFORTS.includes(eb)) return null;
|
|
28
|
+
return Math.sign(EFFORTS.indexOf(ea) - EFFORTS.indexOf(eb));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Whether two routes send the same request: the same model at the same effort once a session effort is resolved.
|
|
32
|
+
export const sameRequest = (config, a, b, sentEffort) =>
|
|
33
|
+
a.model === b.model && routeEffort(config, a, sentEffort) === routeEffort(config, b, sentEffort);
|
|
34
|
+
|
|
35
|
+
// The activity a turn runs with, given the incumbent cell `{ tier, activity }` and its resolved `route`. Without
|
|
36
|
+
// advice, or without an activity answer (a skipped, failed or malformed activity step), the incumbent's activity
|
|
37
|
+
// stays: a flaky step must not move the route every turn. A tool continuation keeps one route per task, but may move
|
|
38
|
+
// up (explore, then code): a weaker label never takes over mid-task.
|
|
39
|
+
export function chooseActivity(config, advice, incumbent, sentEffort) {
|
|
40
|
+
if (!advice?.activity) return incumbent.activity;
|
|
41
|
+
if (advice.continuation >= config.policy.continuationMass) {
|
|
42
|
+
const label = advice.activity.choice;
|
|
43
|
+
const stronger =
|
|
44
|
+
ACTIVITIES.includes(label) &&
|
|
45
|
+
compareRoutes(config, resolveRoute(config, incumbent.tier, label), incumbent.route, sentEffort) > 0;
|
|
46
|
+
return stronger ? label : incumbent.activity;
|
|
47
|
+
}
|
|
48
|
+
return acceptActivity(config, advice);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// A route change inside the incumbent's tier, `from` → `to`. Up on evidence: `mass`, the label's probability (1 for
|
|
52
|
+
// the base route), must clear the tier upgrade bar for this switching tax. Down on economics: the move must pay back
|
|
53
|
+
// its cache write within the downgrade horizon. A pair that does not order (an unpriced model, a numeric effort, two
|
|
54
|
+
// models at one price) counts as up: it needs evidence, and the base route is always reachable. Labels alone never
|
|
55
|
+
// switch, so one vote is enough. During an escalation hold only a stronger route may be taken. Returns
|
|
56
|
+
// { move, reason, estimate }.
|
|
57
|
+
export function lateralMove({ config, from, to, mass, hold, facts, now, costs }) {
|
|
58
|
+
const order = compareRoutes(config, to, from, facts.effort);
|
|
59
|
+
if (hold && !(order > 0)) return { move: false, reason: 'hold', estimate: null };
|
|
60
|
+
const p = config.policy;
|
|
61
|
+
let estimate;
|
|
62
|
+
if (order === -1) {
|
|
63
|
+
estimate = { taxUsd: costs.downgradeTaxUsd(config, to, from, facts, now, p.downgradeHorizonTurns) };
|
|
64
|
+
// Before the first measured reply no cache of this history exists, though the tax prices an unknown one as warm:
|
|
65
|
+
// staying protects nothing.
|
|
66
|
+
const fresh = facts.historyMeasured === false;
|
|
67
|
+
if (estimate.taxUsd > 0 && !fresh) return { move: false, reason: 'activity-pending', estimate };
|
|
68
|
+
} else {
|
|
69
|
+
const tax = Math.max(0, costs.switchingTaxUsd(config, to, from, facts, now));
|
|
70
|
+
const threshold = p.upgradeBase + p.upgradeSlope * (tax / (tax + p.upgradePivotUsd));
|
|
71
|
+
estimate = { taxUsd: tax, threshold, upgradeMass: mass };
|
|
72
|
+
if (mass < threshold) return { move: false, reason: 'activity-pending', estimate };
|
|
73
|
+
}
|
|
74
|
+
const gated = cashGate(config, to, facts, now, costs);
|
|
75
|
+
if (gated.blocked) return { move: false, reason: 'cash-gate', estimate: gated.estimate };
|
|
76
|
+
return { move: true, reason: order === -1 ? 'activity-down' : 'activity-up', estimate };
|
|
77
|
+
}
|