@alexeiled/claude-router 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +2 -2
- package/CHANGELOG.md +41 -0
- package/README.md +12 -10
- package/docs/architecture.md +59 -17
- package/docs/configuration.md +86 -32
- package/docs/evaluation.md +56 -1
- package/docs/native-router.md +38 -28
- package/docs/router-band.svg +83 -42
- package/docs/router-pane-now.svg +97 -71
- package/docs/router-pane-routing.svg +103 -32
- package/docs/user-guide.md +75 -27
- package/hooks/native-router.mjs +215 -33
- package/lib/activity-stats.mjs +245 -0
- package/lib/activity.mjs +77 -0
- package/lib/band.mjs +32 -11
- package/lib/classifier-apis.mjs +133 -69
- package/lib/classifier-client.mjs +63 -28
- package/lib/classifier-contract.mjs +48 -0
- package/lib/config.mjs +138 -4
- package/lib/cost.mjs +23 -11
- package/lib/display.mjs +100 -4
- package/lib/facts.mjs +48 -0
- package/lib/panel.mjs +323 -29
- package/lib/policy.mjs +45 -26
- package/lib/route.mjs +169 -55
- package/lib/view.mjs +35 -4
- package/package.json +1 -1
- package/types/index.d.ts +88 -1
package/hooks/native-router.mjs
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
|
+
import {
|
|
2
|
+
advanceRun,
|
|
3
|
+
emptySession,
|
|
4
|
+
emptyStore,
|
|
5
|
+
readStore,
|
|
6
|
+
recordStore,
|
|
7
|
+
recordTurn,
|
|
8
|
+
STORE_KEY,
|
|
9
|
+
turnActivity,
|
|
10
|
+
} from '../lib/activity-stats.mjs';
|
|
1
11
|
import { renderBand, switchToast } from '../lib/band.mjs';
|
|
2
12
|
import { ClassifierClient } from '../lib/classifier-client.mjs';
|
|
3
13
|
import { resolveCredentials } from '../lib/classifier-contract.mjs';
|
|
4
14
|
import {
|
|
15
|
+
ACTIVITY_MODES,
|
|
5
16
|
activeClassifier,
|
|
6
17
|
DEFAULTS,
|
|
18
|
+
editActivity,
|
|
19
|
+
effectiveActivities,
|
|
7
20
|
effectiveRoutes,
|
|
8
21
|
loadConfig,
|
|
9
22
|
MIGRATION_HINT,
|
|
10
23
|
supportedVersion,
|
|
11
24
|
TIERS,
|
|
12
25
|
tuningOf,
|
|
26
|
+
withActivities,
|
|
13
27
|
withClassifier,
|
|
14
28
|
withClassifierTimeout,
|
|
15
29
|
withRoutes,
|
|
@@ -17,16 +31,19 @@ import {
|
|
|
17
31
|
} from '../lib/config.mjs';
|
|
18
32
|
import { changedLeaves, notSaved, restored, rewrittenConfig } from '../lib/config-file.mjs';
|
|
19
33
|
import {
|
|
34
|
+
activityDetailLines,
|
|
20
35
|
classifierStatus,
|
|
21
36
|
classifierTimed,
|
|
22
37
|
GATEWAY_CLEANUP,
|
|
23
38
|
GATEWAY_SETTINGS,
|
|
24
39
|
missingCredentials,
|
|
25
40
|
NOT_A_TIER_REASON,
|
|
41
|
+
storeAgreementLine,
|
|
26
42
|
} from '../lib/display.mjs';
|
|
27
|
-
import { clip } from '../lib/facts.mjs';
|
|
43
|
+
import { clip, observedActivity, promptIndex } from '../lib/facts.mjs';
|
|
28
44
|
import { renderPanel, routeDraftOf, routingChanges } from '../lib/panel.mjs';
|
|
29
45
|
import {
|
|
46
|
+
cellForModel,
|
|
30
47
|
chooseRoute,
|
|
31
48
|
continueRoute,
|
|
32
49
|
emptyLoop,
|
|
@@ -37,9 +54,8 @@ import {
|
|
|
37
54
|
observeResponse,
|
|
38
55
|
prepareLoop,
|
|
39
56
|
resetHistory,
|
|
40
|
-
tierForModel,
|
|
41
57
|
} from '../lib/route.mjs';
|
|
42
|
-
import { CLEARED_READINGS, healthOf, initialView, responseMetrics } from '../lib/view.mjs';
|
|
58
|
+
import { CLEARED_READINGS, continuationView, healthOf, initialView, responseMetrics } from '../lib/view.mjs';
|
|
43
59
|
|
|
44
60
|
// The engine follows $ only into functions declared in this file, never across an import: every helper that takes $
|
|
45
61
|
// lives here, and the pure parts live in lib/.
|
|
@@ -68,6 +84,12 @@ function createRouter(options) {
|
|
|
68
84
|
turnControllers: new Map(),
|
|
69
85
|
view: null,
|
|
70
86
|
modes: new Map(),
|
|
87
|
+
// Each routed turn's activity record until turn.complete adds it to the stats, and the session's open run of
|
|
88
|
+
// one activity, flushed to the store when it ends.
|
|
89
|
+
turnActivities: new Map(),
|
|
90
|
+
run: null,
|
|
91
|
+
// Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
|
|
92
|
+
statsResets: 0,
|
|
71
93
|
};
|
|
72
94
|
}
|
|
73
95
|
|
|
@@ -98,7 +120,7 @@ async function modeOf($, router, fallback = 'auto') {
|
|
|
98
120
|
|
|
99
121
|
// A saved preference wins; a fresh session starts with routing on for a model some tier routes to, and off for any other.
|
|
100
122
|
async function startMode($, router, model) {
|
|
101
|
-
const mode = await modeOf($, router,
|
|
123
|
+
const mode = await modeOf($, router, cellForModel(router.config, model) === null ? 'manual' : 'auto');
|
|
102
124
|
router.modes.set(await $.session.id(), mode);
|
|
103
125
|
return mode;
|
|
104
126
|
}
|
|
@@ -125,10 +147,20 @@ async function updateView($, router, patch) {
|
|
|
125
147
|
await $.state.set(VIEW, router.view);
|
|
126
148
|
}
|
|
127
149
|
|
|
150
|
+
// The counts kept across sessions for the view, read when the session starts and the pane opens: another session
|
|
151
|
+
// may have added to them. A failed read keeps what the view has; it never breaks a turn or a render.
|
|
152
|
+
async function storeView($) {
|
|
153
|
+
try {
|
|
154
|
+
return { activityStore: readStore(await $.store.get(STORE_KEY)) };
|
|
155
|
+
} catch {
|
|
156
|
+
return {};
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
128
160
|
// A notice answers the last press in the pane, so the pane opens without one; the last write keeps its Undo in the
|
|
129
161
|
// status bar.
|
|
130
162
|
async function openPane($, router) {
|
|
131
|
-
await updateView($, router, { notice: null });
|
|
163
|
+
await updateView($, router, { notice: null, ...(await storeView($)) });
|
|
132
164
|
await $.ui.open({ id: PANE, title: PANE_TITLE, focus: true, closeOnEscape: true });
|
|
133
165
|
}
|
|
134
166
|
|
|
@@ -155,6 +187,37 @@ async function setPin($, router, tier) {
|
|
|
155
187
|
return `${tier} pinned for the next turn and its tool continuations.`;
|
|
156
188
|
}
|
|
157
189
|
|
|
190
|
+
// `/router [auto|off|pin <tier>|activities <mode>]`; with no action, the pane, or the status text without a surface.
|
|
191
|
+
async function routerCommand($, router, args) {
|
|
192
|
+
const [action, value] = args.trim().split(/\s+/);
|
|
193
|
+
if (action === 'auto' || action === 'off') {
|
|
194
|
+
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
195
|
+
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
196
|
+
}
|
|
197
|
+
if (action === 'pin')
|
|
198
|
+
return { text: TIERS.includes(value) ? await setPin($, router, value) : `Choose ${TIERS.join(', ')}.` };
|
|
199
|
+
if (action === 'activities')
|
|
200
|
+
return {
|
|
201
|
+
text: ACTIVITY_MODES.includes(value)
|
|
202
|
+
? await setActivityRouting($, router, value)
|
|
203
|
+
: `Choose activities ${ACTIVITY_MODES.join(', ')}.`,
|
|
204
|
+
};
|
|
205
|
+
const storedView = await readView($, router);
|
|
206
|
+
const view = { ...storedView, mode: await modeOf($, router, storedView.mode) };
|
|
207
|
+
if (!(await $.session.surfaces()).length) return { text: detailText(router.config, view) };
|
|
208
|
+
await openPane($, router);
|
|
209
|
+
return {};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// `/router activities <mode>`: the pane's write, so the file keeps the rest and Undo in the pane restores it.
|
|
213
|
+
async function setActivityRouting($, router, mode) {
|
|
214
|
+
const view = router.view ?? (await readView($, router));
|
|
215
|
+
if (view.phase === 'unavailable') return `Routing unavailable: ${view.error}.`;
|
|
216
|
+
if (mode === router.config.activityRouting) return `Activity routing is already ${mode}.`;
|
|
217
|
+
await paneActions($, router, view).activityRouting(mode);
|
|
218
|
+
return router.view.notice;
|
|
219
|
+
}
|
|
220
|
+
|
|
158
221
|
// The environment variables that stand in for a classifier option: its upper-case name, then the one Claude Code
|
|
159
222
|
// exports for a plugin option. Spelled out because $.env.get takes literal names only. Cases: CLASSIFIER_OPTIONS.
|
|
160
223
|
async function envSettingOf($, name) {
|
|
@@ -236,6 +299,8 @@ function detailText(config, view) {
|
|
|
236
299
|
`Selected: ${view.selectedModel ?? 'not selected'}`,
|
|
237
300
|
`Observed: ${view.actualModel ?? 'no response yet'}`,
|
|
238
301
|
`Reason: ${view.reason}`,
|
|
302
|
+
...activityDetailLines(config, view),
|
|
303
|
+
storeAgreementLine(view),
|
|
239
304
|
view.error || missing ? classifierStatus(config, view.error ?? missing) : `${activeClassifier(config).label} ready`,
|
|
240
305
|
`Context: ${view.contextKnown ? `${view.contextTokens} tokens (estimate)` : 'unknown'}`,
|
|
241
306
|
`Observed cache: ${view.cacheRead ?? 'unknown'} read, ${view.cacheWrite ?? 'unknown'} written tokens`,
|
|
@@ -348,6 +413,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
348
413
|
if (!written.isSet || controller.signal.aborted) return null;
|
|
349
414
|
if (previous?.model && previous.model !== selected.decision.model && !selected.decision.pinned)
|
|
350
415
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
416
|
+
const activity = turnActivity(cfg, result.advice, previous, selected.decision);
|
|
417
|
+
router.turnActivities.set(e.turnId, { ...activity, sessionId, requests: 0, inputTokens: 0, outputTokens: 0 });
|
|
351
418
|
await updateView($, router, {
|
|
352
419
|
phase: 'routed',
|
|
353
420
|
selectedModel: selected.decision.model,
|
|
@@ -355,6 +422,7 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
355
422
|
tier: selected.decision.tier,
|
|
356
423
|
effort: selected.decision.effort,
|
|
357
424
|
reason: selected.decision.reason,
|
|
425
|
+
activity: activity.label,
|
|
358
426
|
contextTokens: context.tokens,
|
|
359
427
|
contextKnown: context.known,
|
|
360
428
|
comparison: selected.decision.comparison,
|
|
@@ -368,6 +436,9 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
368
436
|
adviceChoice: result.advice?.choice ?? null,
|
|
369
437
|
probabilities: result.advice?.probabilities ?? null,
|
|
370
438
|
estimate: selected.decision.estimate ?? null,
|
|
439
|
+
activityChoice: activity.predicted,
|
|
440
|
+
activityProbabilities: result.advice?.activity?.probabilities ?? null,
|
|
441
|
+
wouldRoute: selected.decision.wouldRoute ?? null,
|
|
371
442
|
}
|
|
372
443
|
: {}),
|
|
373
444
|
});
|
|
@@ -379,6 +450,78 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
379
450
|
}
|
|
380
451
|
}
|
|
381
452
|
|
|
453
|
+
// A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
|
|
454
|
+
// turn's activity record.
|
|
455
|
+
async function publishReply($, router, turnId, response, tier) {
|
|
456
|
+
const turn = router.turnActivities.get(turnId);
|
|
457
|
+
const metrics = responseMetrics(router.view, response, tier, tier ? (turn?.label ?? null) : null);
|
|
458
|
+
await updateView($, router, metrics);
|
|
459
|
+
if (turn)
|
|
460
|
+
router.turnActivities.set(turnId, {
|
|
461
|
+
...turn,
|
|
462
|
+
requests: turn.requests + 1,
|
|
463
|
+
inputTokens: turn.inputTokens + (metrics.inputTokens ?? 0),
|
|
464
|
+
outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
469
|
+
// break a turn: a failed read or write loses this turn's counts only.
|
|
470
|
+
async function recordActivity($, router, turn) {
|
|
471
|
+
try {
|
|
472
|
+
if ((await $.session.id()) !== turn.sessionId) return;
|
|
473
|
+
const messages = await $.session.messages({ as: 'api' });
|
|
474
|
+
const start = promptIndex(messages);
|
|
475
|
+
if (start < 0) return;
|
|
476
|
+
const observed = observedActivity(messages, start);
|
|
477
|
+
const view = router.view ?? (await readView($, router));
|
|
478
|
+
// The session may have changed during the reads; this turn is not the new session's.
|
|
479
|
+
if ((await $.session.id()) !== turn.sessionId) return;
|
|
480
|
+
// Advanced before any further await, so a session change meanwhile cannot carry this run into the new session.
|
|
481
|
+
const { run, ended } = advanceRun(router.run, turn.answer);
|
|
482
|
+
router.run = run;
|
|
483
|
+
await updateView($, router, {
|
|
484
|
+
activityStats: recordTurn(view.activityStats ?? emptySession(), {
|
|
485
|
+
activity: turn.label,
|
|
486
|
+
answer: turn.answer,
|
|
487
|
+
observed,
|
|
488
|
+
requests: turn.requests,
|
|
489
|
+
inputTokens: turn.inputTokens,
|
|
490
|
+
outputTokens: turn.outputTokens,
|
|
491
|
+
tierSwitches: turn.switched === 'tier' ? 1 : 0,
|
|
492
|
+
activitySwitches: turn.switched === 'activity' ? 1 : 0,
|
|
493
|
+
wouldDiffer: turn.wouldDiffer,
|
|
494
|
+
wouldUsd: turn.wouldUsd,
|
|
495
|
+
shadow: turn.mode === 'shadow',
|
|
496
|
+
}),
|
|
497
|
+
});
|
|
498
|
+
const resets = router.statsResets;
|
|
499
|
+
const store = readStore(await $.store.get(STORE_KEY));
|
|
500
|
+
// Writing counts read before a reset would bring them back; this turn's are lost instead.
|
|
501
|
+
if (router.statsResets !== resets) return;
|
|
502
|
+
const written = recordStore(store, {
|
|
503
|
+
predicted: turn.predicted,
|
|
504
|
+
observed,
|
|
505
|
+
runEnded: ended,
|
|
506
|
+
lateral: turn.lateral,
|
|
507
|
+
wouldDiffer: turn.wouldDiffer,
|
|
508
|
+
wouldUsd: turn.wouldUsd,
|
|
509
|
+
});
|
|
510
|
+
await $.store.set(STORE_KEY, written);
|
|
511
|
+
// A reset during the write emptied the view; these counts are older than it.
|
|
512
|
+
if (router.statsResets === resets) await updateView($, router, { activityStore: written });
|
|
513
|
+
} catch {}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// The session's open run ends with it; without this flush it is never counted.
|
|
517
|
+
async function flushRun($, run) {
|
|
518
|
+
try {
|
|
519
|
+
const store = readStore(await $.store.get(STORE_KEY));
|
|
520
|
+
const ended = { predicted: null, observed: null, runEnded: run, lateral: null, wouldDiffer: null, wouldUsd: null };
|
|
521
|
+
await $.store.set(STORE_KEY, recordStore(store, ended));
|
|
522
|
+
} catch {}
|
|
523
|
+
}
|
|
524
|
+
|
|
382
525
|
// The pane's handlers. `view` is the view the pane was drawn from; a handler reads `router.view` first, which holds
|
|
383
526
|
// any press since.
|
|
384
527
|
function paneActions($, router, view) {
|
|
@@ -390,6 +533,18 @@ function paneActions($, router, view) {
|
|
|
390
533
|
notice: null,
|
|
391
534
|
});
|
|
392
535
|
};
|
|
536
|
+
const editCell = (activity, tier, kind, value) =>
|
|
537
|
+
updateView($, router, {
|
|
538
|
+
routeDraft: editActivity(
|
|
539
|
+
routeDraftOf(router.config, router.view ?? view),
|
|
540
|
+
router.config,
|
|
541
|
+
activity,
|
|
542
|
+
tier,
|
|
543
|
+
kind,
|
|
544
|
+
value,
|
|
545
|
+
),
|
|
546
|
+
notice: null,
|
|
547
|
+
});
|
|
393
548
|
// Every pane save adopts the file as written, which may name another classifier than before: a hand edit
|
|
394
549
|
// meanwhile, or a row press. Then the new classifier starts clean and the old one's readings leave the view;
|
|
395
550
|
// an unavailable reason stays.
|
|
@@ -410,7 +565,10 @@ function paneActions($, router, view) {
|
|
|
410
565
|
// follows the file, and the file becomes what the draft compares against, so picking a value the write replaced
|
|
411
566
|
// counts as a change again.
|
|
412
567
|
const rebased = ({ routeDraft, tuning, tuningBase }) => {
|
|
413
|
-
const routes = routeDraft &&
|
|
568
|
+
const routes = routeDraft && {
|
|
569
|
+
...effectiveRoutes(routeDraft, router.config),
|
|
570
|
+
...effectiveActivities(routeDraft, router.config),
|
|
571
|
+
};
|
|
414
572
|
const edited = Object.entries(tuning ?? {}).filter(([key, value]) => value !== tuningBase?.[key]);
|
|
415
573
|
return {
|
|
416
574
|
routeDraft: routes ? { ...routes, base: routeDraftOf(router.config, {}).base } : null,
|
|
@@ -489,17 +647,50 @@ function paneActions($, router, view) {
|
|
|
489
647
|
return write(
|
|
490
648
|
`${changes} routing change${changes === 1 ? '' : 's'}`,
|
|
491
649
|
(file) => {
|
|
492
|
-
|
|
650
|
+
// withActivities reads the routes withRoutes wrote: a removed built-in override is saved as its tier's route.
|
|
651
|
+
const routed = routeDraft ? withActivities(withRoutes(file, routeDraft), routeDraft) : file;
|
|
493
652
|
return edited.length ? withTuning(routed, { ...base, ...after }, base) : routed;
|
|
494
653
|
},
|
|
495
654
|
{ routeDraft: null, tuning: null, tuningBase: null },
|
|
496
655
|
);
|
|
497
656
|
},
|
|
657
|
+
activityMode: (mode) =>
|
|
658
|
+
updateView($, router, {
|
|
659
|
+
routeDraft: { ...routeDraftOf(router.config, router.view ?? view), activityRouting: mode },
|
|
660
|
+
notice: null,
|
|
661
|
+
}),
|
|
662
|
+
activityModel: (activity, tier, alias) => editCell(activity, tier, 'model', alias),
|
|
663
|
+
activityEffort: (activity, tier, effort) => editCell(activity, tier, 'effort', effort),
|
|
664
|
+
addActivity: (activity, tier) => editCell(activity, tier, 'add'),
|
|
665
|
+
removeActivity: (activity, tier) => editCell(activity, tier, 'remove'),
|
|
666
|
+
// `/router activities <mode>`: the mode alone, saved at once like a classifier press, with Undo.
|
|
667
|
+
activityRouting: (mode) => {
|
|
668
|
+
const from = router.config.activityRouting;
|
|
669
|
+
if (mode === from) return undefined;
|
|
670
|
+
return write(`activity routing ${from} → ${mode}`, (file) =>
|
|
671
|
+
withActivities(file, { base: { activityRouting: from }, activityRouting: mode }),
|
|
672
|
+
);
|
|
673
|
+
},
|
|
674
|
+
resetActivityStats: async () => {
|
|
675
|
+
router.statsResets += 1;
|
|
676
|
+
router.run = null;
|
|
677
|
+
const cleared = await $.store
|
|
678
|
+
.set(STORE_KEY, emptyStore())
|
|
679
|
+
.then(() => true)
|
|
680
|
+
.catch(() => false);
|
|
681
|
+
await updateView($, router, {
|
|
682
|
+
activityStats: null,
|
|
683
|
+
...(cleared ? { activityStore: emptyStore() } : {}),
|
|
684
|
+
notice: cleared ? 'Activity stats reset.' : 'Session activity stats reset. Saved stats could not be cleared.',
|
|
685
|
+
});
|
|
686
|
+
},
|
|
498
687
|
discardRouting: () => updateView($, router, { routeDraft: null, tuning: null, tuningBase: null, notice: null }),
|
|
688
|
+
// Routes only: pending activity edits and the mode stay in the draft.
|
|
499
689
|
resetRoutes: () =>
|
|
500
690
|
loadDefaults(
|
|
501
691
|
{
|
|
502
692
|
routeDraft: {
|
|
693
|
+
...routeDraftOf(router.config, router.view ?? view),
|
|
503
694
|
...structuredClone({ routes: DEFAULTS.routes, baselineTier: DEFAULTS.baselineTier }),
|
|
504
695
|
base: routeDraftOf(router.config, {}).base,
|
|
505
696
|
},
|
|
@@ -550,8 +741,8 @@ export function register(on, options) {
|
|
|
550
741
|
on('session.start', async ($, e, next) => {
|
|
551
742
|
await $.command.register({
|
|
552
743
|
name: 'router',
|
|
553
|
-
description: 'Open the Router pane, or switch routing: auto, off, pin <tier>.',
|
|
554
|
-
argumentHint: '[auto|off|pin <tier
|
|
744
|
+
description: 'Open the Router pane, or switch routing: auto, off, pin <tier>, activities <mode>.',
|
|
745
|
+
argumentHint: '[auto|off|pin <tier>|activities off|shadow|on]',
|
|
555
746
|
immediate: true,
|
|
556
747
|
});
|
|
557
748
|
const model = await $.session.model();
|
|
@@ -582,6 +773,7 @@ export function register(on, options) {
|
|
|
582
773
|
routeDraft: null,
|
|
583
774
|
lastWrite: null,
|
|
584
775
|
bandDetail: (await $.store.get(BAND_DETAIL).catch(() => false)) === true,
|
|
776
|
+
...(await storeView($)),
|
|
585
777
|
});
|
|
586
778
|
} catch (error) {
|
|
587
779
|
await updateView($, router, {
|
|
@@ -612,20 +804,7 @@ export function register(on, options) {
|
|
|
612
804
|
return result;
|
|
613
805
|
});
|
|
614
806
|
|
|
615
|
-
on('command.run', { command: 'router' },
|
|
616
|
-
const [action, tier] = e.args.trim().split(/\s+/);
|
|
617
|
-
if (action === 'auto' || action === 'off') {
|
|
618
|
-
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
619
|
-
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
620
|
-
}
|
|
621
|
-
if (action === 'pin')
|
|
622
|
-
return { text: TIERS.includes(tier) ? await setPin($, router, tier) : `Choose ${TIERS.join(', ')}.` };
|
|
623
|
-
const storedView = await readView($, router);
|
|
624
|
-
const view = { ...storedView, mode: await modeOf($, router, storedView.mode) };
|
|
625
|
-
if (!(await $.session.surfaces()).length) return { text: detailText(router.config, view) };
|
|
626
|
-
await openPane($, router);
|
|
627
|
-
return {};
|
|
628
|
-
});
|
|
807
|
+
on('command.run', { command: 'router' }, ($, e) => routerCommand($, router, e.args));
|
|
629
808
|
|
|
630
809
|
on('turn.start', (_$, e, next) => {
|
|
631
810
|
router.turnConfigs.set(e.turnId, router.config);
|
|
@@ -694,14 +873,7 @@ export function register(on, options) {
|
|
|
694
873
|
availableModels,
|
|
695
874
|
effort: e.effort,
|
|
696
875
|
});
|
|
697
|
-
await updateView($, router,
|
|
698
|
-
selectedModel: loop.decision.model,
|
|
699
|
-
effort: loop.decision.effort,
|
|
700
|
-
tier: loop.decision.tier,
|
|
701
|
-
reason: loop.decision.reason,
|
|
702
|
-
contextTokens: context.tokens,
|
|
703
|
-
contextKnown: context.known,
|
|
704
|
-
});
|
|
876
|
+
await updateView($, router, continuationView(cfg, loop.decision, context));
|
|
705
877
|
}
|
|
706
878
|
const request = { ...e, model: loop.decision.model };
|
|
707
879
|
if (loop.decision.effort === null) delete request.effort;
|
|
@@ -719,7 +891,7 @@ export function register(on, options) {
|
|
|
719
891
|
const written = await $.state.set(ref, observed, { ifVersion: loopVersion });
|
|
720
892
|
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
721
893
|
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
722
|
-
if (written.isSet) await
|
|
894
|
+
if (written.isSet) await publishReply($, router, e.turnId, response, tier);
|
|
723
895
|
}
|
|
724
896
|
return response;
|
|
725
897
|
});
|
|
@@ -735,6 +907,10 @@ export function register(on, options) {
|
|
|
735
907
|
}
|
|
736
908
|
router.prompts.delete(e.turnId);
|
|
737
909
|
for (const key of router.decisions.keys()) if (key.includes(`:${e.turnId}:`)) router.decisions.delete(key);
|
|
910
|
+
// `off` asks nothing, so it records nothing.
|
|
911
|
+
const turn = router.turnActivities.get(e.turnId);
|
|
912
|
+
router.turnActivities.delete(e.turnId);
|
|
913
|
+
if (turn && turn.mode !== 'off') await recordActivity($, router, turn);
|
|
738
914
|
return next(e);
|
|
739
915
|
});
|
|
740
916
|
|
|
@@ -750,15 +926,21 @@ export function register(on, options) {
|
|
|
750
926
|
return result;
|
|
751
927
|
});
|
|
752
928
|
|
|
753
|
-
on('session.end', (
|
|
929
|
+
on('session.end', async ($, e, next) => {
|
|
754
930
|
for (const controller of router.controllers) controller.abort();
|
|
755
931
|
router.decisions.clear();
|
|
756
932
|
router.prompts.clear();
|
|
757
933
|
router.turnConfigs.clear();
|
|
934
|
+
router.turnActivities.clear();
|
|
935
|
+
const run = router.run;
|
|
936
|
+
router.run = null;
|
|
937
|
+
if (run) await flushRun($, run);
|
|
758
938
|
router.view = {
|
|
759
939
|
...initialView(router.view?.nativeModel ?? ''),
|
|
760
940
|
mode: 'auto',
|
|
761
941
|
credentials: router.view?.credentials ?? null,
|
|
942
|
+
// Counts kept across sessions: the next session starts from them.
|
|
943
|
+
activityStore: router.view?.activityStore ?? null,
|
|
762
944
|
health: healthOf(clientOf(router, router.config.classifier), router.config.classifier),
|
|
763
945
|
configPath: router.config.nativePath,
|
|
764
946
|
phase: router.view?.phase === 'unavailable' ? 'unavailable' : 'ready',
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducer feeds the
|
|
2
|
+
// counts kept across sessions under `activity:stats:v1`. Pure: state and a turn in, new state out.
|
|
3
|
+
|
|
4
|
+
import { acceptActivity } from './activity.mjs';
|
|
5
|
+
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
6
|
+
|
|
7
|
+
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
8
|
+
export const STORE_KEY = 'activity:stats:v1';
|
|
9
|
+
|
|
10
|
+
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
11
|
+
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
12
|
+
export const ALLOWED = {
|
|
13
|
+
code: ['code'],
|
|
14
|
+
debug: ['code', 'ops', 'read'],
|
|
15
|
+
explore: ['read', 'talk'],
|
|
16
|
+
plan: ['talk', 'read', 'docs'],
|
|
17
|
+
review: ['read', 'talk'],
|
|
18
|
+
// git status or a test listing is still running commands: a read-only Bash turn is ops too.
|
|
19
|
+
ops: ['ops', 'read'],
|
|
20
|
+
docs: ['docs'],
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
// An answer without an allowed set (uncertain, none) never agrees; callers count agreement only for activities.
|
|
24
|
+
export function agrees(predicted, observed) {
|
|
25
|
+
return Boolean(ALLOWED[predicted]?.includes(observed));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const RUN_BUCKETS = 5;
|
|
29
|
+
const CONFUSION_ROWS = [...ACTIVITY_VALUES, 'none'];
|
|
30
|
+
|
|
31
|
+
const count = (n) => (Number.isFinite(n) && n > 0 ? Math.floor(n) : 0);
|
|
32
|
+
|
|
33
|
+
// The shadow readout: turns in shadow, those `on` would have routed differently, and of those the ones with a
|
|
34
|
+
// next-request estimate (`estimated`) and the sum of their [min, max] ranges in USD at configured list prices.
|
|
35
|
+
const emptyShadow = () => ({ differs: 0, turns: 0, estimated: 0, minUsd: 0, maxUsd: 0 });
|
|
36
|
+
const isRange = (r) => Number.isFinite(r?.minUsd) && Number.isFinite(r?.maxUsd);
|
|
37
|
+
|
|
38
|
+
function addShadow(shadow, wouldDiffer, range) {
|
|
39
|
+
const priced = wouldDiffer && isRange(range);
|
|
40
|
+
return {
|
|
41
|
+
differs: shadow.differs + (wouldDiffer ? 1 : 0),
|
|
42
|
+
turns: shadow.turns + 1,
|
|
43
|
+
estimated: shadow.estimated + (priced ? 1 : 0),
|
|
44
|
+
minUsd: shadow.minUsd + (priced ? range.minUsd : 0),
|
|
45
|
+
maxUsd: shadow.maxUsd + (priced ? range.maxUsd : 0),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function emptySession() {
|
|
50
|
+
return {
|
|
51
|
+
byActivity: {},
|
|
52
|
+
switches: { tier: 0, activity: 0 },
|
|
53
|
+
agreement: { matched: 0, total: 0 },
|
|
54
|
+
shadow: emptyShadow(),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
|
|
59
|
+
// agreement compares with the tools (null: not counted), `shadow` marks a turn in shadow mode, `wouldDiffer` says
|
|
60
|
+
// whether 'on' would have routed it differently (null: no would-route) and `wouldUsd` prices that difference.
|
|
61
|
+
export function recordTurn(
|
|
62
|
+
session,
|
|
63
|
+
{
|
|
64
|
+
activity,
|
|
65
|
+
answer,
|
|
66
|
+
observed,
|
|
67
|
+
requests,
|
|
68
|
+
inputTokens,
|
|
69
|
+
outputTokens,
|
|
70
|
+
tierSwitches,
|
|
71
|
+
activitySwitches,
|
|
72
|
+
wouldDiffer,
|
|
73
|
+
wouldUsd,
|
|
74
|
+
shadow,
|
|
75
|
+
},
|
|
76
|
+
) {
|
|
77
|
+
const key = activity ?? 'none';
|
|
78
|
+
const before = session.byActivity[key] ?? { turns: 0, requests: 0, inputTokens: 0, outputTokens: 0 };
|
|
79
|
+
return {
|
|
80
|
+
byActivity: {
|
|
81
|
+
...session.byActivity,
|
|
82
|
+
[key]: {
|
|
83
|
+
turns: before.turns + 1,
|
|
84
|
+
requests: before.requests + count(requests),
|
|
85
|
+
inputTokens: before.inputTokens + count(inputTokens),
|
|
86
|
+
outputTokens: before.outputTokens + count(outputTokens),
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
switches: {
|
|
90
|
+
tier: session.switches.tier + count(tierSwitches),
|
|
91
|
+
activity: session.switches.activity + count(activitySwitches),
|
|
92
|
+
},
|
|
93
|
+
agreement: answer
|
|
94
|
+
? {
|
|
95
|
+
matched: session.agreement.matched + (agrees(answer, observed) ? 1 : 0),
|
|
96
|
+
total: session.agreement.total + 1,
|
|
97
|
+
}
|
|
98
|
+
: session.agreement,
|
|
99
|
+
// A view saved by 1.6.0 has no estimate fields.
|
|
100
|
+
shadow: shadow
|
|
101
|
+
? addShadow({ ...emptyShadow(), ...session.shadow }, wouldDiffer === true, wouldUsd)
|
|
102
|
+
: { ...emptyShadow(), ...session.shadow },
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function emptyStore() {
|
|
107
|
+
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: emptyShadow() };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
111
|
+
const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
112
|
+
const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
|
|
113
|
+
const countsOf = (v, keys) => onlyKeys(v, keys) && Object.values(v).every(isCount);
|
|
114
|
+
const exactCountsOf = (v, keys) =>
|
|
115
|
+
isRecord(v) && Object.keys(v).length === keys.length && keys.every((k) => isCount(v[k]));
|
|
116
|
+
|
|
117
|
+
const SHADOW_COUNTS = ['differs', 'turns'];
|
|
118
|
+
const SHADOW_ESTIMATE = ['estimated', 'minUsd', 'maxUsd'];
|
|
119
|
+
// 1.6.0 kept the shadow counts only; 1.6.1 adds the estimate. Either shape, never a part of the estimate.
|
|
120
|
+
const isShadow = (v) =>
|
|
121
|
+
exactCountsOf(v, SHADOW_COUNTS) ||
|
|
122
|
+
(onlyKeys(v, [...SHADOW_COUNTS, ...SHADOW_ESTIMATE]) &&
|
|
123
|
+
Object.keys(v).length === 5 &&
|
|
124
|
+
SHADOW_COUNTS.every((k) => isCount(v[k])) &&
|
|
125
|
+
isCount(v.estimated) &&
|
|
126
|
+
Number.isFinite(v.minUsd) &&
|
|
127
|
+
Number.isFinite(v.maxUsd));
|
|
128
|
+
|
|
129
|
+
// The stored value if it has a known shape, filled to the current one, else an empty store: a hand-edited or
|
|
130
|
+
// unknown file must not break stats.
|
|
131
|
+
export function readStore(value) {
|
|
132
|
+
const ok =
|
|
133
|
+
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
134
|
+
value.version === 1 &&
|
|
135
|
+
onlyKeys(value.confusion, CONFUSION_ROWS) &&
|
|
136
|
+
Object.values(value.confusion).every((row) => countsOf(row, OBSERVED)) &&
|
|
137
|
+
onlyKeys(value.runs, ACTIVITIES) &&
|
|
138
|
+
Object.values(value.runs).every(
|
|
139
|
+
(lengths) => Array.isArray(lengths) && lengths.length === RUN_BUCKETS && lengths.every(isCount),
|
|
140
|
+
) &&
|
|
141
|
+
exactCountsOf(value.lateral, ['taken', 'refused']) &&
|
|
142
|
+
isShadow(value.shadow);
|
|
143
|
+
return ok ? { ...value, shadow: { ...emptyShadow(), ...value.shadow } } : emptyStore();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// One finished turn. `predicted` is the classifier's answer (null: none), `runEnded` a run that closed this turn,
|
|
147
|
+
// `lateral` the outcome of a lateral switch decision, `wouldDiffer` the shadow readout (null: not in shadow) and
|
|
148
|
+
// `wouldUsd` its estimate. Unknown labels are dropped, so the store keeps a fixed set of keys.
|
|
149
|
+
export function recordStore(store, { predicted, observed, runEnded, lateral, wouldDiffer, wouldUsd }) {
|
|
150
|
+
const row = predicted ?? 'none';
|
|
151
|
+
const next = {
|
|
152
|
+
version: 1,
|
|
153
|
+
confusion: { ...store.confusion },
|
|
154
|
+
runs: { ...store.runs },
|
|
155
|
+
lateral: { ...store.lateral },
|
|
156
|
+
shadow: { ...store.shadow },
|
|
157
|
+
};
|
|
158
|
+
if (CONFUSION_ROWS.includes(row) && OBSERVED.includes(observed))
|
|
159
|
+
next.confusion[row] = { ...next.confusion[row], [observed]: (next.confusion[row]?.[observed] ?? 0) + 1 };
|
|
160
|
+
if (runEnded && ACTIVITIES.includes(runEnded.activity) && runEnded.length >= 1) {
|
|
161
|
+
const lengths = [...(next.runs[runEnded.activity] ?? Array(RUN_BUCKETS).fill(0))];
|
|
162
|
+
lengths[Math.min(Math.floor(runEnded.length), RUN_BUCKETS) - 1] += 1;
|
|
163
|
+
next.runs[runEnded.activity] = lengths;
|
|
164
|
+
}
|
|
165
|
+
if (lateral === 'taken' || lateral === 'refused') next.lateral[lateral] += 1;
|
|
166
|
+
if (typeof wouldDiffer === 'boolean') next.shadow = addShadow(next.shadow, wouldDiffer, wouldUsd);
|
|
167
|
+
return next;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// A run is consecutive turns with the same predicted activity. Given the run so far ({ activity, length } or null)
|
|
171
|
+
// and this turn's predicted activity (null or 'uncertain' end it), returns the run now and the run that just ended.
|
|
172
|
+
export function advanceRun(run, predicted) {
|
|
173
|
+
const activity = ACTIVITIES.includes(predicted) ? predicted : null;
|
|
174
|
+
if (run && run.activity === activity) return { run: { activity, length: run.length + 1 }, ended: null };
|
|
175
|
+
return { run: activity ? { activity, length: 1 } : null, ended: run };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const LATERAL = { 'activity-up': 'taken', 'activity-down': 'taken', 'activity-pending': 'refused' };
|
|
179
|
+
const differs = (a, b) => a.model !== b.model || (a.effort ?? null) !== (b.effort ?? null);
|
|
180
|
+
|
|
181
|
+
// What one routed turn adds to the stats, from its config, the classifier's advice, and the decisions before and
|
|
182
|
+
// after it. `label` is the turn's activity: the applied one in `on`, the accepted answer in `shadow`; `answer` is the
|
|
183
|
+
// accepted answer in both, which the tools are checked against and runs follow. `switched` names
|
|
184
|
+
// what moved the route from the previous turn's: 'activity' for a lateral move, 'tier' for any other, null for none
|
|
185
|
+
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`; a lateral move refused
|
|
186
|
+
// by a hold or the cash gate shares its reason with tier decisions and is not counted.
|
|
187
|
+
export function turnActivity(config, advice, previous, decision) {
|
|
188
|
+
const mode = config.activityRouting;
|
|
189
|
+
const would = decision.wouldRoute ?? null;
|
|
190
|
+
const switched =
|
|
191
|
+
previous?.model && !decision.pinned && differs(previous, decision)
|
|
192
|
+
? LATERAL[decision.reason] === 'taken'
|
|
193
|
+
? 'activity'
|
|
194
|
+
: 'tier'
|
|
195
|
+
: null;
|
|
196
|
+
const answer = mode === 'off' ? null : acceptActivity(config, advice);
|
|
197
|
+
const wouldDiffer = mode === 'shadow' ? Boolean(would && differs(would, decision)) : null;
|
|
198
|
+
return {
|
|
199
|
+
mode,
|
|
200
|
+
answer,
|
|
201
|
+
label: mode === 'on' ? (decision.activity ?? null) : mode === 'shadow' ? answer : null,
|
|
202
|
+
predicted: advice?.activity?.choice ?? null,
|
|
203
|
+
switched,
|
|
204
|
+
lateral: LATERAL[(mode === 'shadow' ? would : mode === 'on' ? decision : null)?.reason] ?? null,
|
|
205
|
+
// A shadow turn always counts, so the session and store denominators agree.
|
|
206
|
+
wouldDiffer,
|
|
207
|
+
wouldUsd: wouldDiffer ? (would.difference ?? null) : null,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// The activities plan §11 checks agreement on before activity routing turns on by default.
|
|
212
|
+
const FOCUS = ['code', 'ops', 'explore'];
|
|
213
|
+
|
|
214
|
+
// The counts kept across sessions as the Usage tab and /router show them. Agreement covers the rows that name an
|
|
215
|
+
// activity: the store holds the raw answer, and `uncertain` or no answer never agrees; `focus` is the same over the
|
|
216
|
+
// FOCUS rows only. The two largest disagreements come first; ties keep the matrix order, so the readout is stable.
|
|
217
|
+
export function storeSummary(store) {
|
|
218
|
+
let labelled = 0;
|
|
219
|
+
let matched = 0;
|
|
220
|
+
let total = 0;
|
|
221
|
+
const focus = { matched: 0, total: 0 };
|
|
222
|
+
const misses = [];
|
|
223
|
+
for (const predicted of CONFUSION_ROWS)
|
|
224
|
+
for (const observed of OBSERVED) {
|
|
225
|
+
const n = store.confusion[predicted]?.[observed] ?? 0;
|
|
226
|
+
labelled += n;
|
|
227
|
+
if (!n || !ALLOWED[predicted]) continue;
|
|
228
|
+
total += n;
|
|
229
|
+
const agreed = agrees(predicted, observed);
|
|
230
|
+
if (agreed) matched += n;
|
|
231
|
+
else misses.push({ predicted, observed, n });
|
|
232
|
+
if (FOCUS.includes(predicted)) {
|
|
233
|
+
focus.total += n;
|
|
234
|
+
if (agreed) focus.matched += n;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
labelled,
|
|
239
|
+
agreement: { matched, total },
|
|
240
|
+
focus,
|
|
241
|
+
disagreements: misses.sort((a, b) => b.n - a.n).slice(0, 2),
|
|
242
|
+
lateral: store.lateral,
|
|
243
|
+
shadow: store.shadow,
|
|
244
|
+
};
|
|
245
|
+
}
|