@alexeiled/claude-router 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +31 -0
- package/docs/architecture.md +4 -4
- package/docs/evaluation.md +5 -5
- package/docs/native-router.md +4 -4
- package/docs/router-pane-classifier.svg +39 -34
- package/docs/router-pane-usage.svg +32 -21
- package/docs/user-guide.md +6 -6
- package/hooks/native-router.mjs +116 -20
- package/lib/activity-stats.mjs +164 -10
- package/lib/display.mjs +33 -0
- package/lib/panel.mjs +61 -15
- package/lib/probe-results.mjs +40 -0
- package/lib/route.mjs +26 -8
- package/lib/savings.mjs +1 -0
- package/lib/view.mjs +1 -0
- package/package.json +2 -1
- package/types/index.d.ts +23 -0
package/hooks/native-router.mjs
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
+
advanceDown,
|
|
2
3
|
advanceRun,
|
|
4
|
+
emptyMetrics,
|
|
3
5
|
emptySession,
|
|
4
6
|
emptyStore,
|
|
7
|
+
METRICS_PREFIX,
|
|
8
|
+
mergeMetrics,
|
|
9
|
+
metricsAfterReset,
|
|
10
|
+
readMetrics,
|
|
5
11
|
readStore,
|
|
12
|
+
recordMetrics,
|
|
6
13
|
recordStore,
|
|
7
14
|
recordTurn,
|
|
8
15
|
STORE_KEY,
|
|
@@ -103,6 +110,8 @@ function createRouter(options) {
|
|
|
103
110
|
// one activity, flushed to the store when it ends.
|
|
104
111
|
turnActivities: new Map(),
|
|
105
112
|
run: null,
|
|
113
|
+
// The route a cheaper activity move went to, while the turns stay on it (advanceDown); else null.
|
|
114
|
+
afterDown: null,
|
|
106
115
|
// Each turn's routing-vs-your-model totals until turn.complete adds them to the store.
|
|
107
116
|
turnSavings: new Map(),
|
|
108
117
|
// The totals kept across sessions: this session's record (`{ sessionId, record }`), held here so a refresh never
|
|
@@ -110,6 +119,9 @@ function createRouter(options) {
|
|
|
110
119
|
// refresh. The view shows their sum.
|
|
111
120
|
savingsOwn: null,
|
|
112
121
|
savingsOthers: null,
|
|
122
|
+
// The activity metrics kept across sessions, held the same way: this session's record and the others' sum.
|
|
123
|
+
metricsOwn: null,
|
|
124
|
+
metricsOthers: null,
|
|
113
125
|
// Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
|
|
114
126
|
statsResets: 0,
|
|
115
127
|
};
|
|
@@ -191,6 +203,27 @@ async function otherSavings($, sessionId, resetAt) {
|
|
|
191
203
|
return mergeSavingsStores(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
|
|
192
204
|
}
|
|
193
205
|
|
|
206
|
+
// This session's metrics record and the last reset, as ownSavings: Reset stats clears both with one marker.
|
|
207
|
+
async function ownMetrics($, router) {
|
|
208
|
+
const sessionId = await $.session.id();
|
|
209
|
+
const resetAt = await $.store.get(SAVINGS_RESET_KEY);
|
|
210
|
+
if (router.metricsOwn?.sessionId !== sessionId) {
|
|
211
|
+
const record = readMetrics(await $.store.get(`${METRICS_PREFIX}${sessionId}`));
|
|
212
|
+
if (router.metricsOwn?.sessionId !== sessionId) router.metricsOwn = { sessionId, record };
|
|
213
|
+
}
|
|
214
|
+
router.metricsOwn.record = metricsAfterReset(router.metricsOwn.record, resetAt);
|
|
215
|
+
return { own: router.metricsOwn, resetAt };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Every other session's metrics since the last reset, added up.
|
|
219
|
+
async function otherMetrics($, sessionId, resetAt) {
|
|
220
|
+
const own = `${METRICS_PREFIX}${sessionId}`;
|
|
221
|
+
const keys = (await $.store.keys()).filter((key) => key.startsWith(METRICS_PREFIX) && key !== own);
|
|
222
|
+
return mergeMetrics(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const metricsView = (router) => mergeMetrics([router.metricsOthers, router.metricsOwn?.record]);
|
|
226
|
+
|
|
194
227
|
// The view's totals kept across sessions, from what the router holds; nothing until a read succeeded.
|
|
195
228
|
const savedView = (router) =>
|
|
196
229
|
router.savingsOthers ? { savingsStore: mergeSavingsStores([router.savingsOthers, router.savingsOwn?.record]) } : {};
|
|
@@ -203,6 +236,11 @@ async function storeView($, router) {
|
|
|
203
236
|
try {
|
|
204
237
|
out.activityStore = readStore(await $.store.get(STORE_KEY));
|
|
205
238
|
} catch {}
|
|
239
|
+
try {
|
|
240
|
+
const { own, resetAt } = await ownMetrics($, router);
|
|
241
|
+
router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
|
|
242
|
+
out.activityMetrics = metricsView(router);
|
|
243
|
+
} catch {}
|
|
206
244
|
try {
|
|
207
245
|
const { own, resetAt } = await ownSavings($, router);
|
|
208
246
|
router.savingsOthers = await otherSavings($, own.sessionId, resetAt);
|
|
@@ -220,6 +258,8 @@ async function openPane($, router) {
|
|
|
220
258
|
async function changeMode($, router, mode) {
|
|
221
259
|
const view = router.view ?? (await readView($, router));
|
|
222
260
|
for (const controller of router.controllers) controller.abort();
|
|
261
|
+
// Turns with routing off run on Claude's model, off the cheaper move's route: the escalation watch ends.
|
|
262
|
+
if (mode === 'manual') router.afterDown = null;
|
|
223
263
|
router.modes.set(await $.session.id(), mode);
|
|
224
264
|
const saved = await rememberMode($, mode);
|
|
225
265
|
await updateView($, router, {
|
|
@@ -388,6 +428,9 @@ async function contextOf($, loop) {
|
|
|
388
428
|
}
|
|
389
429
|
|
|
390
430
|
async function* passMain($, e, next, loop, version, nativeModel, reason, router) {
|
|
431
|
+
// A turn routing did not choose is a turn on another route: it ends the watch for escalations after a cheaper
|
|
432
|
+
// activity move, including one a turn started before routing was turned off mid-turn.
|
|
433
|
+
if (reason === 'manual') router.afterDown = null;
|
|
391
434
|
const ref = { ...LOOP, id: 'main' };
|
|
392
435
|
const context = await contextOf($, loop);
|
|
393
436
|
await updateView($, router, {
|
|
@@ -461,6 +504,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
461
504
|
turns: facts.turns,
|
|
462
505
|
signal: controller.signal,
|
|
463
506
|
});
|
|
507
|
+
// The wait for advice alone, comparable with the activity probe's p95; the routing after it is not the classifier's.
|
|
508
|
+
const adviceMs = pin || !classifierTimed(result.error) ? null : Date.now() - adviceStarted;
|
|
464
509
|
if (controller.signal.aborted || (await $.session.id()) !== sessionId || (await modeOf($, router)) !== 'auto')
|
|
465
510
|
return null;
|
|
466
511
|
const previous = loop.decision;
|
|
@@ -480,7 +525,22 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
480
525
|
if (previous?.model && previous.model !== selected.decision.model && !selected.decision.pinned)
|
|
481
526
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
482
527
|
const activity = turnActivity(cfg, result.advice, previous, selected.decision);
|
|
483
|
-
router.turnActivities.set(e.turnId, {
|
|
528
|
+
router.turnActivities.set(e.turnId, {
|
|
529
|
+
...activity,
|
|
530
|
+
reason: selected.decision.reason,
|
|
531
|
+
escalated: Boolean(selected.decision.escalated),
|
|
532
|
+
pinned: selected.decision.pinned,
|
|
533
|
+
route: `${selected.decision.model}@${selected.decision.effort ?? 'session'}`,
|
|
534
|
+
classifier: cfg.classifier,
|
|
535
|
+
adviceMs,
|
|
536
|
+
sessionId,
|
|
537
|
+
requests: 0,
|
|
538
|
+
inputTokens: 0,
|
|
539
|
+
outputTokens: 0,
|
|
540
|
+
routedUsd: null,
|
|
541
|
+
pricedRequests: 0,
|
|
542
|
+
routes: {},
|
|
543
|
+
});
|
|
484
544
|
await updateView($, router, {
|
|
485
545
|
phase: 'routed',
|
|
486
546
|
selectedModel: selected.decision.model,
|
|
@@ -498,7 +558,7 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
498
558
|
? {
|
|
499
559
|
error: result.error,
|
|
500
560
|
health: healthOf(clientOf(router, cfg.classifier), cfg.classifier),
|
|
501
|
-
adviceMs
|
|
561
|
+
adviceMs,
|
|
502
562
|
adviceChoice: result.advice?.choice ?? null,
|
|
503
563
|
probabilities: result.advice?.probabilities ?? null,
|
|
504
564
|
estimate: selected.decision.estimate ?? null,
|
|
@@ -517,8 +577,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
517
577
|
}
|
|
518
578
|
|
|
519
579
|
// A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
|
|
520
|
-
// turn's activity record.
|
|
521
|
-
async function publishReply($, router, turnId, response, tier, route) {
|
|
580
|
+
// turn's activity record with its list price (`routedUsd`, null when unpriced) and, when its tier served it, its route.
|
|
581
|
+
async function publishReply($, router, turnId, response, tier, route, routedUsd) {
|
|
522
582
|
const turn = router.turnActivities.get(turnId);
|
|
523
583
|
const metrics = responseMetrics(
|
|
524
584
|
router.view,
|
|
@@ -534,6 +594,9 @@ async function publishReply($, router, turnId, response, tier, route) {
|
|
|
534
594
|
requests: turn.requests + 1,
|
|
535
595
|
inputTokens: turn.inputTokens + (metrics.inputTokens ?? 0),
|
|
536
596
|
outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
|
|
597
|
+
routedUsd: Number.isFinite(routedUsd) ? (turn.routedUsd ?? 0) + routedUsd : turn.routedUsd,
|
|
598
|
+
pricedRequests: turn.pricedRequests + (Number.isFinite(routedUsd) ? 1 : 0),
|
|
599
|
+
routes: tier ? { ...turn.routes, [route]: (turn.routes[route] ?? 0) + 1 } : turn.routes,
|
|
537
600
|
});
|
|
538
601
|
}
|
|
539
602
|
|
|
@@ -554,7 +617,8 @@ async function observeRouted($, router, { turnId, cfg, loop, loopVersion, reques
|
|
|
554
617
|
if (!written.isSet) return;
|
|
555
618
|
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
556
619
|
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
557
|
-
|
|
620
|
+
const route = `${request.model}@${request.effort ?? 'session'}`;
|
|
621
|
+
await publishReply($, router, turnId, response, tier, route, compared?.reply?.routedUsd ?? null);
|
|
558
622
|
if (compared?.reply) await addSavings($, router, turnId, compared.reply, tier);
|
|
559
623
|
}
|
|
560
624
|
|
|
@@ -623,7 +687,8 @@ async function flushSavings($, router, totals) {
|
|
|
623
687
|
}
|
|
624
688
|
|
|
625
689
|
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
626
|
-
// break a turn: a failed read or write loses this turn's counts
|
|
690
|
+
// break a turn: a failed read or write loses this turn's counts. Readings already in the session's metrics record stay
|
|
691
|
+
// in memory after a failed write and go out with the next turn that adds a reading.
|
|
627
692
|
async function recordActivity($, router, turn) {
|
|
628
693
|
try {
|
|
629
694
|
if ((await $.session.id()) !== turn.sessionId) return;
|
|
@@ -637,6 +702,8 @@ async function recordActivity($, router, turn) {
|
|
|
637
702
|
// Advanced before any further await, so a session change meanwhile cannot carry this run into the new session.
|
|
638
703
|
const { run, ended } = advanceRun(router.run, turn.answer);
|
|
639
704
|
router.run = run;
|
|
705
|
+
const down = advanceDown(router.afterDown, turn);
|
|
706
|
+
router.afterDown = down.after;
|
|
640
707
|
await updateView($, router, {
|
|
641
708
|
activityStats: recordTurn(view.activityStats ?? emptySession(), {
|
|
642
709
|
activity: turn.label,
|
|
@@ -645,8 +712,12 @@ async function recordActivity($, router, turn) {
|
|
|
645
712
|
requests: turn.requests,
|
|
646
713
|
inputTokens: turn.inputTokens,
|
|
647
714
|
outputTokens: turn.outputTokens,
|
|
715
|
+
routedUsd: turn.routedUsd,
|
|
716
|
+
pricedRequests: turn.pricedRequests,
|
|
717
|
+
routes: turn.routes,
|
|
648
718
|
tierSwitches: turn.switched === 'tier' ? 1 : 0,
|
|
649
719
|
activitySwitches: turn.switched === 'activity' ? 1 : 0,
|
|
720
|
+
lateral: turn.lateral,
|
|
650
721
|
wouldDiffer: turn.wouldDiffer,
|
|
651
722
|
wouldUsd: turn.wouldUsd,
|
|
652
723
|
shadow: turn.mode === 'shadow',
|
|
@@ -654,6 +725,10 @@ async function recordActivity($, router, turn) {
|
|
|
654
725
|
});
|
|
655
726
|
const resets = router.statsResets;
|
|
656
727
|
const store = readStore(await $.store.get(STORE_KEY));
|
|
728
|
+
const { own, resetAt } = await ownMetrics($, router);
|
|
729
|
+
// The others held from before a reset in another session are read again.
|
|
730
|
+
if (!router.metricsOthers || metricsAfterReset(router.metricsOthers, resetAt) !== router.metricsOthers)
|
|
731
|
+
router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
|
|
657
732
|
// Writing counts read before a reset would bring them back; this turn's are lost instead.
|
|
658
733
|
if (router.statsResets !== resets) return;
|
|
659
734
|
const written = recordStore(store, {
|
|
@@ -664,9 +739,23 @@ async function recordActivity($, router, turn) {
|
|
|
664
739
|
wouldDiffer: turn.wouldDiffer,
|
|
665
740
|
wouldUsd: turn.wouldUsd,
|
|
666
741
|
});
|
|
667
|
-
|
|
742
|
+
const measured = recordMetrics(own.record, {
|
|
743
|
+
classifier: turn.classifier,
|
|
744
|
+
adviceMs: turn.adviceMs,
|
|
745
|
+
downMove: down.event,
|
|
746
|
+
now: Date.now(),
|
|
747
|
+
});
|
|
748
|
+
// Added to in memory before the write, so a refresh during the write counts the turn once.
|
|
749
|
+
const changed = measured !== own.record;
|
|
750
|
+
own.record = measured;
|
|
751
|
+
// Both writes start before any reset can run between them.
|
|
752
|
+
await Promise.all([
|
|
753
|
+
$.store.set(STORE_KEY, written),
|
|
754
|
+
changed ? $.store.set(`${METRICS_PREFIX}${own.sessionId}`, measured) : null,
|
|
755
|
+
]);
|
|
668
756
|
// A reset during the write emptied the view; these counts are older than it.
|
|
669
|
-
if (router.statsResets === resets)
|
|
757
|
+
if (router.statsResets === resets)
|
|
758
|
+
await updateView($, router, { activityStore: written, activityMetrics: metricsView(router) });
|
|
670
759
|
} catch {}
|
|
671
760
|
}
|
|
672
761
|
|
|
@@ -832,33 +921,36 @@ function paneActions($, router, view) {
|
|
|
832
921
|
resetStats: async () => {
|
|
833
922
|
router.statsResets += 1;
|
|
834
923
|
router.run = null;
|
|
924
|
+
router.afterDown = null;
|
|
835
925
|
router.turnSavings.clear();
|
|
836
926
|
const clear = (key, value) =>
|
|
837
927
|
$.store
|
|
838
928
|
.set(key, value)
|
|
839
929
|
.then(() => true)
|
|
840
930
|
.catch(() => false);
|
|
841
|
-
|
|
931
|
+
// The per-session records: savings and activity metrics.
|
|
932
|
+
const clearRecords = async () => {
|
|
842
933
|
// Written first, so a session still holding its record from before the reset drops it, unless that record's
|
|
843
|
-
// first
|
|
934
|
+
// first reading has the reset's millisecond or the clock moved back across the reset.
|
|
844
935
|
await $.store.set(SAVINGS_RESET_KEY, Date.now());
|
|
845
|
-
const keys = (await $.store.keys()).filter(
|
|
936
|
+
const keys = (await $.store.keys()).filter(
|
|
937
|
+
(key) => key.startsWith(SAVINGS_PREFIX) || key.startsWith(METRICS_PREFIX),
|
|
938
|
+
);
|
|
846
939
|
await Promise.all(keys.map((key) => $.store.delete(key)));
|
|
847
940
|
router.savingsOwn = null;
|
|
848
941
|
router.savingsOthers = emptySavingsStore();
|
|
942
|
+
router.metricsOwn = null;
|
|
943
|
+
router.metricsOthers = emptyMetrics();
|
|
849
944
|
return true;
|
|
850
945
|
};
|
|
851
|
-
const [
|
|
852
|
-
clear(STORE_KEY, emptyStore()),
|
|
853
|
-
clearSavings().catch(() => false),
|
|
854
|
-
]);
|
|
946
|
+
const [stats, records] = await Promise.all([clear(STORE_KEY, emptyStore()), clearRecords().catch(() => false)]);
|
|
855
947
|
await updateView($, router, {
|
|
856
948
|
activityStats: null,
|
|
857
949
|
savings: null,
|
|
858
|
-
...(
|
|
859
|
-
...(
|
|
950
|
+
...(stats ? { activityStore: emptyStore() } : {}),
|
|
951
|
+
...(records ? { activityMetrics: emptyMetrics(), savingsStore: emptySavingsStore() } : {}),
|
|
860
952
|
notice:
|
|
861
|
-
|
|
953
|
+
stats && records
|
|
862
954
|
? 'Stats reset: activity and routing vs your model, this session and saved.'
|
|
863
955
|
: 'This session’s stats reset. Saved stats could not be cleared.',
|
|
864
956
|
});
|
|
@@ -1076,10 +1168,12 @@ export function register(on, options) {
|
|
|
1076
1168
|
}
|
|
1077
1169
|
router.prompts.delete(e.turnId);
|
|
1078
1170
|
for (const key of router.decisions.keys()) if (key.includes(`:${e.turnId}:`)) router.decisions.delete(key);
|
|
1079
|
-
// `off` asks nothing, so it records nothing.
|
|
1171
|
+
// `off` asks nothing, so it records nothing; it only ends the watch after a cheaper activity move.
|
|
1080
1172
|
const turn = router.turnActivities.get(e.turnId);
|
|
1081
1173
|
router.turnActivities.delete(e.turnId);
|
|
1082
|
-
if (turn
|
|
1174
|
+
if (turn?.mode === 'off') {
|
|
1175
|
+
if ((await $.session.id()) === turn.sessionId) router.afterDown = advanceDown(router.afterDown, turn).after;
|
|
1176
|
+
} else if (turn) await recordActivity($, router, turn);
|
|
1083
1177
|
const savings = router.turnSavings.get(e.turnId);
|
|
1084
1178
|
router.turnSavings.delete(e.turnId);
|
|
1085
1179
|
if (savings) await flushSavings($, router, savings);
|
|
@@ -1107,6 +1201,7 @@ export function register(on, options) {
|
|
|
1107
1201
|
router.turnSavings.clear();
|
|
1108
1202
|
const run = router.run;
|
|
1109
1203
|
router.run = null;
|
|
1204
|
+
router.afterDown = null;
|
|
1110
1205
|
if (run) await flushRun($, run);
|
|
1111
1206
|
router.view = {
|
|
1112
1207
|
...initialView(router.view?.nativeModel ?? ''),
|
|
@@ -1116,6 +1211,7 @@ export function register(on, options) {
|
|
|
1116
1211
|
credentials: router.view?.credentials ?? null,
|
|
1117
1212
|
// Counts kept across sessions: the next session starts from them.
|
|
1118
1213
|
activityStore: router.view?.activityStore ?? null,
|
|
1214
|
+
activityMetrics: router.view?.activityMetrics ?? null,
|
|
1119
1215
|
savingsStore: router.view?.savingsStore ?? null,
|
|
1120
1216
|
health: healthOf(clientOf(router, router.config.classifier), router.config.classifier),
|
|
1121
1217
|
configPath: router.config.nativePath,
|
package/lib/activity-stats.mjs
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store
|
|
2
|
-
// counts kept across sessions under `activity:stats:v1
|
|
1
|
+
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducers feed the
|
|
2
|
+
// counts kept across sessions under `activity:stats:v1` and, per session, under METRICS_PREFIX. Pure: state and a turn
|
|
3
|
+
// in, new state out.
|
|
3
4
|
|
|
4
5
|
import { acceptActivity } from './activity.mjs';
|
|
5
6
|
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
6
7
|
|
|
7
8
|
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
8
9
|
export const STORE_KEY = 'activity:stats:v1';
|
|
10
|
+
// Each session keeps its own metrics under this prefix and its session id, as savings does (SAVINGS_PREFIX): sessions
|
|
11
|
+
// finishing turns at the same moment do not overwrite each other. The readout is their sum (mergeMetrics).
|
|
12
|
+
export const METRICS_PREFIX = 'activity:metrics:v1:';
|
|
9
13
|
|
|
10
14
|
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
11
15
|
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
@@ -46,18 +50,35 @@ function addShadow(shadow, wouldDiffer, range) {
|
|
|
46
50
|
};
|
|
47
51
|
}
|
|
48
52
|
|
|
53
|
+
function addRoutes(before, added) {
|
|
54
|
+
const next = { ...before };
|
|
55
|
+
for (const [route, n] of Object.entries(added ?? {})) if (count(n)) next[route] = (next[route] ?? 0) + count(n);
|
|
56
|
+
return next;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// The route most of an activity's routed replies went out with, `model@effort`; ties go to the first in name order,
|
|
60
|
+
// so the readout is stable. Null without routed replies.
|
|
61
|
+
export function mostlyOn(counts) {
|
|
62
|
+
const [top] = Object.entries(counts?.routes ?? {}).sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
|
|
63
|
+
return top?.[0] ?? null;
|
|
64
|
+
}
|
|
65
|
+
|
|
49
66
|
export function emptySession() {
|
|
50
67
|
return {
|
|
51
68
|
byActivity: {},
|
|
52
69
|
switches: { tier: 0, activity: 0 },
|
|
70
|
+
lateral: { taken: 0, refused: 0 },
|
|
53
71
|
agreement: { matched: 0, total: 0 },
|
|
54
72
|
shadow: emptyShadow(),
|
|
55
73
|
};
|
|
56
74
|
}
|
|
57
75
|
|
|
58
76
|
// One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
|
|
59
|
-
// agreement compares with the tools (null: not counted), `
|
|
60
|
-
//
|
|
77
|
+
// agreement compares with the tools (null: not counted), `lateral` the turn's lateral outcome (turnActivity, counted
|
|
78
|
+
// outside shadow only), `shadow` marks a turn in shadow mode, `wouldDiffer` says whether 'on' would have routed it
|
|
79
|
+
// differently (null: no would-route) and `wouldUsd` prices that difference. `routedUsd` is the list price of the
|
|
80
|
+
// turn's priced replies (null: none had a price), `pricedRequests` how many of its replies had one, and `routes` counts
|
|
81
|
+
// its routed replies by the `model@effort` they went out with.
|
|
61
82
|
export function recordTurn(
|
|
62
83
|
session,
|
|
63
84
|
{
|
|
@@ -67,8 +88,12 @@ export function recordTurn(
|
|
|
67
88
|
requests,
|
|
68
89
|
inputTokens,
|
|
69
90
|
outputTokens,
|
|
91
|
+
routedUsd = null,
|
|
92
|
+
pricedRequests = 0,
|
|
93
|
+
routes = {},
|
|
70
94
|
tierSwitches,
|
|
71
95
|
activitySwitches,
|
|
96
|
+
lateral,
|
|
72
97
|
wouldDiffer,
|
|
73
98
|
wouldUsd,
|
|
74
99
|
shadow,
|
|
@@ -76,6 +101,8 @@ export function recordTurn(
|
|
|
76
101
|
) {
|
|
77
102
|
const key = activity ?? 'none';
|
|
78
103
|
const before = session.byActivity[key] ?? { turns: 0, requests: 0, inputTokens: 0, outputTokens: 0 };
|
|
104
|
+
// A view saved by 1.7.0 has no lateral counts.
|
|
105
|
+
const moves = { taken: 0, refused: 0, ...session.lateral };
|
|
79
106
|
return {
|
|
80
107
|
byActivity: {
|
|
81
108
|
...session.byActivity,
|
|
@@ -84,12 +111,19 @@ export function recordTurn(
|
|
|
84
111
|
requests: before.requests + count(requests),
|
|
85
112
|
inputTokens: before.inputTokens + count(inputTokens),
|
|
86
113
|
outputTokens: before.outputTokens + count(outputTokens),
|
|
114
|
+
// Counts saved by 1.7.0 have no cost, priced replies or routes: their replies stay unpriced.
|
|
115
|
+
routedUsd: Number.isFinite(routedUsd) ? (before.routedUsd ?? 0) + routedUsd : (before.routedUsd ?? null),
|
|
116
|
+
pricedRequests: (before.pricedRequests ?? 0) + count(pricedRequests),
|
|
117
|
+
routes: addRoutes(before.routes ?? {}, routes),
|
|
87
118
|
},
|
|
88
119
|
},
|
|
89
120
|
switches: {
|
|
90
121
|
tier: session.switches.tier + count(tierSwitches),
|
|
91
122
|
activity: session.switches.activity + count(activitySwitches),
|
|
92
123
|
},
|
|
124
|
+
// In shadow the lateral outcome is the would-route's, which no route followed; the Shadow line covers it.
|
|
125
|
+
lateral:
|
|
126
|
+
!shadow && (lateral === 'taken' || lateral === 'refused') ? { ...moves, [lateral]: moves[lateral] + 1 } : moves,
|
|
93
127
|
agreement: answer
|
|
94
128
|
? {
|
|
95
129
|
matched: session.agreement.matched + (agrees(answer, observed) ? 1 : 0),
|
|
@@ -103,10 +137,33 @@ export function recordTurn(
|
|
|
103
137
|
};
|
|
104
138
|
}
|
|
105
139
|
|
|
140
|
+
// Classifier wait per turn, bucketed by these upper bounds in ms, plus one bucket above the last; kept per classifier
|
|
141
|
+
// id for at most LATENCY_CLASSIFIERS ids, so the store stays bounded.
|
|
142
|
+
export const LATENCY_BOUNDS = [
|
|
143
|
+
100, 150, 200, 250, 300, 350, 400, 450, 500, 600, 700, 800, 900, 1000, 1250, 1500, 2000, 2500, 3000, 4000, 5000,
|
|
144
|
+
];
|
|
145
|
+
const LATENCY_BUCKETS = LATENCY_BOUNDS.length + 1;
|
|
146
|
+
const LATENCY_CLASSIFIERS = 8;
|
|
147
|
+
|
|
106
148
|
export function emptyStore() {
|
|
107
149
|
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: emptyShadow() };
|
|
108
150
|
}
|
|
109
151
|
|
|
152
|
+
// The p95 of a latency histogram: the upper bound of its bucket in ms, `over` when it lies above the last bound, and
|
|
153
|
+
// the turns counted. Null without turns.
|
|
154
|
+
export function latencyP95(buckets) {
|
|
155
|
+
const turns = (buckets ?? []).reduce((sum, n) => sum + n, 0);
|
|
156
|
+
if (!turns) return null;
|
|
157
|
+
let seen = 0;
|
|
158
|
+
const at = buckets.findIndex((n) => {
|
|
159
|
+
seen += n;
|
|
160
|
+
return seen >= 0.95 * turns;
|
|
161
|
+
});
|
|
162
|
+
return at < LATENCY_BOUNDS.length
|
|
163
|
+
? { ms: LATENCY_BOUNDS[at], over: false, turns }
|
|
164
|
+
: { ms: LATENCY_BOUNDS.at(-1), over: true, turns };
|
|
165
|
+
}
|
|
166
|
+
|
|
110
167
|
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
111
168
|
const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
112
169
|
const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
|
|
@@ -127,7 +184,8 @@ const isShadow = (v) =>
|
|
|
127
184
|
Number.isFinite(v.maxUsd));
|
|
128
185
|
|
|
129
186
|
// The stored value if it has a known shape, filled to the current one, else an empty store: a hand-edited or
|
|
130
|
-
// unknown file must not break stats.
|
|
187
|
+
// unknown file must not break stats. The shape is 1.7.0's, which rejects any other key: a new count goes under its own
|
|
188
|
+
// key, such as METRICS_KEY, so a downgrade keeps these.
|
|
131
189
|
export function readStore(value) {
|
|
132
190
|
const ok =
|
|
133
191
|
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
@@ -167,6 +225,101 @@ export function recordStore(store, { predicted, observed, runEnded, lateral, wou
|
|
|
167
225
|
return next;
|
|
168
226
|
}
|
|
169
227
|
|
|
228
|
+
// The readings kept across sessions beside the stats store, one session's or their sum: the classifier wait per
|
|
229
|
+
// classifier id, and the cheaper activity moves taken in `on` with the escalations that followed one, since `since`
|
|
230
|
+
// (ms), the first reading after a reset.
|
|
231
|
+
export function emptyMetrics() {
|
|
232
|
+
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 } };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const isLatency = (v) =>
|
|
236
|
+
isRecord(v) &&
|
|
237
|
+
Object.keys(v).length <= LATENCY_CLASSIFIERS &&
|
|
238
|
+
Object.values(v).every((b) => Array.isArray(b) && b.length === LATENCY_BUCKETS && b.every(isCount));
|
|
239
|
+
|
|
240
|
+
// Each part read on its own: a bad or outdated one, such as latency after LATENCY_BOUNDS changes, reads as empty and
|
|
241
|
+
// leaves the others.
|
|
242
|
+
export function readMetrics(value) {
|
|
243
|
+
const empty = emptyMetrics();
|
|
244
|
+
if (!isRecord(value) || value.version !== 1) return empty;
|
|
245
|
+
return {
|
|
246
|
+
version: 1,
|
|
247
|
+
since: Number.isFinite(value.since) ? value.since : null,
|
|
248
|
+
latency: isLatency(value.latency) ? value.latency : empty.latency,
|
|
249
|
+
downMoves: exactCountsOf(value.downMoves, ['moves', 'escalations']) ? value.downMoves : empty.downMoves,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed), and `downMove` 'moved' or
|
|
254
|
+
// 'escalated' (advanceDown). A turn that adds nothing returns `metrics` itself.
|
|
255
|
+
export function recordMetrics(metrics, { classifier = null, adviceMs = null, downMove = null, now = null }) {
|
|
256
|
+
const next = {
|
|
257
|
+
version: 1,
|
|
258
|
+
since: metrics.since ?? now,
|
|
259
|
+
latency: { ...metrics.latency },
|
|
260
|
+
downMoves: { ...metrics.downMoves },
|
|
261
|
+
};
|
|
262
|
+
let counted = downMove === 'moved' || downMove === 'escalated';
|
|
263
|
+
const known = Object.hasOwn(next.latency, classifier);
|
|
264
|
+
if (
|
|
265
|
+
typeof classifier === 'string' &&
|
|
266
|
+
Number.isFinite(adviceMs) &&
|
|
267
|
+
adviceMs >= 0 &&
|
|
268
|
+
(known || Object.keys(next.latency).length < LATENCY_CLASSIFIERS)
|
|
269
|
+
) {
|
|
270
|
+
const buckets = known ? [...next.latency[classifier]] : Array(LATENCY_BUCKETS).fill(0);
|
|
271
|
+
const at = LATENCY_BOUNDS.findIndex((bound) => adviceMs <= bound);
|
|
272
|
+
buckets[at === -1 ? LATENCY_BOUNDS.length : at] += 1;
|
|
273
|
+
next.latency[classifier] = buckets;
|
|
274
|
+
counted = true;
|
|
275
|
+
}
|
|
276
|
+
if (!counted) return metrics;
|
|
277
|
+
if (downMove === 'moved') next.downMoves.moves += 1;
|
|
278
|
+
if (downMove === 'escalated') next.downMoves.escalations += 1;
|
|
279
|
+
return next;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// A record whose first reading came before the last reset at `resetAt` (ms) reads as empty, as savings' afterReset.
|
|
283
|
+
export function metricsAfterReset(record, resetAt) {
|
|
284
|
+
return Number.isFinite(resetAt) && record.since !== null && record.since < resetAt ? emptyMetrics() : record;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// The metrics kept across sessions: every session's record since the last reset added up, since the earliest. Latency
|
|
288
|
+
// keeps the first LATENCY_CLASSIFIERS classifier ids in the order given, so the sum still reads back (readMetrics).
|
|
289
|
+
export function mergeMetrics(values, resetAt = null) {
|
|
290
|
+
const merged = emptyMetrics();
|
|
291
|
+
for (const one of values.map((value) => metricsAfterReset(readMetrics(value), resetAt))) {
|
|
292
|
+
const since = [merged.since, one.since].filter(Number.isFinite);
|
|
293
|
+
merged.since = since.length ? Math.min(...since) : null;
|
|
294
|
+
for (const [id, buckets] of Object.entries(one.latency)) {
|
|
295
|
+
// Own keys only: a classifier id such as `toString` must not read Object.prototype.
|
|
296
|
+
const known = Object.hasOwn(merged.latency, id) ? merged.latency[id] : null;
|
|
297
|
+
if (known) merged.latency[id] = known.map((n, i) => n + buckets[i]);
|
|
298
|
+
else if (Object.keys(merged.latency).length < LATENCY_CLASSIFIERS) merged.latency[id] = [...buckets];
|
|
299
|
+
}
|
|
300
|
+
merged.downMoves = {
|
|
301
|
+
moves: merged.downMoves.moves + one.downMoves.moves,
|
|
302
|
+
escalations: merged.downMoves.escalations + one.downMoves.escalations,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
return merged;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Escalations after a cheaper activity move. `after` is the `model@effort` route a cheaper move went to while the
|
|
309
|
+
// watch lasts, else null; `turn` is the turn's mode, decision reason and lateral outcome, whether a failure asked for
|
|
310
|
+
// an escalation (also one the cash gate held or context-fit replaced), whether it was pinned, and its route. A taken
|
|
311
|
+
// cheaper move in `on` starts the watch; an escalation while it lasts counts once and ends it, as does a turn on
|
|
312
|
+
// another route or a turn in another mode (an `off` turn too, which records nothing else); the hook also ends it when
|
|
313
|
+
// routing is turned off. A pin is the person's choice, not a route change: the watch goes on if the turn after it returns.
|
|
314
|
+
// Returns the watch now and the turn's event: 'moved', 'escalated' or null.
|
|
315
|
+
export function advanceDown(after, { mode, reason, lateral, escalated, pinned, route }) {
|
|
316
|
+
if (mode !== 'on') return { after: null, event: null };
|
|
317
|
+
if (reason === 'activity-down' && lateral === 'taken') return { after: route, event: 'moved' };
|
|
318
|
+
if (!after || pinned) return { after, event: null };
|
|
319
|
+
if (escalated) return { after: null, event: 'escalated' };
|
|
320
|
+
return { after: route === after ? after : null, event: null };
|
|
321
|
+
}
|
|
322
|
+
|
|
170
323
|
// A run is consecutive turns with the same predicted activity. Given the run so far ({ activity, length } or null)
|
|
171
324
|
// and this turn's predicted activity (null or 'uncertain' end it), returns the run now and the run that just ended.
|
|
172
325
|
export function advanceRun(run, predicted) {
|
|
@@ -175,21 +328,22 @@ export function advanceRun(run, predicted) {
|
|
|
175
328
|
return { run: activity ? { activity, length: 1 } : null, ended: run };
|
|
176
329
|
}
|
|
177
330
|
|
|
178
|
-
const LATERAL = { 'activity-up': 'taken', 'activity-down': 'taken', 'activity-pending': 'refused' };
|
|
179
331
|
const differs = (a, b) => a.model !== b.model || (a.effort ?? null) !== (b.effort ?? null);
|
|
332
|
+
// A decision's lateral outcome: 'taken', 'refused' (not worth it, a hold, the cash gate), or null for none.
|
|
333
|
+
const lateralOf = (decision) =>
|
|
334
|
+
decision?.lateral === 'taken' || decision?.lateral === 'refused' ? decision.lateral : null;
|
|
180
335
|
|
|
181
336
|
// What one routed turn adds to the stats, from its config, the classifier's advice, and the decisions before and
|
|
182
337
|
// after it. `label` is the turn's activity: the applied one in `on`, the accepted answer in `shadow`; `answer` is the
|
|
183
338
|
// accepted answer in both, which the tools are checked against and runs follow. `switched` names
|
|
184
339
|
// what moved the route from the previous turn's: 'activity' for a lateral move, 'tier' for any other, null for none
|
|
185
|
-
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow
|
|
186
|
-
// by a hold or the cash gate shares its reason with tier decisions and is not counted.
|
|
340
|
+
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`.
|
|
187
341
|
export function turnActivity(config, advice, previous, decision) {
|
|
188
342
|
const mode = config.activityRouting;
|
|
189
343
|
const would = decision.wouldRoute ?? null;
|
|
190
344
|
const switched =
|
|
191
345
|
previous?.model && !decision.pinned && differs(previous, decision)
|
|
192
|
-
?
|
|
346
|
+
? lateralOf(decision) === 'taken'
|
|
193
347
|
? 'activity'
|
|
194
348
|
: 'tier'
|
|
195
349
|
: null;
|
|
@@ -201,7 +355,7 @@ export function turnActivity(config, advice, previous, decision) {
|
|
|
201
355
|
label: mode === 'on' ? (decision.activity ?? null) : mode === 'shadow' ? answer : null,
|
|
202
356
|
predicted: advice?.activity?.choice ?? null,
|
|
203
357
|
switched,
|
|
204
|
-
lateral:
|
|
358
|
+
lateral: lateralOf(mode === 'shadow' ? would : mode === 'on' ? decision : null),
|
|
205
359
|
// A shadow turn always counts, so the session and store denominators agree.
|
|
206
360
|
wouldDiffer,
|
|
207
361
|
wouldUsd: wouldDiffer ? (would.difference ?? null) : null,
|
package/lib/display.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
sameRoute,
|
|
10
10
|
} from './config.mjs';
|
|
11
11
|
import { ratesAt } from './cost.mjs';
|
|
12
|
+
import { PROBE_RESULTS } from './probe-results.mjs';
|
|
12
13
|
import { cellForModel, isSameModel } from './route.mjs';
|
|
13
14
|
import { emptyTotals, readout, readSavingsStore } from './savings.mjs';
|
|
14
15
|
|
|
@@ -206,6 +207,25 @@ export function activityRouteText(config, view) {
|
|
|
206
207
|
: `${cell} → ${routeName(config, route)} (override) base: ${routeName(config, base)}`;
|
|
207
208
|
}
|
|
208
209
|
|
|
210
|
+
// Reasons that only say the route stayed: on an override cell they leave out why it is not the tier's base route.
|
|
211
|
+
const STAYED = ['ready', 'same-tier', 'no-advice', 'continuation', 'uncertain'];
|
|
212
|
+
|
|
213
|
+
// The Now tab's Why. In `on`, an activity move, or a stay on an override cell, says the cell plainly: "code at low
|
|
214
|
+
// runs on Sonnet 5.5 · high", or for a move to no activity "low runs on its base route, Haiku 5.5 · high".
|
|
215
|
+
// Every other decision keeps its reason: a refusal, a hold or a tier move says more than the cell.
|
|
216
|
+
export function whyText(config, view) {
|
|
217
|
+
const reason = REASONS[view.reason] ?? view.reason ?? 'ready';
|
|
218
|
+
if (config.activityRouting !== 'on' || !view.tier) return reason;
|
|
219
|
+
const activity = view.activity ?? null;
|
|
220
|
+
const route = resolveRoute(config, view.tier, activity);
|
|
221
|
+
const override = activity !== null && !sameRoute(route, resolveRoute(config, view.tier));
|
|
222
|
+
const moved = view.reason === 'activity-up' || view.reason === 'activity-down';
|
|
223
|
+
if (!moved && !(override && STAYED.includes(view.reason ?? 'ready'))) return reason;
|
|
224
|
+
return activity
|
|
225
|
+
? `${activity} at ${view.tier} runs on ${routeName(config, route)}`
|
|
226
|
+
: `${view.tier} runs on its base route, ${routeName(config, route)}`;
|
|
227
|
+
}
|
|
228
|
+
|
|
209
229
|
// The activity lines of /router without a UI surface: "Activity: ops (81%)" and the resolved route. None in `off`.
|
|
210
230
|
export function activityDetailLines(config, view) {
|
|
211
231
|
if (config.activityRouting === 'off') return [];
|
|
@@ -227,6 +247,19 @@ export function storeAgreementLine(view) {
|
|
|
227
247
|
return `Agreement across sessions: ${all}${code}`;
|
|
228
248
|
}
|
|
229
249
|
|
|
250
|
+
// The classifier's last checked-in activity probe, "activity probe 70/70 · p95 299 ms · Oct 10", or null when none
|
|
251
|
+
// was run on the model it is configured with.
|
|
252
|
+
export function probeText(config, id) {
|
|
253
|
+
const probe = PROBE_RESULTS[id];
|
|
254
|
+
if (!probe || probe.model !== config.classifiers[id]?.model) return null;
|
|
255
|
+
const day = new Date(`${probe.date}T00:00:00Z`).toLocaleDateString('en-US', {
|
|
256
|
+
month: 'short',
|
|
257
|
+
day: 'numeric',
|
|
258
|
+
timeZone: 'UTC',
|
|
259
|
+
});
|
|
260
|
+
return `activity probe ${probe.correct}/${probe.probes} · p95 ${probe.p95Ms} ms · ${day}`;
|
|
261
|
+
}
|
|
262
|
+
|
|
230
263
|
// Routing vs your model: less is green, more is amber, not red: a stronger model is a purchase, not a fault.
|
|
231
264
|
export const SAVINGS_COLOR = { less: '#6fcf97', more: '#f0b429' };
|
|
232
265
|
|