@alexeiled/claude-router 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +21 -0
- package/docs/architecture.md +4 -4
- package/docs/evaluation.md +5 -5
- package/docs/native-router.md +4 -4
- package/docs/router-pane-classifier.svg +39 -34
- package/docs/router-pane-usage.svg +32 -21
- package/docs/user-guide.md +6 -6
- package/hooks/native-router.mjs +113 -20
- package/lib/activity-stats.mjs +164 -10
- package/lib/display.mjs +33 -0
- package/lib/panel.mjs +61 -15
- package/lib/probe-results.mjs +40 -0
- package/lib/route.mjs +26 -8
- package/lib/savings.mjs +1 -0
- package/lib/view.mjs +1 -0
- package/package.json +2 -1
- package/types/index.d.ts +23 -0
package/hooks/native-router.mjs
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
+
advanceDown,
|
|
2
3
|
advanceRun,
|
|
4
|
+
emptyMetrics,
|
|
3
5
|
emptySession,
|
|
4
6
|
emptyStore,
|
|
7
|
+
METRICS_PREFIX,
|
|
8
|
+
mergeMetrics,
|
|
9
|
+
metricsAfterReset,
|
|
10
|
+
readMetrics,
|
|
5
11
|
readStore,
|
|
12
|
+
recordMetrics,
|
|
6
13
|
recordStore,
|
|
7
14
|
recordTurn,
|
|
8
15
|
STORE_KEY,
|
|
@@ -103,6 +110,8 @@ function createRouter(options) {
|
|
|
103
110
|
// one activity, flushed to the store when it ends.
|
|
104
111
|
turnActivities: new Map(),
|
|
105
112
|
run: null,
|
|
113
|
+
// The route a cheaper activity move went to, while the turns stay on it (advanceDown); else null.
|
|
114
|
+
afterDown: null,
|
|
106
115
|
// Each turn's routing-vs-your-model totals until turn.complete adds them to the store.
|
|
107
116
|
turnSavings: new Map(),
|
|
108
117
|
// The totals kept across sessions: this session's record (`{ sessionId, record }`), held here so a refresh never
|
|
@@ -110,6 +119,9 @@ function createRouter(options) {
|
|
|
110
119
|
// refresh. The view shows their sum.
|
|
111
120
|
savingsOwn: null,
|
|
112
121
|
savingsOthers: null,
|
|
122
|
+
// The activity metrics kept across sessions, held the same way: this session's record and the others' sum.
|
|
123
|
+
metricsOwn: null,
|
|
124
|
+
metricsOthers: null,
|
|
113
125
|
// Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
|
|
114
126
|
statsResets: 0,
|
|
115
127
|
};
|
|
@@ -191,6 +203,27 @@ async function otherSavings($, sessionId, resetAt) {
|
|
|
191
203
|
return mergeSavingsStores(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
|
|
192
204
|
}
|
|
193
205
|
|
|
206
|
+
// This session's metrics record and the last reset, as ownSavings: Reset stats clears both with one marker.
|
|
207
|
+
async function ownMetrics($, router) {
|
|
208
|
+
const sessionId = await $.session.id();
|
|
209
|
+
const resetAt = await $.store.get(SAVINGS_RESET_KEY);
|
|
210
|
+
if (router.metricsOwn?.sessionId !== sessionId) {
|
|
211
|
+
const record = readMetrics(await $.store.get(`${METRICS_PREFIX}${sessionId}`));
|
|
212
|
+
if (router.metricsOwn?.sessionId !== sessionId) router.metricsOwn = { sessionId, record };
|
|
213
|
+
}
|
|
214
|
+
router.metricsOwn.record = metricsAfterReset(router.metricsOwn.record, resetAt);
|
|
215
|
+
return { own: router.metricsOwn, resetAt };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Every other session's metrics since the last reset, added up.
|
|
219
|
+
async function otherMetrics($, sessionId, resetAt) {
|
|
220
|
+
const own = `${METRICS_PREFIX}${sessionId}`;
|
|
221
|
+
const keys = (await $.store.keys()).filter((key) => key.startsWith(METRICS_PREFIX) && key !== own);
|
|
222
|
+
return mergeMetrics(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const metricsView = (router) => mergeMetrics([router.metricsOthers, router.metricsOwn?.record]);
|
|
226
|
+
|
|
194
227
|
// The view's totals kept across sessions, from what the router holds; nothing until a read succeeded.
|
|
195
228
|
const savedView = (router) =>
|
|
196
229
|
router.savingsOthers ? { savingsStore: mergeSavingsStores([router.savingsOthers, router.savingsOwn?.record]) } : {};
|
|
@@ -203,6 +236,11 @@ async function storeView($, router) {
|
|
|
203
236
|
try {
|
|
204
237
|
out.activityStore = readStore(await $.store.get(STORE_KEY));
|
|
205
238
|
} catch {}
|
|
239
|
+
try {
|
|
240
|
+
const { own, resetAt } = await ownMetrics($, router);
|
|
241
|
+
router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
|
|
242
|
+
out.activityMetrics = metricsView(router);
|
|
243
|
+
} catch {}
|
|
206
244
|
try {
|
|
207
245
|
const { own, resetAt } = await ownSavings($, router);
|
|
208
246
|
router.savingsOthers = await otherSavings($, own.sessionId, resetAt);
|
|
@@ -220,6 +258,8 @@ async function openPane($, router) {
|
|
|
220
258
|
async function changeMode($, router, mode) {
|
|
221
259
|
const view = router.view ?? (await readView($, router));
|
|
222
260
|
for (const controller of router.controllers) controller.abort();
|
|
261
|
+
// Turns with routing off run on Claude's model, off the cheaper move's route: the escalation watch ends.
|
|
262
|
+
if (mode === 'manual') router.afterDown = null;
|
|
223
263
|
router.modes.set(await $.session.id(), mode);
|
|
224
264
|
const saved = await rememberMode($, mode);
|
|
225
265
|
await updateView($, router, {
|
|
@@ -461,6 +501,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
461
501
|
turns: facts.turns,
|
|
462
502
|
signal: controller.signal,
|
|
463
503
|
});
|
|
504
|
+
// The wait for advice alone, comparable with the activity probe's p95; the routing after it is not the classifier's.
|
|
505
|
+
const adviceMs = pin || !classifierTimed(result.error) ? null : Date.now() - adviceStarted;
|
|
464
506
|
if (controller.signal.aborted || (await $.session.id()) !== sessionId || (await modeOf($, router)) !== 'auto')
|
|
465
507
|
return null;
|
|
466
508
|
const previous = loop.decision;
|
|
@@ -480,7 +522,22 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
480
522
|
if (previous?.model && previous.model !== selected.decision.model && !selected.decision.pinned)
|
|
481
523
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
482
524
|
const activity = turnActivity(cfg, result.advice, previous, selected.decision);
|
|
483
|
-
router.turnActivities.set(e.turnId, {
|
|
525
|
+
router.turnActivities.set(e.turnId, {
|
|
526
|
+
...activity,
|
|
527
|
+
reason: selected.decision.reason,
|
|
528
|
+
escalated: Boolean(selected.decision.escalated),
|
|
529
|
+
pinned: selected.decision.pinned,
|
|
530
|
+
route: `${selected.decision.model}@${selected.decision.effort ?? 'session'}`,
|
|
531
|
+
classifier: cfg.classifier,
|
|
532
|
+
adviceMs,
|
|
533
|
+
sessionId,
|
|
534
|
+
requests: 0,
|
|
535
|
+
inputTokens: 0,
|
|
536
|
+
outputTokens: 0,
|
|
537
|
+
routedUsd: null,
|
|
538
|
+
pricedRequests: 0,
|
|
539
|
+
routes: {},
|
|
540
|
+
});
|
|
484
541
|
await updateView($, router, {
|
|
485
542
|
phase: 'routed',
|
|
486
543
|
selectedModel: selected.decision.model,
|
|
@@ -498,7 +555,7 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
498
555
|
? {
|
|
499
556
|
error: result.error,
|
|
500
557
|
health: healthOf(clientOf(router, cfg.classifier), cfg.classifier),
|
|
501
|
-
adviceMs
|
|
558
|
+
adviceMs,
|
|
502
559
|
adviceChoice: result.advice?.choice ?? null,
|
|
503
560
|
probabilities: result.advice?.probabilities ?? null,
|
|
504
561
|
estimate: selected.decision.estimate ?? null,
|
|
@@ -517,8 +574,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
517
574
|
}
|
|
518
575
|
|
|
519
576
|
// A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
|
|
520
|
-
// turn's activity record.
|
|
521
|
-
async function publishReply($, router, turnId, response, tier, route) {
|
|
577
|
+
// turn's activity record with its list price (`routedUsd`, null when unpriced) and, when its tier served it, its route.
|
|
578
|
+
async function publishReply($, router, turnId, response, tier, route, routedUsd) {
|
|
522
579
|
const turn = router.turnActivities.get(turnId);
|
|
523
580
|
const metrics = responseMetrics(
|
|
524
581
|
router.view,
|
|
@@ -534,6 +591,9 @@ async function publishReply($, router, turnId, response, tier, route) {
|
|
|
534
591
|
requests: turn.requests + 1,
|
|
535
592
|
inputTokens: turn.inputTokens + (metrics.inputTokens ?? 0),
|
|
536
593
|
outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
|
|
594
|
+
routedUsd: Number.isFinite(routedUsd) ? (turn.routedUsd ?? 0) + routedUsd : turn.routedUsd,
|
|
595
|
+
pricedRequests: turn.pricedRequests + (Number.isFinite(routedUsd) ? 1 : 0),
|
|
596
|
+
routes: tier ? { ...turn.routes, [route]: (turn.routes[route] ?? 0) + 1 } : turn.routes,
|
|
537
597
|
});
|
|
538
598
|
}
|
|
539
599
|
|
|
@@ -554,7 +614,8 @@ async function observeRouted($, router, { turnId, cfg, loop, loopVersion, reques
|
|
|
554
614
|
if (!written.isSet) return;
|
|
555
615
|
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
556
616
|
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
557
|
-
|
|
617
|
+
const route = `${request.model}@${request.effort ?? 'session'}`;
|
|
618
|
+
await publishReply($, router, turnId, response, tier, route, compared?.reply?.routedUsd ?? null);
|
|
558
619
|
if (compared?.reply) await addSavings($, router, turnId, compared.reply, tier);
|
|
559
620
|
}
|
|
560
621
|
|
|
@@ -623,7 +684,8 @@ async function flushSavings($, router, totals) {
|
|
|
623
684
|
}
|
|
624
685
|
|
|
625
686
|
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
626
|
-
// break a turn: a failed read or write loses this turn's counts
|
|
687
|
+
// break a turn: a failed read or write loses this turn's counts. Readings already in the session's metrics record stay
|
|
688
|
+
// in memory after a failed write and go out with the next turn that adds a reading.
|
|
627
689
|
async function recordActivity($, router, turn) {
|
|
628
690
|
try {
|
|
629
691
|
if ((await $.session.id()) !== turn.sessionId) return;
|
|
@@ -637,6 +699,8 @@ async function recordActivity($, router, turn) {
|
|
|
637
699
|
// Advanced before any further await, so a session change meanwhile cannot carry this run into the new session.
|
|
638
700
|
const { run, ended } = advanceRun(router.run, turn.answer);
|
|
639
701
|
router.run = run;
|
|
702
|
+
const down = advanceDown(router.afterDown, turn);
|
|
703
|
+
router.afterDown = down.after;
|
|
640
704
|
await updateView($, router, {
|
|
641
705
|
activityStats: recordTurn(view.activityStats ?? emptySession(), {
|
|
642
706
|
activity: turn.label,
|
|
@@ -645,8 +709,12 @@ async function recordActivity($, router, turn) {
|
|
|
645
709
|
requests: turn.requests,
|
|
646
710
|
inputTokens: turn.inputTokens,
|
|
647
711
|
outputTokens: turn.outputTokens,
|
|
712
|
+
routedUsd: turn.routedUsd,
|
|
713
|
+
pricedRequests: turn.pricedRequests,
|
|
714
|
+
routes: turn.routes,
|
|
648
715
|
tierSwitches: turn.switched === 'tier' ? 1 : 0,
|
|
649
716
|
activitySwitches: turn.switched === 'activity' ? 1 : 0,
|
|
717
|
+
lateral: turn.lateral,
|
|
650
718
|
wouldDiffer: turn.wouldDiffer,
|
|
651
719
|
wouldUsd: turn.wouldUsd,
|
|
652
720
|
shadow: turn.mode === 'shadow',
|
|
@@ -654,6 +722,10 @@ async function recordActivity($, router, turn) {
|
|
|
654
722
|
});
|
|
655
723
|
const resets = router.statsResets;
|
|
656
724
|
const store = readStore(await $.store.get(STORE_KEY));
|
|
725
|
+
const { own, resetAt } = await ownMetrics($, router);
|
|
726
|
+
// The others held from before a reset in another session are read again.
|
|
727
|
+
if (!router.metricsOthers || metricsAfterReset(router.metricsOthers, resetAt) !== router.metricsOthers)
|
|
728
|
+
router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
|
|
657
729
|
// Writing counts read before a reset would bring them back; this turn's are lost instead.
|
|
658
730
|
if (router.statsResets !== resets) return;
|
|
659
731
|
const written = recordStore(store, {
|
|
@@ -664,9 +736,23 @@ async function recordActivity($, router, turn) {
|
|
|
664
736
|
wouldDiffer: turn.wouldDiffer,
|
|
665
737
|
wouldUsd: turn.wouldUsd,
|
|
666
738
|
});
|
|
667
|
-
|
|
739
|
+
const measured = recordMetrics(own.record, {
|
|
740
|
+
classifier: turn.classifier,
|
|
741
|
+
adviceMs: turn.adviceMs,
|
|
742
|
+
downMove: down.event,
|
|
743
|
+
now: Date.now(),
|
|
744
|
+
});
|
|
745
|
+
// Added to in memory before the write, so a refresh during the write counts the turn once.
|
|
746
|
+
const changed = measured !== own.record;
|
|
747
|
+
own.record = measured;
|
|
748
|
+
// Both writes start before any reset can run between them.
|
|
749
|
+
await Promise.all([
|
|
750
|
+
$.store.set(STORE_KEY, written),
|
|
751
|
+
changed ? $.store.set(`${METRICS_PREFIX}${own.sessionId}`, measured) : null,
|
|
752
|
+
]);
|
|
668
753
|
// A reset during the write emptied the view; these counts are older than it.
|
|
669
|
-
if (router.statsResets === resets)
|
|
754
|
+
if (router.statsResets === resets)
|
|
755
|
+
await updateView($, router, { activityStore: written, activityMetrics: metricsView(router) });
|
|
670
756
|
} catch {}
|
|
671
757
|
}
|
|
672
758
|
|
|
@@ -832,33 +918,36 @@ function paneActions($, router, view) {
|
|
|
832
918
|
resetStats: async () => {
|
|
833
919
|
router.statsResets += 1;
|
|
834
920
|
router.run = null;
|
|
921
|
+
router.afterDown = null;
|
|
835
922
|
router.turnSavings.clear();
|
|
836
923
|
const clear = (key, value) =>
|
|
837
924
|
$.store
|
|
838
925
|
.set(key, value)
|
|
839
926
|
.then(() => true)
|
|
840
927
|
.catch(() => false);
|
|
841
|
-
|
|
928
|
+
// The per-session records: savings and activity metrics.
|
|
929
|
+
const clearRecords = async () => {
|
|
842
930
|
// Written first, so a session still holding its record from before the reset drops it, unless that record's
|
|
843
|
-
// first
|
|
931
|
+
// first reading has the reset's millisecond or the clock moved back across the reset.
|
|
844
932
|
await $.store.set(SAVINGS_RESET_KEY, Date.now());
|
|
845
|
-
const keys = (await $.store.keys()).filter(
|
|
933
|
+
const keys = (await $.store.keys()).filter(
|
|
934
|
+
(key) => key.startsWith(SAVINGS_PREFIX) || key.startsWith(METRICS_PREFIX),
|
|
935
|
+
);
|
|
846
936
|
await Promise.all(keys.map((key) => $.store.delete(key)));
|
|
847
937
|
router.savingsOwn = null;
|
|
848
938
|
router.savingsOthers = emptySavingsStore();
|
|
939
|
+
router.metricsOwn = null;
|
|
940
|
+
router.metricsOthers = emptyMetrics();
|
|
849
941
|
return true;
|
|
850
942
|
};
|
|
851
|
-
const [
|
|
852
|
-
clear(STORE_KEY, emptyStore()),
|
|
853
|
-
clearSavings().catch(() => false),
|
|
854
|
-
]);
|
|
943
|
+
const [stats, records] = await Promise.all([clear(STORE_KEY, emptyStore()), clearRecords().catch(() => false)]);
|
|
855
944
|
await updateView($, router, {
|
|
856
945
|
activityStats: null,
|
|
857
946
|
savings: null,
|
|
858
|
-
...(
|
|
859
|
-
...(
|
|
947
|
+
...(stats ? { activityStore: emptyStore() } : {}),
|
|
948
|
+
...(records ? { activityMetrics: emptyMetrics(), savingsStore: emptySavingsStore() } : {}),
|
|
860
949
|
notice:
|
|
861
|
-
|
|
950
|
+
stats && records
|
|
862
951
|
? 'Stats reset: activity and routing vs your model, this session and saved.'
|
|
863
952
|
: 'This session’s stats reset. Saved stats could not be cleared.',
|
|
864
953
|
});
|
|
@@ -1076,10 +1165,12 @@ export function register(on, options) {
|
|
|
1076
1165
|
}
|
|
1077
1166
|
router.prompts.delete(e.turnId);
|
|
1078
1167
|
for (const key of router.decisions.keys()) if (key.includes(`:${e.turnId}:`)) router.decisions.delete(key);
|
|
1079
|
-
// `off` asks nothing, so it records nothing.
|
|
1168
|
+
// `off` asks nothing, so it records nothing; it only ends the watch after a cheaper activity move.
|
|
1080
1169
|
const turn = router.turnActivities.get(e.turnId);
|
|
1081
1170
|
router.turnActivities.delete(e.turnId);
|
|
1082
|
-
if (turn
|
|
1171
|
+
if (turn?.mode === 'off') {
|
|
1172
|
+
if ((await $.session.id()) === turn.sessionId) router.afterDown = advanceDown(router.afterDown, turn).after;
|
|
1173
|
+
} else if (turn) await recordActivity($, router, turn);
|
|
1083
1174
|
const savings = router.turnSavings.get(e.turnId);
|
|
1084
1175
|
router.turnSavings.delete(e.turnId);
|
|
1085
1176
|
if (savings) await flushSavings($, router, savings);
|
|
@@ -1107,6 +1198,7 @@ export function register(on, options) {
|
|
|
1107
1198
|
router.turnSavings.clear();
|
|
1108
1199
|
const run = router.run;
|
|
1109
1200
|
router.run = null;
|
|
1201
|
+
router.afterDown = null;
|
|
1110
1202
|
if (run) await flushRun($, run);
|
|
1111
1203
|
router.view = {
|
|
1112
1204
|
...initialView(router.view?.nativeModel ?? ''),
|
|
@@ -1116,6 +1208,7 @@ export function register(on, options) {
|
|
|
1116
1208
|
credentials: router.view?.credentials ?? null,
|
|
1117
1209
|
// Counts kept across sessions: the next session starts from them.
|
|
1118
1210
|
activityStore: router.view?.activityStore ?? null,
|
|
1211
|
+
activityMetrics: router.view?.activityMetrics ?? null,
|
|
1119
1212
|
savingsStore: router.view?.savingsStore ?? null,
|
|
1120
1213
|
health: healthOf(clientOf(router, router.config.classifier), router.config.classifier),
|
|
1121
1214
|
configPath: router.config.nativePath,
|
package/lib/activity-stats.mjs
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store
|
|
2
|
-
// counts kept across sessions under `activity:stats:v1
|
|
1
|
+
// Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducers feed the
|
|
2
|
+
// counts kept across sessions under `activity:stats:v1` and, per session, under METRICS_PREFIX. Pure: state and a turn
|
|
3
|
+
// in, new state out.
|
|
3
4
|
|
|
4
5
|
import { acceptActivity } from './activity.mjs';
|
|
5
6
|
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
6
7
|
|
|
7
8
|
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
8
9
|
export const STORE_KEY = 'activity:stats:v1';
|
|
10
|
+
// Each session keeps its own metrics under this prefix and its session id, as savings does (SAVINGS_PREFIX): sessions
|
|
11
|
+
// finishing turns at the same moment do not overwrite each other. The readout is their sum (mergeMetrics).
|
|
12
|
+
export const METRICS_PREFIX = 'activity:metrics:v1:';
|
|
9
13
|
|
|
10
14
|
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
11
15
|
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
@@ -46,18 +50,35 @@ function addShadow(shadow, wouldDiffer, range) {
|
|
|
46
50
|
};
|
|
47
51
|
}
|
|
48
52
|
|
|
53
|
+
function addRoutes(before, added) {
|
|
54
|
+
const next = { ...before };
|
|
55
|
+
for (const [route, n] of Object.entries(added ?? {})) if (count(n)) next[route] = (next[route] ?? 0) + count(n);
|
|
56
|
+
return next;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// The route most of an activity's routed replies went out with, `model@effort`; ties go to the first in name order,
|
|
60
|
+
// so the readout is stable. Null without routed replies.
|
|
61
|
+
export function mostlyOn(counts) {
|
|
62
|
+
const [top] = Object.entries(counts?.routes ?? {}).sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
|
|
63
|
+
return top?.[0] ?? null;
|
|
64
|
+
}
|
|
65
|
+
|
|
49
66
|
export function emptySession() {
|
|
50
67
|
return {
|
|
51
68
|
byActivity: {},
|
|
52
69
|
switches: { tier: 0, activity: 0 },
|
|
70
|
+
lateral: { taken: 0, refused: 0 },
|
|
53
71
|
agreement: { matched: 0, total: 0 },
|
|
54
72
|
shadow: emptyShadow(),
|
|
55
73
|
};
|
|
56
74
|
}
|
|
57
75
|
|
|
58
76
|
// One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
|
|
59
|
-
// agreement compares with the tools (null: not counted), `
|
|
60
|
-
//
|
|
77
|
+
// agreement compares with the tools (null: not counted), `lateral` the turn's lateral outcome (turnActivity, counted
|
|
78
|
+
// outside shadow only), `shadow` marks a turn in shadow mode, `wouldDiffer` says whether 'on' would have routed it
|
|
79
|
+
// differently (null: no would-route) and `wouldUsd` prices that difference. `routedUsd` is the list price of the
|
|
80
|
+
// turn's priced replies (null: none had a price), `pricedRequests` how many of its replies had one, and `routes` counts
|
|
81
|
+
// its routed replies by the `model@effort` they went out with.
|
|
61
82
|
export function recordTurn(
|
|
62
83
|
session,
|
|
63
84
|
{
|
|
@@ -67,8 +88,12 @@ export function recordTurn(
|
|
|
67
88
|
requests,
|
|
68
89
|
inputTokens,
|
|
69
90
|
outputTokens,
|
|
91
|
+
routedUsd = null,
|
|
92
|
+
pricedRequests = 0,
|
|
93
|
+
routes = {},
|
|
70
94
|
tierSwitches,
|
|
71
95
|
activitySwitches,
|
|
96
|
+
lateral,
|
|
72
97
|
wouldDiffer,
|
|
73
98
|
wouldUsd,
|
|
74
99
|
shadow,
|
|
@@ -76,6 +101,8 @@ export function recordTurn(
|
|
|
76
101
|
) {
|
|
77
102
|
const key = activity ?? 'none';
|
|
78
103
|
const before = session.byActivity[key] ?? { turns: 0, requests: 0, inputTokens: 0, outputTokens: 0 };
|
|
104
|
+
// A view saved by 1.7.0 has no lateral counts.
|
|
105
|
+
const moves = { taken: 0, refused: 0, ...session.lateral };
|
|
79
106
|
return {
|
|
80
107
|
byActivity: {
|
|
81
108
|
...session.byActivity,
|
|
@@ -84,12 +111,19 @@ export function recordTurn(
|
|
|
84
111
|
requests: before.requests + count(requests),
|
|
85
112
|
inputTokens: before.inputTokens + count(inputTokens),
|
|
86
113
|
outputTokens: before.outputTokens + count(outputTokens),
|
|
114
|
+
// Counts saved by 1.7.0 have no cost, priced replies or routes: their replies stay unpriced.
|
|
115
|
+
routedUsd: Number.isFinite(routedUsd) ? (before.routedUsd ?? 0) + routedUsd : (before.routedUsd ?? null),
|
|
116
|
+
pricedRequests: (before.pricedRequests ?? 0) + count(pricedRequests),
|
|
117
|
+
routes: addRoutes(before.routes ?? {}, routes),
|
|
87
118
|
},
|
|
88
119
|
},
|
|
89
120
|
switches: {
|
|
90
121
|
tier: session.switches.tier + count(tierSwitches),
|
|
91
122
|
activity: session.switches.activity + count(activitySwitches),
|
|
92
123
|
},
|
|
124
|
+
// In shadow the lateral outcome is the would-route's, which no route followed; the Shadow line covers it.
|
|
125
|
+
lateral:
|
|
126
|
+
!shadow && (lateral === 'taken' || lateral === 'refused') ? { ...moves, [lateral]: moves[lateral] + 1 } : moves,
|
|
93
127
|
agreement: answer
|
|
94
128
|
? {
|
|
95
129
|
matched: session.agreement.matched + (agrees(answer, observed) ? 1 : 0),
|
|
@@ -103,10 +137,33 @@ export function recordTurn(
|
|
|
103
137
|
};
|
|
104
138
|
}
|
|
105
139
|
|
|
140
|
+
// Classifier wait per turn, bucketed by these upper bounds in ms, plus one bucket above the last; kept per classifier
|
|
141
|
+
// id for at most LATENCY_CLASSIFIERS ids, so the store stays bounded.
|
|
142
|
+
export const LATENCY_BOUNDS = [
|
|
143
|
+
100, 150, 200, 250, 300, 350, 400, 450, 500, 600, 700, 800, 900, 1000, 1250, 1500, 2000, 2500, 3000, 4000, 5000,
|
|
144
|
+
];
|
|
145
|
+
const LATENCY_BUCKETS = LATENCY_BOUNDS.length + 1;
|
|
146
|
+
const LATENCY_CLASSIFIERS = 8;
|
|
147
|
+
|
|
106
148
|
export function emptyStore() {
|
|
107
149
|
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: emptyShadow() };
|
|
108
150
|
}
|
|
109
151
|
|
|
152
|
+
// The p95 of a latency histogram: the upper bound of its bucket in ms, `over` when it lies above the last bound, and
|
|
153
|
+
// the turns counted. Null without turns.
|
|
154
|
+
export function latencyP95(buckets) {
|
|
155
|
+
const turns = (buckets ?? []).reduce((sum, n) => sum + n, 0);
|
|
156
|
+
if (!turns) return null;
|
|
157
|
+
let seen = 0;
|
|
158
|
+
const at = buckets.findIndex((n) => {
|
|
159
|
+
seen += n;
|
|
160
|
+
return seen >= 0.95 * turns;
|
|
161
|
+
});
|
|
162
|
+
return at < LATENCY_BOUNDS.length
|
|
163
|
+
? { ms: LATENCY_BOUNDS[at], over: false, turns }
|
|
164
|
+
: { ms: LATENCY_BOUNDS.at(-1), over: true, turns };
|
|
165
|
+
}
|
|
166
|
+
|
|
110
167
|
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
111
168
|
const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
112
169
|
const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
|
|
@@ -127,7 +184,8 @@ const isShadow = (v) =>
|
|
|
127
184
|
Number.isFinite(v.maxUsd));
|
|
128
185
|
|
|
129
186
|
// The stored value if it has a known shape, filled to the current one, else an empty store: a hand-edited or
|
|
130
|
-
// unknown file must not break stats.
|
|
187
|
+
// unknown file must not break stats. The shape is 1.7.0's, which rejects any other key: a new count goes under its own
|
|
188
|
+
// key, such as METRICS_KEY, so a downgrade keeps these.
|
|
131
189
|
export function readStore(value) {
|
|
132
190
|
const ok =
|
|
133
191
|
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
@@ -167,6 +225,101 @@ export function recordStore(store, { predicted, observed, runEnded, lateral, wou
|
|
|
167
225
|
return next;
|
|
168
226
|
}
|
|
169
227
|
|
|
228
|
+
// The readings kept across sessions beside the stats store, one session's or their sum: the classifier wait per
|
|
229
|
+
// classifier id, and the cheaper activity moves taken in `on` with the escalations that followed one, since `since`
|
|
230
|
+
// (ms), the first reading after a reset.
|
|
231
|
+
export function emptyMetrics() {
|
|
232
|
+
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 } };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const isLatency = (v) =>
|
|
236
|
+
isRecord(v) &&
|
|
237
|
+
Object.keys(v).length <= LATENCY_CLASSIFIERS &&
|
|
238
|
+
Object.values(v).every((b) => Array.isArray(b) && b.length === LATENCY_BUCKETS && b.every(isCount));
|
|
239
|
+
|
|
240
|
+
// Each part read on its own: a bad or outdated one, such as latency after LATENCY_BOUNDS changes, reads as empty and
|
|
241
|
+
// leaves the others.
|
|
242
|
+
export function readMetrics(value) {
|
|
243
|
+
const empty = emptyMetrics();
|
|
244
|
+
if (!isRecord(value) || value.version !== 1) return empty;
|
|
245
|
+
return {
|
|
246
|
+
version: 1,
|
|
247
|
+
since: Number.isFinite(value.since) ? value.since : null,
|
|
248
|
+
latency: isLatency(value.latency) ? value.latency : empty.latency,
|
|
249
|
+
downMoves: exactCountsOf(value.downMoves, ['moves', 'escalations']) ? value.downMoves : empty.downMoves,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed), and `downMove` 'moved' or
|
|
254
|
+
// 'escalated' (advanceDown). A turn that adds nothing returns `metrics` itself.
|
|
255
|
+
export function recordMetrics(metrics, { classifier = null, adviceMs = null, downMove = null, now = null }) {
|
|
256
|
+
const next = {
|
|
257
|
+
version: 1,
|
|
258
|
+
since: metrics.since ?? now,
|
|
259
|
+
latency: { ...metrics.latency },
|
|
260
|
+
downMoves: { ...metrics.downMoves },
|
|
261
|
+
};
|
|
262
|
+
let counted = downMove === 'moved' || downMove === 'escalated';
|
|
263
|
+
const known = Object.hasOwn(next.latency, classifier);
|
|
264
|
+
if (
|
|
265
|
+
typeof classifier === 'string' &&
|
|
266
|
+
Number.isFinite(adviceMs) &&
|
|
267
|
+
adviceMs >= 0 &&
|
|
268
|
+
(known || Object.keys(next.latency).length < LATENCY_CLASSIFIERS)
|
|
269
|
+
) {
|
|
270
|
+
const buckets = known ? [...next.latency[classifier]] : Array(LATENCY_BUCKETS).fill(0);
|
|
271
|
+
const at = LATENCY_BOUNDS.findIndex((bound) => adviceMs <= bound);
|
|
272
|
+
buckets[at === -1 ? LATENCY_BOUNDS.length : at] += 1;
|
|
273
|
+
next.latency[classifier] = buckets;
|
|
274
|
+
counted = true;
|
|
275
|
+
}
|
|
276
|
+
if (!counted) return metrics;
|
|
277
|
+
if (downMove === 'moved') next.downMoves.moves += 1;
|
|
278
|
+
if (downMove === 'escalated') next.downMoves.escalations += 1;
|
|
279
|
+
return next;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// A record whose first reading came before the last reset at `resetAt` (ms) reads as empty, as savings' afterReset.
|
|
283
|
+
export function metricsAfterReset(record, resetAt) {
|
|
284
|
+
return Number.isFinite(resetAt) && record.since !== null && record.since < resetAt ? emptyMetrics() : record;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// The metrics kept across sessions: every session's record since the last reset added up, since the earliest. Latency
|
|
288
|
+
// keeps the first LATENCY_CLASSIFIERS classifier ids in the order given, so the sum still reads back (readMetrics).
|
|
289
|
+
export function mergeMetrics(values, resetAt = null) {
|
|
290
|
+
const merged = emptyMetrics();
|
|
291
|
+
for (const one of values.map((value) => metricsAfterReset(readMetrics(value), resetAt))) {
|
|
292
|
+
const since = [merged.since, one.since].filter(Number.isFinite);
|
|
293
|
+
merged.since = since.length ? Math.min(...since) : null;
|
|
294
|
+
for (const [id, buckets] of Object.entries(one.latency)) {
|
|
295
|
+
// Own keys only: a classifier id such as `toString` must not read Object.prototype.
|
|
296
|
+
const known = Object.hasOwn(merged.latency, id) ? merged.latency[id] : null;
|
|
297
|
+
if (known) merged.latency[id] = known.map((n, i) => n + buckets[i]);
|
|
298
|
+
else if (Object.keys(merged.latency).length < LATENCY_CLASSIFIERS) merged.latency[id] = [...buckets];
|
|
299
|
+
}
|
|
300
|
+
merged.downMoves = {
|
|
301
|
+
moves: merged.downMoves.moves + one.downMoves.moves,
|
|
302
|
+
escalations: merged.downMoves.escalations + one.downMoves.escalations,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
return merged;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Escalations after a cheaper activity move. `after` is the `model@effort` route a cheaper move went to while the
|
|
309
|
+
// watch lasts, else null; `turn` is the turn's mode, decision reason and lateral outcome, whether a failure asked for
|
|
310
|
+
// an escalation (also one the cash gate held or context-fit replaced), whether it was pinned, and its route. A taken
|
|
311
|
+
// cheaper move in `on` starts the watch; an escalation while it lasts counts once and ends it, as does a turn on
|
|
312
|
+
// another route or a turn in another mode (an `off` turn too, which records nothing else); the hook also ends it when
|
|
313
|
+
// routing is turned off. A pin is the person's choice, not a route change: the watch goes on if the turn after it returns.
|
|
314
|
+
// Returns the watch now and the turn's event: 'moved', 'escalated' or null.
|
|
315
|
+
export function advanceDown(after, { mode, reason, lateral, escalated, pinned, route }) {
|
|
316
|
+
if (mode !== 'on') return { after: null, event: null };
|
|
317
|
+
if (reason === 'activity-down' && lateral === 'taken') return { after: route, event: 'moved' };
|
|
318
|
+
if (!after || pinned) return { after, event: null };
|
|
319
|
+
if (escalated) return { after: null, event: 'escalated' };
|
|
320
|
+
return { after: route === after ? after : null, event: null };
|
|
321
|
+
}
|
|
322
|
+
|
|
170
323
|
// A run is consecutive turns with the same predicted activity. Given the run so far ({ activity, length } or null)
|
|
171
324
|
// and this turn's predicted activity (null or 'uncertain' end it), returns the run now and the run that just ended.
|
|
172
325
|
export function advanceRun(run, predicted) {
|
|
@@ -175,21 +328,22 @@ export function advanceRun(run, predicted) {
|
|
|
175
328
|
return { run: activity ? { activity, length: 1 } : null, ended: run };
|
|
176
329
|
}
|
|
177
330
|
|
|
178
|
-
const LATERAL = { 'activity-up': 'taken', 'activity-down': 'taken', 'activity-pending': 'refused' };
|
|
179
331
|
const differs = (a, b) => a.model !== b.model || (a.effort ?? null) !== (b.effort ?? null);
|
|
332
|
+
// A decision's lateral outcome: 'taken', 'refused' (not worth it, a hold, the cash gate), or null for none.
|
|
333
|
+
const lateralOf = (decision) =>
|
|
334
|
+
decision?.lateral === 'taken' || decision?.lateral === 'refused' ? decision.lateral : null;
|
|
180
335
|
|
|
181
336
|
// What one routed turn adds to the stats, from its config, the classifier's advice, and the decisions before and
|
|
182
337
|
// after it. `label` is the turn's activity: the applied one in `on`, the accepted answer in `shadow`; `answer` is the
|
|
183
338
|
// accepted answer in both, which the tools are checked against and runs follow. `switched` names
|
|
184
339
|
// what moved the route from the previous turn's: 'activity' for a lateral move, 'tier' for any other, null for none
|
|
185
|
-
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow
|
|
186
|
-
// by a hold or the cash gate shares its reason with tier decisions and is not counted.
|
|
340
|
+
// or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`.
|
|
187
341
|
export function turnActivity(config, advice, previous, decision) {
|
|
188
342
|
const mode = config.activityRouting;
|
|
189
343
|
const would = decision.wouldRoute ?? null;
|
|
190
344
|
const switched =
|
|
191
345
|
previous?.model && !decision.pinned && differs(previous, decision)
|
|
192
|
-
?
|
|
346
|
+
? lateralOf(decision) === 'taken'
|
|
193
347
|
? 'activity'
|
|
194
348
|
: 'tier'
|
|
195
349
|
: null;
|
|
@@ -201,7 +355,7 @@ export function turnActivity(config, advice, previous, decision) {
|
|
|
201
355
|
label: mode === 'on' ? (decision.activity ?? null) : mode === 'shadow' ? answer : null,
|
|
202
356
|
predicted: advice?.activity?.choice ?? null,
|
|
203
357
|
switched,
|
|
204
|
-
lateral:
|
|
358
|
+
lateral: lateralOf(mode === 'shadow' ? would : mode === 'on' ? decision : null),
|
|
205
359
|
// A shadow turn always counts, so the session and store denominators agree.
|
|
206
360
|
wouldDiffer,
|
|
207
361
|
wouldUsd: wouldDiffer ? (would.difference ?? null) : null,
|
package/lib/display.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
sameRoute,
|
|
10
10
|
} from './config.mjs';
|
|
11
11
|
import { ratesAt } from './cost.mjs';
|
|
12
|
+
import { PROBE_RESULTS } from './probe-results.mjs';
|
|
12
13
|
import { cellForModel, isSameModel } from './route.mjs';
|
|
13
14
|
import { emptyTotals, readout, readSavingsStore } from './savings.mjs';
|
|
14
15
|
|
|
@@ -206,6 +207,25 @@ export function activityRouteText(config, view) {
|
|
|
206
207
|
: `${cell} → ${routeName(config, route)} (override) base: ${routeName(config, base)}`;
|
|
207
208
|
}
|
|
208
209
|
|
|
210
|
+
// Reasons that only say the route stayed: on an override cell they leave out why it is not the tier's base route.
|
|
211
|
+
const STAYED = ['ready', 'same-tier', 'no-advice', 'continuation', 'uncertain'];
|
|
212
|
+
|
|
213
|
+
// The Now tab's Why. In `on`, an activity move, or a stay on an override cell, says the cell plainly: "code at low
|
|
214
|
+
// runs on Sonnet 5.5 · high", or for a move to no activity "low runs on its base route, Haiku 5.5 · high".
|
|
215
|
+
// Every other decision keeps its reason: a refusal, a hold or a tier move says more than the cell.
|
|
216
|
+
export function whyText(config, view) {
|
|
217
|
+
const reason = REASONS[view.reason] ?? view.reason ?? 'ready';
|
|
218
|
+
if (config.activityRouting !== 'on' || !view.tier) return reason;
|
|
219
|
+
const activity = view.activity ?? null;
|
|
220
|
+
const route = resolveRoute(config, view.tier, activity);
|
|
221
|
+
const override = activity !== null && !sameRoute(route, resolveRoute(config, view.tier));
|
|
222
|
+
const moved = view.reason === 'activity-up' || view.reason === 'activity-down';
|
|
223
|
+
if (!moved && !(override && STAYED.includes(view.reason ?? 'ready'))) return reason;
|
|
224
|
+
return activity
|
|
225
|
+
? `${activity} at ${view.tier} runs on ${routeName(config, route)}`
|
|
226
|
+
: `${view.tier} runs on its base route, ${routeName(config, route)}`;
|
|
227
|
+
}
|
|
228
|
+
|
|
209
229
|
// The activity lines of /router without a UI surface: "Activity: ops (81%)" and the resolved route. None in `off`.
|
|
210
230
|
export function activityDetailLines(config, view) {
|
|
211
231
|
if (config.activityRouting === 'off') return [];
|
|
@@ -227,6 +247,19 @@ export function storeAgreementLine(view) {
|
|
|
227
247
|
return `Agreement across sessions: ${all}${code}`;
|
|
228
248
|
}
|
|
229
249
|
|
|
250
|
+
// The classifier's last checked-in activity probe, "activity probe 70/70 · p95 299 ms · Oct 10", or null when none
|
|
251
|
+
// was run on the model it is configured with.
|
|
252
|
+
export function probeText(config, id) {
|
|
253
|
+
const probe = PROBE_RESULTS[id];
|
|
254
|
+
if (!probe || probe.model !== config.classifiers[id]?.model) return null;
|
|
255
|
+
const day = new Date(`${probe.date}T00:00:00Z`).toLocaleDateString('en-US', {
|
|
256
|
+
month: 'short',
|
|
257
|
+
day: 'numeric',
|
|
258
|
+
timeZone: 'UTC',
|
|
259
|
+
});
|
|
260
|
+
return `activity probe ${probe.correct}/${probe.probes} · p95 ${probe.p95Ms} ms · ${day}`;
|
|
261
|
+
}
|
|
262
|
+
|
|
230
263
|
// Routing vs your model: less is green, more is amber, not red: a stronger model is a purchase, not a fault.
|
|
231
264
|
export const SAVINGS_COLOR = { less: '#6fcf97', more: '#f0b429' };
|
|
232
265
|
|