@alexeiled/claude-router 1.7.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,15 @@
1
1
  import {
2
+ advanceDown,
2
3
  advanceRun,
4
+ emptyMetrics,
3
5
  emptySession,
4
6
  emptyStore,
7
+ METRICS_PREFIX,
8
+ mergeMetrics,
9
+ metricsAfterReset,
10
+ readMetrics,
5
11
  readStore,
12
+ recordMetrics,
6
13
  recordStore,
7
14
  recordTurn,
8
15
  STORE_KEY,
@@ -103,6 +110,8 @@ function createRouter(options) {
103
110
  // one activity, flushed to the store when it ends.
104
111
  turnActivities: new Map(),
105
112
  run: null,
113
+ // The route a cheaper activity move went to, while the turns stay on it (advanceDown); else null.
114
+ afterDown: null,
106
115
  // Each turn's routing-vs-your-model totals until turn.complete adds them to the store.
107
116
  turnSavings: new Map(),
108
117
  // The totals kept across sessions: this session's record (`{ sessionId, record }`), held here so a refresh never
@@ -110,6 +119,9 @@ function createRouter(options) {
110
119
  // refresh. The view shows their sum.
111
120
  savingsOwn: null,
112
121
  savingsOthers: null,
122
+ // The activity metrics kept across sessions, held the same way: this session's record and the others' sum.
123
+ metricsOwn: null,
124
+ metricsOthers: null,
113
125
  // Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
114
126
  statsResets: 0,
115
127
  };
@@ -191,6 +203,27 @@ async function otherSavings($, sessionId, resetAt) {
191
203
  return mergeSavingsStores(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
192
204
  }
193
205
 
206
+ // This session's metrics record and the last reset, as ownSavings: Reset stats clears both with one marker.
207
+ async function ownMetrics($, router) {
208
+ const sessionId = await $.session.id();
209
+ const resetAt = await $.store.get(SAVINGS_RESET_KEY);
210
+ if (router.metricsOwn?.sessionId !== sessionId) {
211
+ const record = readMetrics(await $.store.get(`${METRICS_PREFIX}${sessionId}`));
212
+ if (router.metricsOwn?.sessionId !== sessionId) router.metricsOwn = { sessionId, record };
213
+ }
214
+ router.metricsOwn.record = metricsAfterReset(router.metricsOwn.record, resetAt);
215
+ return { own: router.metricsOwn, resetAt };
216
+ }
217
+
218
+ // Every other session's metrics since the last reset, added up.
219
+ async function otherMetrics($, sessionId, resetAt) {
220
+ const own = `${METRICS_PREFIX}${sessionId}`;
221
+ const keys = (await $.store.keys()).filter((key) => key.startsWith(METRICS_PREFIX) && key !== own);
222
+ return mergeMetrics(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
223
+ }
224
+
225
+ const metricsView = (router) => mergeMetrics([router.metricsOthers, router.metricsOwn?.record]);
226
+
194
227
  // The view's totals kept across sessions, from what the router holds; nothing until a read succeeded.
195
228
  const savedView = (router) =>
196
229
  router.savingsOthers ? { savingsStore: mergeSavingsStores([router.savingsOthers, router.savingsOwn?.record]) } : {};
@@ -203,6 +236,11 @@ async function storeView($, router) {
203
236
  try {
204
237
  out.activityStore = readStore(await $.store.get(STORE_KEY));
205
238
  } catch {}
239
+ try {
240
+ const { own, resetAt } = await ownMetrics($, router);
241
+ router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
242
+ out.activityMetrics = metricsView(router);
243
+ } catch {}
206
244
  try {
207
245
  const { own, resetAt } = await ownSavings($, router);
208
246
  router.savingsOthers = await otherSavings($, own.sessionId, resetAt);
@@ -220,6 +258,8 @@ async function openPane($, router) {
220
258
  async function changeMode($, router, mode) {
221
259
  const view = router.view ?? (await readView($, router));
222
260
  for (const controller of router.controllers) controller.abort();
261
+ // Turns with routing off run on Claude's model, off the cheaper move's route: the escalation watch ends.
262
+ if (mode === 'manual') router.afterDown = null;
223
263
  router.modes.set(await $.session.id(), mode);
224
264
  const saved = await rememberMode($, mode);
225
265
  await updateView($, router, {
@@ -461,6 +501,8 @@ async function decideTurn($, router, e, signal, step) {
461
501
  turns: facts.turns,
462
502
  signal: controller.signal,
463
503
  });
504
+ // The wait for advice alone, comparable with the activity probe's p95; the routing after it is not the classifier's.
505
+ const adviceMs = pin || !classifierTimed(result.error) ? null : Date.now() - adviceStarted;
464
506
  if (controller.signal.aborted || (await $.session.id()) !== sessionId || (await modeOf($, router)) !== 'auto')
465
507
  return null;
466
508
  const previous = loop.decision;
@@ -480,7 +522,22 @@ async function decideTurn($, router, e, signal, step) {
480
522
  if (previous?.model && previous.model !== selected.decision.model && !selected.decision.pinned)
481
523
  $.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
482
524
  const activity = turnActivity(cfg, result.advice, previous, selected.decision);
483
- router.turnActivities.set(e.turnId, { ...activity, sessionId, requests: 0, inputTokens: 0, outputTokens: 0 });
525
+ router.turnActivities.set(e.turnId, {
526
+ ...activity,
527
+ reason: selected.decision.reason,
528
+ escalated: Boolean(selected.decision.escalated),
529
+ pinned: selected.decision.pinned,
530
+ route: `${selected.decision.model}@${selected.decision.effort ?? 'session'}`,
531
+ classifier: cfg.classifier,
532
+ adviceMs,
533
+ sessionId,
534
+ requests: 0,
535
+ inputTokens: 0,
536
+ outputTokens: 0,
537
+ routedUsd: null,
538
+ pricedRequests: 0,
539
+ routes: {},
540
+ });
484
541
  await updateView($, router, {
485
542
  phase: 'routed',
486
543
  selectedModel: selected.decision.model,
@@ -498,7 +555,7 @@ async function decideTurn($, router, e, signal, step) {
498
555
  ? {
499
556
  error: result.error,
500
557
  health: healthOf(clientOf(router, cfg.classifier), cfg.classifier),
501
- adviceMs: pin || !classifierTimed(result.error) ? null : Date.now() - adviceStarted,
558
+ adviceMs,
502
559
  adviceChoice: result.advice?.choice ?? null,
503
560
  probabilities: result.advice?.probabilities ?? null,
504
561
  estimate: selected.decision.estimate ?? null,
@@ -517,8 +574,8 @@ async function decideTurn($, router, e, signal, step) {
517
574
  }
518
575
 
519
576
  // A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
520
- // turn's activity record.
521
- async function publishReply($, router, turnId, response, tier, route) {
577
+ // turn's activity record with its list price (`routedUsd`, null when unpriced) and, when its tier served it, its route.
578
+ async function publishReply($, router, turnId, response, tier, route, routedUsd) {
522
579
  const turn = router.turnActivities.get(turnId);
523
580
  const metrics = responseMetrics(
524
581
  router.view,
@@ -534,6 +591,9 @@ async function publishReply($, router, turnId, response, tier, route) {
534
591
  requests: turn.requests + 1,
535
592
  inputTokens: turn.inputTokens + (metrics.inputTokens ?? 0),
536
593
  outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
594
+ routedUsd: Number.isFinite(routedUsd) ? (turn.routedUsd ?? 0) + routedUsd : turn.routedUsd,
595
+ pricedRequests: turn.pricedRequests + (Number.isFinite(routedUsd) ? 1 : 0),
596
+ routes: tier ? { ...turn.routes, [route]: (turn.routes[route] ?? 0) + 1 } : turn.routes,
537
597
  });
538
598
  }
539
599
 
@@ -554,7 +614,8 @@ async function observeRouted($, router, { turnId, cfg, loop, loopVersion, reques
554
614
  if (!written.isSet) return;
555
615
  // A substituted reply is not the tier's: the strip and trend must not count it as one.
556
616
  const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
557
- await publishReply($, router, turnId, response, tier, `${request.model}@${request.effort ?? 'session'}`);
617
+ const route = `${request.model}@${request.effort ?? 'session'}`;
618
+ await publishReply($, router, turnId, response, tier, route, compared?.reply?.routedUsd ?? null);
558
619
  if (compared?.reply) await addSavings($, router, turnId, compared.reply, tier);
559
620
  }
560
621
 
@@ -623,7 +684,8 @@ async function flushSavings($, router, totals) {
623
684
  }
624
685
 
625
686
  // A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
626
- // break a turn: a failed read or write loses this turn's counts only.
687
+ // break a turn: a failed read or write loses this turn's counts. Readings already in the session's metrics record stay
688
+ // in memory after a failed write and go out with the next turn that adds a reading.
627
689
  async function recordActivity($, router, turn) {
628
690
  try {
629
691
  if ((await $.session.id()) !== turn.sessionId) return;
@@ -637,6 +699,8 @@ async function recordActivity($, router, turn) {
637
699
  // Advanced before any further await, so a session change meanwhile cannot carry this run into the new session.
638
700
  const { run, ended } = advanceRun(router.run, turn.answer);
639
701
  router.run = run;
702
+ const down = advanceDown(router.afterDown, turn);
703
+ router.afterDown = down.after;
640
704
  await updateView($, router, {
641
705
  activityStats: recordTurn(view.activityStats ?? emptySession(), {
642
706
  activity: turn.label,
@@ -645,8 +709,12 @@ async function recordActivity($, router, turn) {
645
709
  requests: turn.requests,
646
710
  inputTokens: turn.inputTokens,
647
711
  outputTokens: turn.outputTokens,
712
+ routedUsd: turn.routedUsd,
713
+ pricedRequests: turn.pricedRequests,
714
+ routes: turn.routes,
648
715
  tierSwitches: turn.switched === 'tier' ? 1 : 0,
649
716
  activitySwitches: turn.switched === 'activity' ? 1 : 0,
717
+ lateral: turn.lateral,
650
718
  wouldDiffer: turn.wouldDiffer,
651
719
  wouldUsd: turn.wouldUsd,
652
720
  shadow: turn.mode === 'shadow',
@@ -654,6 +722,10 @@ async function recordActivity($, router, turn) {
654
722
  });
655
723
  const resets = router.statsResets;
656
724
  const store = readStore(await $.store.get(STORE_KEY));
725
+ const { own, resetAt } = await ownMetrics($, router);
726
+ // The others held from before a reset in another session are read again.
727
+ if (!router.metricsOthers || metricsAfterReset(router.metricsOthers, resetAt) !== router.metricsOthers)
728
+ router.metricsOthers = await otherMetrics($, own.sessionId, resetAt);
657
729
  // Writing counts read before a reset would bring them back; this turn's are lost instead.
658
730
  if (router.statsResets !== resets) return;
659
731
  const written = recordStore(store, {
@@ -664,9 +736,23 @@ async function recordActivity($, router, turn) {
664
736
  wouldDiffer: turn.wouldDiffer,
665
737
  wouldUsd: turn.wouldUsd,
666
738
  });
667
- await $.store.set(STORE_KEY, written);
739
+ const measured = recordMetrics(own.record, {
740
+ classifier: turn.classifier,
741
+ adviceMs: turn.adviceMs,
742
+ downMove: down.event,
743
+ now: Date.now(),
744
+ });
745
+ // Added to in memory before the write, so a refresh during the write counts the turn once.
746
+ const changed = measured !== own.record;
747
+ own.record = measured;
748
+ // Both writes start before any reset can run between them.
749
+ await Promise.all([
750
+ $.store.set(STORE_KEY, written),
751
+ changed ? $.store.set(`${METRICS_PREFIX}${own.sessionId}`, measured) : null,
752
+ ]);
668
753
  // A reset during the write emptied the view; these counts are older than it.
669
- if (router.statsResets === resets) await updateView($, router, { activityStore: written });
754
+ if (router.statsResets === resets)
755
+ await updateView($, router, { activityStore: written, activityMetrics: metricsView(router) });
670
756
  } catch {}
671
757
  }
672
758
 
@@ -832,33 +918,36 @@ function paneActions($, router, view) {
832
918
  resetStats: async () => {
833
919
  router.statsResets += 1;
834
920
  router.run = null;
921
+ router.afterDown = null;
835
922
  router.turnSavings.clear();
836
923
  const clear = (key, value) =>
837
924
  $.store
838
925
  .set(key, value)
839
926
  .then(() => true)
840
927
  .catch(() => false);
841
- const clearSavings = async () => {
928
+ // The per-session records: savings and activity metrics.
929
+ const clearRecords = async () => {
842
930
  // Written first, so a session still holding its record from before the reset drops it, unless that record's
843
- // first reply has the reset's millisecond or the clock moved back across the reset.
931
+ // first reading has the reset's millisecond or the clock moved back across the reset.
844
932
  await $.store.set(SAVINGS_RESET_KEY, Date.now());
845
- const keys = (await $.store.keys()).filter((key) => key.startsWith(SAVINGS_PREFIX));
933
+ const keys = (await $.store.keys()).filter(
934
+ (key) => key.startsWith(SAVINGS_PREFIX) || key.startsWith(METRICS_PREFIX),
935
+ );
846
936
  await Promise.all(keys.map((key) => $.store.delete(key)));
847
937
  router.savingsOwn = null;
848
938
  router.savingsOthers = emptySavingsStore();
939
+ router.metricsOwn = null;
940
+ router.metricsOthers = emptyMetrics();
849
941
  return true;
850
942
  };
851
- const [activity, savings] = await Promise.all([
852
- clear(STORE_KEY, emptyStore()),
853
- clearSavings().catch(() => false),
854
- ]);
943
+ const [stats, records] = await Promise.all([clear(STORE_KEY, emptyStore()), clearRecords().catch(() => false)]);
855
944
  await updateView($, router, {
856
945
  activityStats: null,
857
946
  savings: null,
858
- ...(activity ? { activityStore: emptyStore() } : {}),
859
- ...(savings ? { savingsStore: emptySavingsStore() } : {}),
947
+ ...(stats ? { activityStore: emptyStore() } : {}),
948
+ ...(records ? { activityMetrics: emptyMetrics(), savingsStore: emptySavingsStore() } : {}),
860
949
  notice:
861
- activity && savings
950
+ stats && records
862
951
  ? 'Stats reset: activity and routing vs your model, this session and saved.'
863
952
  : 'This session’s stats reset. Saved stats could not be cleared.',
864
953
  });
@@ -1076,10 +1165,12 @@ export function register(on, options) {
1076
1165
  }
1077
1166
  router.prompts.delete(e.turnId);
1078
1167
  for (const key of router.decisions.keys()) if (key.includes(`:${e.turnId}:`)) router.decisions.delete(key);
1079
- // `off` asks nothing, so it records nothing.
1168
+ // `off` asks nothing, so it records nothing; it only ends the watch after a cheaper activity move.
1080
1169
  const turn = router.turnActivities.get(e.turnId);
1081
1170
  router.turnActivities.delete(e.turnId);
1082
- if (turn && turn.mode !== 'off') await recordActivity($, router, turn);
1171
+ if (turn?.mode === 'off') {
1172
+ if ((await $.session.id()) === turn.sessionId) router.afterDown = advanceDown(router.afterDown, turn).after;
1173
+ } else if (turn) await recordActivity($, router, turn);
1083
1174
  const savings = router.turnSavings.get(e.turnId);
1084
1175
  router.turnSavings.delete(e.turnId);
1085
1176
  if (savings) await flushSavings($, router, savings);
@@ -1107,6 +1198,7 @@ export function register(on, options) {
1107
1198
  router.turnSavings.clear();
1108
1199
  const run = router.run;
1109
1200
  router.run = null;
1201
+ router.afterDown = null;
1110
1202
  if (run) await flushRun($, run);
1111
1203
  router.view = {
1112
1204
  ...initialView(router.view?.nativeModel ?? ''),
@@ -1116,6 +1208,7 @@ export function register(on, options) {
1116
1208
  credentials: router.view?.credentials ?? null,
1117
1209
  // Counts kept across sessions: the next session starts from them.
1118
1210
  activityStore: router.view?.activityStore ?? null,
1211
+ activityMetrics: router.view?.activityMetrics ?? null,
1119
1212
  savingsStore: router.view?.savingsStore ?? null,
1120
1213
  health: healthOf(clientOf(router, router.config.classifier), router.config.classifier),
1121
1214
  configPath: router.config.nativePath,
@@ -1,11 +1,15 @@
1
- // Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducer feeds the
2
- // counts kept across sessions under `activity:stats:v1`. Pure: state and a turn in, new state out.
1
+ // Activity statistics: counts only, never text. The session reducer feeds the Usage tab; the store reducers feed the
2
+ // counts kept across sessions under `activity:stats:v1` and, per session, under METRICS_PREFIX. Pure: state and a turn
3
+ // in, new state out.
3
4
 
4
5
  import { acceptActivity } from './activity.mjs';
5
6
  import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
6
7
 
7
8
  export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
8
9
  export const STORE_KEY = 'activity:stats:v1';
10
+ // Each session keeps its own metrics under this prefix and its session id, as savings does (SAVINGS_PREFIX): sessions
11
+ // finishing turns at the same moment do not overwrite each other. The readout is their sum (mergeMetrics).
12
+ export const METRICS_PREFIX = 'activity:metrics:v1:';
9
13
 
10
14
  // The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
11
15
  // separate plan from review. That is enough to catch a cheap route on a coding turn.
@@ -46,18 +50,35 @@ function addShadow(shadow, wouldDiffer, range) {
46
50
  };
47
51
  }
48
52
 
53
+ function addRoutes(before, added) {
54
+ const next = { ...before };
55
+ for (const [route, n] of Object.entries(added ?? {})) if (count(n)) next[route] = (next[route] ?? 0) + count(n);
56
+ return next;
57
+ }
58
+
59
+ // The route most of an activity's routed replies went out with, `model@effort`; ties go to the first in name order,
60
+ // so the readout is stable. Null without routed replies.
61
+ export function mostlyOn(counts) {
62
+ const [top] = Object.entries(counts?.routes ?? {}).sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
63
+ return top?.[0] ?? null;
64
+ }
65
+
49
66
  export function emptySession() {
50
67
  return {
51
68
  byActivity: {},
52
69
  switches: { tier: 0, activity: 0 },
70
+ lateral: { taken: 0, refused: 0 },
53
71
  agreement: { matched: 0, total: 0 },
54
72
  shadow: emptyShadow(),
55
73
  };
56
74
  }
57
75
 
58
76
  // One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
59
- // agreement compares with the tools (null: not counted), `shadow` marks a turn in shadow mode, `wouldDiffer` says
60
- // whether 'on' would have routed it differently (null: no would-route) and `wouldUsd` prices that difference.
77
+ // agreement compares with the tools (null: not counted), `lateral` the turn's lateral outcome (turnActivity, counted
78
+ // outside shadow only), `shadow` marks a turn in shadow mode, `wouldDiffer` says whether 'on' would have routed it
79
+ // differently (null: no would-route) and `wouldUsd` prices that difference. `routedUsd` is the list price of the
80
+ // turn's priced replies (null: none had a price), `pricedRequests` how many of its replies had one, and `routes` counts
81
+ // its routed replies by the `model@effort` they went out with.
61
82
  export function recordTurn(
62
83
  session,
63
84
  {
@@ -67,8 +88,12 @@ export function recordTurn(
67
88
  requests,
68
89
  inputTokens,
69
90
  outputTokens,
91
+ routedUsd = null,
92
+ pricedRequests = 0,
93
+ routes = {},
70
94
  tierSwitches,
71
95
  activitySwitches,
96
+ lateral,
72
97
  wouldDiffer,
73
98
  wouldUsd,
74
99
  shadow,
@@ -76,6 +101,8 @@ export function recordTurn(
76
101
  ) {
77
102
  const key = activity ?? 'none';
78
103
  const before = session.byActivity[key] ?? { turns: 0, requests: 0, inputTokens: 0, outputTokens: 0 };
104
+ // A view saved by 1.7.0 has no lateral counts.
105
+ const moves = { taken: 0, refused: 0, ...session.lateral };
79
106
  return {
80
107
  byActivity: {
81
108
  ...session.byActivity,
@@ -84,12 +111,19 @@ export function recordTurn(
84
111
  requests: before.requests + count(requests),
85
112
  inputTokens: before.inputTokens + count(inputTokens),
86
113
  outputTokens: before.outputTokens + count(outputTokens),
114
+ // Counts saved by 1.7.0 have no cost, priced replies or routes: their replies stay unpriced.
115
+ routedUsd: Number.isFinite(routedUsd) ? (before.routedUsd ?? 0) + routedUsd : (before.routedUsd ?? null),
116
+ pricedRequests: (before.pricedRequests ?? 0) + count(pricedRequests),
117
+ routes: addRoutes(before.routes ?? {}, routes),
87
118
  },
88
119
  },
89
120
  switches: {
90
121
  tier: session.switches.tier + count(tierSwitches),
91
122
  activity: session.switches.activity + count(activitySwitches),
92
123
  },
124
+ // In shadow the lateral outcome is the would-route's, which no route followed; the Shadow line covers it.
125
+ lateral:
126
+ !shadow && (lateral === 'taken' || lateral === 'refused') ? { ...moves, [lateral]: moves[lateral] + 1 } : moves,
93
127
  agreement: answer
94
128
  ? {
95
129
  matched: session.agreement.matched + (agrees(answer, observed) ? 1 : 0),
@@ -103,10 +137,33 @@ export function recordTurn(
103
137
  };
104
138
  }
105
139
 
140
+ // Classifier wait per turn, bucketed by these upper bounds in ms, plus one bucket above the last; kept per classifier
141
+ // id for at most LATENCY_CLASSIFIERS ids, so the store stays bounded.
142
+ export const LATENCY_BOUNDS = [
143
+ 100, 150, 200, 250, 300, 350, 400, 450, 500, 600, 700, 800, 900, 1000, 1250, 1500, 2000, 2500, 3000, 4000, 5000,
144
+ ];
145
+ const LATENCY_BUCKETS = LATENCY_BOUNDS.length + 1;
146
+ const LATENCY_CLASSIFIERS = 8;
147
+
106
148
  export function emptyStore() {
107
149
  return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: emptyShadow() };
108
150
  }
109
151
 
152
+ // The p95 of a latency histogram: the upper bound of its bucket in ms, `over` when it lies above the last bound, and
153
+ // the turns counted. Null without turns.
154
+ export function latencyP95(buckets) {
155
+ const turns = (buckets ?? []).reduce((sum, n) => sum + n, 0);
156
+ if (!turns) return null;
157
+ let seen = 0;
158
+ const at = buckets.findIndex((n) => {
159
+ seen += n;
160
+ return seen >= 0.95 * turns;
161
+ });
162
+ return at < LATENCY_BOUNDS.length
163
+ ? { ms: LATENCY_BOUNDS[at], over: false, turns }
164
+ : { ms: LATENCY_BOUNDS.at(-1), over: true, turns };
165
+ }
166
+
110
167
  const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
111
168
  const isRecord = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
112
169
  const onlyKeys = (v, keys) => isRecord(v) && Object.keys(v).every((k) => keys.includes(k));
@@ -127,7 +184,8 @@ const isShadow = (v) =>
127
184
  Number.isFinite(v.maxUsd));
128
185
 
129
186
  // The stored value if it has a known shape, filled to the current one, else an empty store: a hand-edited or
130
- // unknown file must not break stats.
187
+ // unknown file must not break stats. The shape is 1.7.0's, which rejects any other key: a new count goes under its own
188
+ // key, such as METRICS_KEY, so a downgrade keeps these.
131
189
  export function readStore(value) {
132
190
  const ok =
133
191
  onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
@@ -167,6 +225,101 @@ export function recordStore(store, { predicted, observed, runEnded, lateral, wou
167
225
  return next;
168
226
  }
169
227
 
228
+ // The readings kept across sessions beside the stats store, one session's or their sum: the classifier wait per
229
+ // classifier id, and the cheaper activity moves taken in `on` with the escalations that followed one, since `since`
230
+ // (ms), the first reading after a reset.
231
+ export function emptyMetrics() {
232
+ return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 } };
233
+ }
234
+
235
+ const isLatency = (v) =>
236
+ isRecord(v) &&
237
+ Object.keys(v).length <= LATENCY_CLASSIFIERS &&
238
+ Object.values(v).every((b) => Array.isArray(b) && b.length === LATENCY_BUCKETS && b.every(isCount));
239
+
240
+ // Each part read on its own: a bad or outdated one, such as latency after LATENCY_BOUNDS changes, reads as empty and
241
+ // leaves the others.
242
+ export function readMetrics(value) {
243
+ const empty = emptyMetrics();
244
+ if (!isRecord(value) || value.version !== 1) return empty;
245
+ return {
246
+ version: 1,
247
+ since: Number.isFinite(value.since) ? value.since : null,
248
+ latency: isLatency(value.latency) ? value.latency : empty.latency,
249
+ downMoves: exactCountsOf(value.downMoves, ['moves', 'escalations']) ? value.downMoves : empty.downMoves,
250
+ };
251
+ }
252
+
253
+ // One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed), and `downMove` 'moved' or
254
+ // 'escalated' (advanceDown). A turn that adds nothing returns `metrics` itself.
255
+ export function recordMetrics(metrics, { classifier = null, adviceMs = null, downMove = null, now = null }) {
256
+ const next = {
257
+ version: 1,
258
+ since: metrics.since ?? now,
259
+ latency: { ...metrics.latency },
260
+ downMoves: { ...metrics.downMoves },
261
+ };
262
+ let counted = downMove === 'moved' || downMove === 'escalated';
263
+ const known = Object.hasOwn(next.latency, classifier);
264
+ if (
265
+ typeof classifier === 'string' &&
266
+ Number.isFinite(adviceMs) &&
267
+ adviceMs >= 0 &&
268
+ (known || Object.keys(next.latency).length < LATENCY_CLASSIFIERS)
269
+ ) {
270
+ const buckets = known ? [...next.latency[classifier]] : Array(LATENCY_BUCKETS).fill(0);
271
+ const at = LATENCY_BOUNDS.findIndex((bound) => adviceMs <= bound);
272
+ buckets[at === -1 ? LATENCY_BOUNDS.length : at] += 1;
273
+ next.latency[classifier] = buckets;
274
+ counted = true;
275
+ }
276
+ if (!counted) return metrics;
277
+ if (downMove === 'moved') next.downMoves.moves += 1;
278
+ if (downMove === 'escalated') next.downMoves.escalations += 1;
279
+ return next;
280
+ }
281
+
282
+ // A record whose first reading came before the last reset at `resetAt` (ms) reads as empty, as savings' afterReset.
283
+ export function metricsAfterReset(record, resetAt) {
284
+ return Number.isFinite(resetAt) && record.since !== null && record.since < resetAt ? emptyMetrics() : record;
285
+ }
286
+
287
+ // The metrics kept across sessions: every session's record since the last reset added up, since the earliest. Latency
288
+ // keeps the first LATENCY_CLASSIFIERS classifier ids in the order given, so the sum still reads back (readMetrics).
289
+ export function mergeMetrics(values, resetAt = null) {
290
+ const merged = emptyMetrics();
291
+ for (const one of values.map((value) => metricsAfterReset(readMetrics(value), resetAt))) {
292
+ const since = [merged.since, one.since].filter(Number.isFinite);
293
+ merged.since = since.length ? Math.min(...since) : null;
294
+ for (const [id, buckets] of Object.entries(one.latency)) {
295
+ // Own keys only: a classifier id such as `toString` must not read Object.prototype.
296
+ const known = Object.hasOwn(merged.latency, id) ? merged.latency[id] : null;
297
+ if (known) merged.latency[id] = known.map((n, i) => n + buckets[i]);
298
+ else if (Object.keys(merged.latency).length < LATENCY_CLASSIFIERS) merged.latency[id] = [...buckets];
299
+ }
300
+ merged.downMoves = {
301
+ moves: merged.downMoves.moves + one.downMoves.moves,
302
+ escalations: merged.downMoves.escalations + one.downMoves.escalations,
303
+ };
304
+ }
305
+ return merged;
306
+ }
307
+
308
+ // Escalations after a cheaper activity move. `after` is the `model@effort` route a cheaper move went to while the
309
+ // watch lasts, else null; `turn` is the turn's mode, decision reason and lateral outcome, whether a failure asked for
310
+ // an escalation (also one the cash gate held or context-fit replaced), whether it was pinned, and its route. A taken
311
+ // cheaper move in `on` starts the watch; an escalation while it lasts counts once and ends it, as does a turn on
312
+ // another route or a turn in another mode (an `off` turn too, which records nothing else); the hook also ends it when
313
+ // routing is turned off. A pin is the person's choice, not a route change: the watch goes on if the turn after it returns.
314
+ // Returns the watch now and the turn's event: 'moved', 'escalated' or null.
315
+ export function advanceDown(after, { mode, reason, lateral, escalated, pinned, route }) {
316
+ if (mode !== 'on') return { after: null, event: null };
317
+ if (reason === 'activity-down' && lateral === 'taken') return { after: route, event: 'moved' };
318
+ if (!after || pinned) return { after, event: null };
319
+ if (escalated) return { after: null, event: 'escalated' };
320
+ return { after: route === after ? after : null, event: null };
321
+ }
322
+
170
323
  // A run is consecutive turns with the same predicted activity. Given the run so far ({ activity, length } or null)
171
324
  // and this turn's predicted activity (null or 'uncertain' end it), returns the run now and the run that just ended.
172
325
  export function advanceRun(run, predicted) {
@@ -175,21 +328,22 @@ export function advanceRun(run, predicted) {
175
328
  return { run: activity ? { activity, length: 1 } : null, ended: run };
176
329
  }
177
330
 
178
- const LATERAL = { 'activity-up': 'taken', 'activity-down': 'taken', 'activity-pending': 'refused' };
179
331
  const differs = (a, b) => a.model !== b.model || (a.effort ?? null) !== (b.effort ?? null);
332
+ // A decision's lateral outcome: 'taken', 'refused' (not worth it, a hold, the cash gate), or null for none.
333
+ const lateralOf = (decision) =>
334
+ decision?.lateral === 'taken' || decision?.lateral === 'refused' ? decision.lateral : null;
180
335
 
181
336
  // What one routed turn adds to the stats, from its config, the classifier's advice, and the decisions before and
182
337
  // after it. `label` is the turn's activity: the applied one in `on`, the accepted answer in `shadow`; `answer` is the
183
338
  // accepted answer in both, which the tools are checked against and runs follow. `switched` names
184
339
  // what moved the route from the previous turn's: 'activity' for a lateral move, 'tier' for any other, null for none
185
- // or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`; a lateral move refused
186
- // by a hold or the cash gate shares its reason with tier decisions and is not counted.
340
+ // or a pin. `lateral` reads the decision that carries an activity, the would-route in `shadow`.
187
341
  export function turnActivity(config, advice, previous, decision) {
188
342
  const mode = config.activityRouting;
189
343
  const would = decision.wouldRoute ?? null;
190
344
  const switched =
191
345
  previous?.model && !decision.pinned && differs(previous, decision)
192
- ? LATERAL[decision.reason] === 'taken'
346
+ ? lateralOf(decision) === 'taken'
193
347
  ? 'activity'
194
348
  : 'tier'
195
349
  : null;
@@ -201,7 +355,7 @@ export function turnActivity(config, advice, previous, decision) {
201
355
  label: mode === 'on' ? (decision.activity ?? null) : mode === 'shadow' ? answer : null,
202
356
  predicted: advice?.activity?.choice ?? null,
203
357
  switched,
204
- lateral: LATERAL[(mode === 'shadow' ? would : mode === 'on' ? decision : null)?.reason] ?? null,
358
+ lateral: lateralOf(mode === 'shadow' ? would : mode === 'on' ? decision : null),
205
359
  // A shadow turn always counts, so the session and store denominators agree.
206
360
  wouldDiffer,
207
361
  wouldUsd: wouldDiffer ? (would.difference ?? null) : null,
package/lib/display.mjs CHANGED
@@ -9,6 +9,7 @@ import {
9
9
  sameRoute,
10
10
  } from './config.mjs';
11
11
  import { ratesAt } from './cost.mjs';
12
+ import { PROBE_RESULTS } from './probe-results.mjs';
12
13
  import { cellForModel, isSameModel } from './route.mjs';
13
14
  import { emptyTotals, readout, readSavingsStore } from './savings.mjs';
14
15
 
@@ -206,6 +207,25 @@ export function activityRouteText(config, view) {
206
207
  : `${cell} → ${routeName(config, route)} (override) base: ${routeName(config, base)}`;
207
208
  }
208
209
 
210
+ // Reasons that only say the route stayed: on an override cell they leave out why it is not the tier's base route.
211
+ const STAYED = ['ready', 'same-tier', 'no-advice', 'continuation', 'uncertain'];
212
+
213
+ // The Now tab's Why. In `on`, an activity move, or a stay on an override cell, says the cell plainly: "code at low
214
+ // runs on Sonnet 5.5 · high", or for a move to no activity "low runs on its base route, Haiku 5.5 · high".
215
+ // Every other decision keeps its reason: a refusal, a hold or a tier move says more than the cell.
216
+ export function whyText(config, view) {
217
+ const reason = REASONS[view.reason] ?? view.reason ?? 'ready';
218
+ if (config.activityRouting !== 'on' || !view.tier) return reason;
219
+ const activity = view.activity ?? null;
220
+ const route = resolveRoute(config, view.tier, activity);
221
+ const override = activity !== null && !sameRoute(route, resolveRoute(config, view.tier));
222
+ const moved = view.reason === 'activity-up' || view.reason === 'activity-down';
223
+ if (!moved && !(override && STAYED.includes(view.reason ?? 'ready'))) return reason;
224
+ return activity
225
+ ? `${activity} at ${view.tier} runs on ${routeName(config, route)}`
226
+ : `${view.tier} runs on its base route, ${routeName(config, route)}`;
227
+ }
228
+
209
229
  // The activity lines of /router without a UI surface: "Activity: ops (81%)" and the resolved route. None in `off`.
210
230
  export function activityDetailLines(config, view) {
211
231
  if (config.activityRouting === 'off') return [];
@@ -227,6 +247,19 @@ export function storeAgreementLine(view) {
227
247
  return `Agreement across sessions: ${all}${code}`;
228
248
  }
229
249
 
250
+ // The classifier's last checked-in activity probe, "activity probe 70/70 · p95 299 ms · Oct 10", or null when none
251
+ // was run on the model it is configured with.
252
+ export function probeText(config, id) {
253
+ const probe = PROBE_RESULTS[id];
254
+ if (!probe || probe.model !== config.classifiers[id]?.model) return null;
255
+ const day = new Date(`${probe.date}T00:00:00Z`).toLocaleDateString('en-US', {
256
+ month: 'short',
257
+ day: 'numeric',
258
+ timeZone: 'UTC',
259
+ });
260
+ return `activity probe ${probe.correct}/${probe.probes} · p95 ${probe.p95Ms} ms · ${day}`;
261
+ }
262
+
230
263
  // Routing vs your model: less is green, more is amber, not red: a stronger model is a purchase, not a fault.
231
264
  export const SAVINGS_COLOR = { less: '#6fcf97', more: '#f0b429' };
232
265