@alexeiled/claude-router 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +47 -0
- package/README.md +14 -4
- package/docs/activity-routing.md +254 -0
- package/docs/architecture.md +9 -3
- package/docs/configuration.md +56 -44
- package/docs/evaluation.md +37 -6
- package/docs/native-router.md +17 -13
- package/docs/router-band.svg +78 -80
- package/docs/router-pane-classifier.svg +7 -7
- package/docs/router-pane-now.svg +9 -9
- package/docs/router-pane-routing.svg +73 -87
- package/docs/router-pane-usage.svg +123 -0
- package/docs/user-guide.md +79 -53
- package/hooks/native-router.mjs +236 -39
- package/lib/activity-stats.mjs +87 -21
- package/lib/band.mjs +82 -55
- package/lib/config.mjs +49 -32
- package/lib/cost.mjs +32 -5
- package/lib/display.mjs +79 -4
- package/lib/panel.mjs +389 -67
- package/lib/route.mjs +27 -12
- package/lib/savings.mjs +237 -0
- package/lib/view.mjs +10 -1
- package/package.json +1 -1
- package/types/index.d.ts +51 -3
package/hooks/native-router.mjs
CHANGED
|
@@ -38,6 +38,9 @@ import {
|
|
|
38
38
|
GATEWAY_SETTINGS,
|
|
39
39
|
missingCredentials,
|
|
40
40
|
NOT_A_TIER_REASON,
|
|
41
|
+
savingsLine,
|
|
42
|
+
storeAgreementLine,
|
|
43
|
+
unavailableReason,
|
|
41
44
|
} from '../lib/display.mjs';
|
|
42
45
|
import { clip, observedActivity, promptIndex } from '../lib/facts.mjs';
|
|
43
46
|
import { renderPanel, routeDraftOf, routingChanges } from '../lib/panel.mjs';
|
|
@@ -54,6 +57,19 @@ import {
|
|
|
54
57
|
prepareLoop,
|
|
55
58
|
resetHistory,
|
|
56
59
|
} from '../lib/route.mjs';
|
|
60
|
+
import {
|
|
61
|
+
addReply,
|
|
62
|
+
afterReset,
|
|
63
|
+
compareReply,
|
|
64
|
+
emptySavingsStore,
|
|
65
|
+
emptyTotals,
|
|
66
|
+
mergeSavingsStores,
|
|
67
|
+
readSavingsStore,
|
|
68
|
+
recordSavingsStore,
|
|
69
|
+
SAVINGS_PREFIX,
|
|
70
|
+
SAVINGS_RESET_KEY,
|
|
71
|
+
sessionCacheTtl,
|
|
72
|
+
} from '../lib/savings.mjs';
|
|
57
73
|
import { CLEARED_READINGS, continuationView, healthOf, initialView, responseMetrics } from '../lib/view.mjs';
|
|
58
74
|
|
|
59
75
|
// The engine follows $ only into functions declared in this file, never across an import: every helper that takes $
|
|
@@ -87,6 +103,13 @@ function createRouter(options) {
|
|
|
87
103
|
// one activity, flushed to the store when it ends.
|
|
88
104
|
turnActivities: new Map(),
|
|
89
105
|
run: null,
|
|
106
|
+
// Each turn's routing-vs-your-model totals until turn.complete adds them to the store.
|
|
107
|
+
turnSavings: new Map(),
|
|
108
|
+
// The totals kept across sessions: this session's record (`{ sessionId, record }`), held here so a refresh never
|
|
109
|
+
// reads it back from the store while a write of it is pending, and the sum of every other session's, read on
|
|
110
|
+
// refresh. The view shows their sum.
|
|
111
|
+
savingsOwn: null,
|
|
112
|
+
savingsOthers: null,
|
|
90
113
|
// Bumped by Reset activity stats, so a store write that read the counts before the reset skips.
|
|
91
114
|
statsResets: 0,
|
|
92
115
|
};
|
|
@@ -117,7 +140,8 @@ async function modeOf($, router, fallback = 'auto') {
|
|
|
117
140
|
}
|
|
118
141
|
}
|
|
119
142
|
|
|
120
|
-
// A saved preference wins; a fresh session starts with routing on for a model some
|
|
143
|
+
// A saved preference wins; a fresh session starts with routing on for a model some route runs (a tier's, or in `on` an
|
|
144
|
+
// activity override's), and off for any other.
|
|
121
145
|
async function startMode($, router, model) {
|
|
122
146
|
const mode = await modeOf($, router, cellForModel(router.config, model) === null ? 'manual' : 'auto');
|
|
123
147
|
router.modes.set(await $.session.id(), mode);
|
|
@@ -146,10 +170,50 @@ async function updateView($, router, patch) {
|
|
|
146
170
|
await $.state.set(VIEW, router.view);
|
|
147
171
|
}
|
|
148
172
|
|
|
173
|
+
// This session's record kept across sessions, read from the store once per session, and the last reset (ms) from
|
|
174
|
+
// any session, read each time: a Reset in another session empties a record held from before it.
|
|
175
|
+
async function ownSavings($, router) {
|
|
176
|
+
const sessionId = await $.session.id();
|
|
177
|
+
const resetAt = await $.store.get(SAVINGS_RESET_KEY);
|
|
178
|
+
if (router.savingsOwn?.sessionId !== sessionId) {
|
|
179
|
+
const record = readSavingsStore(await $.store.get(`${SAVINGS_PREFIX}${sessionId}`));
|
|
180
|
+
// Another call may have read it meanwhile and a turn added to it: that one stands.
|
|
181
|
+
if (router.savingsOwn?.sessionId !== sessionId) router.savingsOwn = { sessionId, record };
|
|
182
|
+
}
|
|
183
|
+
router.savingsOwn.record = afterReset(router.savingsOwn.record, resetAt);
|
|
184
|
+
return { own: router.savingsOwn, resetAt };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Every other session's record since the last reset, added up.
|
|
188
|
+
async function otherSavings($, sessionId, resetAt) {
|
|
189
|
+
const own = `${SAVINGS_PREFIX}${sessionId}`;
|
|
190
|
+
const keys = (await $.store.keys()).filter((key) => key.startsWith(SAVINGS_PREFIX) && key !== own);
|
|
191
|
+
return mergeSavingsStores(await Promise.all(keys.map((key) => $.store.get(key))), resetAt);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// The view's totals kept across sessions, from what the router holds; nothing until a read succeeded.
|
|
195
|
+
const savedView = (router) =>
|
|
196
|
+
router.savingsOthers ? { savingsStore: mergeSavingsStores([router.savingsOthers, router.savingsOwn?.record]) } : {};
|
|
197
|
+
|
|
198
|
+
// The counts kept across sessions for the view, read when the session starts and the pane opens: another session
|
|
199
|
+
// may have added to them. A failed read keeps what the view has; it never breaks a turn or a render. The saved
|
|
200
|
+
// totals are taken from the router after the reads, so a turn that finished meanwhile counts once.
|
|
201
|
+
async function storeView($, router) {
|
|
202
|
+
const out = {};
|
|
203
|
+
try {
|
|
204
|
+
out.activityStore = readStore(await $.store.get(STORE_KEY));
|
|
205
|
+
} catch {}
|
|
206
|
+
try {
|
|
207
|
+
const { own, resetAt } = await ownSavings($, router);
|
|
208
|
+
router.savingsOthers = await otherSavings($, own.sessionId, resetAt);
|
|
209
|
+
} catch {}
|
|
210
|
+
return out;
|
|
211
|
+
}
|
|
212
|
+
|
|
149
213
|
// A notice answers the last press in the pane, so the pane opens without one; the last write keeps its Undo in the
|
|
150
214
|
// status bar.
|
|
151
215
|
async function openPane($, router) {
|
|
152
|
-
await updateView($, router, { notice: null });
|
|
216
|
+
await updateView($, router, { notice: null, ...(await storeView($, router)), ...savedView(router) });
|
|
153
217
|
await $.ui.open({ id: PANE, title: PANE_TITLE, focus: true, closeOnEscape: true });
|
|
154
218
|
}
|
|
155
219
|
|
|
@@ -169,7 +233,7 @@ async function changeMode($, router, mode) {
|
|
|
169
233
|
|
|
170
234
|
async function setPin($, router, tier) {
|
|
171
235
|
const view = router.view ?? (await readView($, router));
|
|
172
|
-
if (view.phase === 'unavailable') return
|
|
236
|
+
if (view.phase === 'unavailable') return unavailableText(view);
|
|
173
237
|
const mode = await modeOf($, router, view.mode);
|
|
174
238
|
if (mode === 'manual') return 'Routing is off. Turn routing on before pinning a turn.';
|
|
175
239
|
await updateView($, router, { pendingPin: tier });
|
|
@@ -180,6 +244,8 @@ async function setPin($, router, tier) {
|
|
|
180
244
|
async function routerCommand($, router, args) {
|
|
181
245
|
const [action, value] = args.trim().split(/\s+/);
|
|
182
246
|
if (action === 'auto' || action === 'off') {
|
|
247
|
+
const view = router.view ?? (await readView($, router));
|
|
248
|
+
if (view.phase === 'unavailable') return { text: unavailableText(view) };
|
|
183
249
|
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
184
250
|
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
185
251
|
}
|
|
@@ -201,7 +267,7 @@ async function routerCommand($, router, args) {
|
|
|
201
267
|
// `/router activities <mode>`: the pane's write, so the file keeps the rest and Undo in the pane restores it.
|
|
202
268
|
async function setActivityRouting($, router, mode) {
|
|
203
269
|
const view = router.view ?? (await readView($, router));
|
|
204
|
-
if (view.phase === 'unavailable') return
|
|
270
|
+
if (view.phase === 'unavailable') return unavailableText(view);
|
|
205
271
|
if (mode === router.config.activityRouting) return `Activity routing is already ${mode}.`;
|
|
206
272
|
await paneActions($, router, view).activityRouting(mode);
|
|
207
273
|
return router.view.notice;
|
|
@@ -280,7 +346,15 @@ async function saveConfig($, path, change) {
|
|
|
280
346
|
}
|
|
281
347
|
}
|
|
282
348
|
|
|
349
|
+
const unavailableText = (view) => `Routing unavailable: ${unavailableReason(view)}. Claude’s model is kept.`;
|
|
350
|
+
|
|
283
351
|
function detailText(config, view) {
|
|
352
|
+
if (view.phase === 'unavailable')
|
|
353
|
+
return [
|
|
354
|
+
unavailableText(view),
|
|
355
|
+
`Model: ${view.nativeModel}`,
|
|
356
|
+
...(view.error === GATEWAY_SETTINGS ? GATEWAY_CLEANUP : []),
|
|
357
|
+
].join('\n');
|
|
284
358
|
const missing = missingCredentials(config, view, config.classifier);
|
|
285
359
|
return [
|
|
286
360
|
`Router — routing ${view.mode === 'auto' ? 'on' : 'off'}`,
|
|
@@ -289,13 +363,14 @@ function detailText(config, view) {
|
|
|
289
363
|
`Observed: ${view.actualModel ?? 'no response yet'}`,
|
|
290
364
|
`Reason: ${view.reason}`,
|
|
291
365
|
...activityDetailLines(config, view),
|
|
366
|
+
storeAgreementLine(view),
|
|
367
|
+
savingsLine(view),
|
|
292
368
|
view.error || missing ? classifierStatus(config, view.error ?? missing) : `${activeClassifier(config).label} ready`,
|
|
293
369
|
`Context: ${view.contextKnown ? `${view.contextTokens} tokens (estimate)` : 'unknown'}`,
|
|
294
370
|
`Observed cache: ${view.cacheRead ?? 'unknown'} read, ${view.cacheWrite ?? 'unknown'} written tokens`,
|
|
295
371
|
view.pendingPin ? `Next turn pin: ${view.pendingPin}` : 'No next-turn pin.',
|
|
296
372
|
'Routing on picks a model for each turn. Routing off keeps Claude’s model. Pins serve one turn only.',
|
|
297
373
|
'Cache lifetime is an estimate. Claude’s cost ledger owns session totals.',
|
|
298
|
-
...(view.error === GATEWAY_SETTINGS ? GATEWAY_CLEANUP : []),
|
|
299
374
|
].join('\n');
|
|
300
375
|
}
|
|
301
376
|
|
|
@@ -333,6 +408,9 @@ async function* passMain($, e, next, loop, version, nativeModel, reason, router)
|
|
|
333
408
|
stopReason: response.stopReason,
|
|
334
409
|
now: Date.now(),
|
|
335
410
|
});
|
|
411
|
+
// Routing chose nothing here, so the reply is not counted; your model's cache still follows it.
|
|
412
|
+
const compared = await compareYours($, router, loop, response, e.effort ?? null, nativeModel);
|
|
413
|
+
if (compared) observed.yours = compared.state;
|
|
336
414
|
const written = await $.state.set(ref, observed, { ifVersion: version });
|
|
337
415
|
if (written.isSet) await updateView($, router, responseMetrics(router.view, response, null));
|
|
338
416
|
}
|
|
@@ -440,9 +518,15 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
440
518
|
|
|
441
519
|
// A routed reply's metrics in the view, labelled with the turn's activity when its tier served it, and added to the
|
|
442
520
|
// turn's activity record.
|
|
443
|
-
async function publishReply($, router, turnId, response, tier) {
|
|
521
|
+
async function publishReply($, router, turnId, response, tier, route) {
|
|
444
522
|
const turn = router.turnActivities.get(turnId);
|
|
445
|
-
const metrics = responseMetrics(
|
|
523
|
+
const metrics = responseMetrics(
|
|
524
|
+
router.view,
|
|
525
|
+
response,
|
|
526
|
+
tier,
|
|
527
|
+
tier ? (turn?.label ?? null) : null,
|
|
528
|
+
tier ? route : null,
|
|
529
|
+
);
|
|
446
530
|
await updateView($, router, metrics);
|
|
447
531
|
if (turn)
|
|
448
532
|
router.turnActivities.set(turnId, {
|
|
@@ -453,6 +537,91 @@ async function publishReply($, router, turnId, response, tier) {
|
|
|
453
537
|
});
|
|
454
538
|
}
|
|
455
539
|
|
|
540
|
+
// A routed reply written to the loop at `loopVersion`, then published: the view's metrics and the routing-vs-your-model
|
|
541
|
+
// totals. A lost write publishes nothing.
|
|
542
|
+
async function observeRouted($, router, { turnId, cfg, loop, loopVersion, request, response, nativeModel }) {
|
|
543
|
+
const observed = observeResponse(loop, {
|
|
544
|
+
usage: response.usage,
|
|
545
|
+
requestedModel: request.model,
|
|
546
|
+
effort: request.effort ?? null,
|
|
547
|
+
stopReason: response.stopReason,
|
|
548
|
+
now: Date.now(),
|
|
549
|
+
});
|
|
550
|
+
if (response.stopReason === null) observed.suspended = true;
|
|
551
|
+
const compared = await compareYours($, router, loop, response, request.effort ?? null, nativeModel, cfg);
|
|
552
|
+
if (compared) observed.yours = compared.state;
|
|
553
|
+
const written = await $.state.set({ ...LOOP, id: 'main' }, observed, { ifVersion: loopVersion });
|
|
554
|
+
if (!written.isSet) return;
|
|
555
|
+
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
556
|
+
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
557
|
+
await publishReply($, router, turnId, response, tier, `${request.model}@${request.effort ?? 'session'}`);
|
|
558
|
+
if (compared?.reply) await addSavings($, router, turnId, compared.reply, tier);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// The main conversation's prompt-cache lifetime by Claude Code's rule (sessionCacheTtl), or null when a reading
|
|
562
|
+
// fails. $.env.get takes literal names only. A plan subscriber is one whose replies report plan rate limits; a usage
|
|
563
|
+
// without them is not one.
|
|
564
|
+
async function cacheTtlOf($) {
|
|
565
|
+
try {
|
|
566
|
+
const settings = await $.settings.read();
|
|
567
|
+
const usage = await $.session.usage();
|
|
568
|
+
return sessionCacheTtl({
|
|
569
|
+
force5m: await $.env.get('FORCE_PROMPT_CACHING_5M'),
|
|
570
|
+
envTtl: await $.env.get('CLAUDE_CODE_PROMPT_CACHE_TTL'),
|
|
571
|
+
settingTtl: settings.promptCacheTtl,
|
|
572
|
+
enable1h: await $.env.get('ENABLE_PROMPT_CACHING_1H'),
|
|
573
|
+
subscriber: (usage?.rateLimits ?? []).some((limit) => limit.kind === 'five_hour' || limit.kind === 'seven_day'),
|
|
574
|
+
});
|
|
575
|
+
} catch {
|
|
576
|
+
return null;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// A main reply against your model (compareReply): the loop's next `yours` state and the reply's prices, or null. The
|
|
581
|
+
// readout never breaks a turn. A reply whose cache lifetime cannot be read still moves your model's cache and is not
|
|
582
|
+
// counted; any other failure leaves the loop's state and the totals as they were.
|
|
583
|
+
async function compareYours($, router, loop, response, servedEffort, nativeModel, cfg = router.config) {
|
|
584
|
+
try {
|
|
585
|
+
if (!response.usage?.model) return null;
|
|
586
|
+
const view = router.view ?? (await readView($, router));
|
|
587
|
+
return compareReply(cfg, loop.yours ?? null, {
|
|
588
|
+
usage: response.usage,
|
|
589
|
+
served: { model: response.usage.model, effort: servedEffort },
|
|
590
|
+
yours: { model: nativeModel, effort: view.nativeEffort ?? null },
|
|
591
|
+
ttl: await cacheTtlOf($),
|
|
592
|
+
now: Date.now(),
|
|
593
|
+
});
|
|
594
|
+
} catch {
|
|
595
|
+
return null;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// A routed reply's prices added to the session totals in the view and to the turn's totals for the store.
|
|
600
|
+
async function addSavings($, router, turnId, reply, tier) {
|
|
601
|
+
try {
|
|
602
|
+
const view = router.view ?? (await readView($, router));
|
|
603
|
+
router.turnSavings.set(turnId, addReply(router.turnSavings.get(turnId) ?? emptyTotals(), reply, tier));
|
|
604
|
+
await updateView($, router, { savings: addReply(view.savings ?? emptyTotals(), reply, tier) });
|
|
605
|
+
} catch {}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// A finished turn's totals added to this session's record kept across sessions, with the guard recordActivity uses:
|
|
609
|
+
// totals read before a reset are not written back. The record is added to in memory before the write, so a refresh
|
|
610
|
+
// during the write counts the turn once.
|
|
611
|
+
async function flushSavings($, router, totals) {
|
|
612
|
+
try {
|
|
613
|
+
const resets = router.statsResets;
|
|
614
|
+
const { own, resetAt } = await ownSavings($, router);
|
|
615
|
+
// The others held from before a reset in another session are read again.
|
|
616
|
+
if (!router.savingsOthers || afterReset(router.savingsOthers, resetAt) !== router.savingsOthers)
|
|
617
|
+
router.savingsOthers = await otherSavings($, own.sessionId, resetAt);
|
|
618
|
+
if (router.statsResets !== resets) return;
|
|
619
|
+
own.record = recordSavingsStore(own.record, totals, Date.now());
|
|
620
|
+
await $.store.set(`${SAVINGS_PREFIX}${own.sessionId}`, own.record);
|
|
621
|
+
if (router.statsResets === resets) await updateView($, router, savedView(router));
|
|
622
|
+
} catch {}
|
|
623
|
+
}
|
|
624
|
+
|
|
456
625
|
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
457
626
|
// break a turn: a failed read or write loses this turn's counts only.
|
|
458
627
|
async function recordActivity($, router, turn) {
|
|
@@ -479,6 +648,7 @@ async function recordActivity($, router, turn) {
|
|
|
479
648
|
tierSwitches: turn.switched === 'tier' ? 1 : 0,
|
|
480
649
|
activitySwitches: turn.switched === 'activity' ? 1 : 0,
|
|
481
650
|
wouldDiffer: turn.wouldDiffer,
|
|
651
|
+
wouldUsd: turn.wouldUsd,
|
|
482
652
|
shadow: turn.mode === 'shadow',
|
|
483
653
|
}),
|
|
484
654
|
});
|
|
@@ -486,16 +656,17 @@ async function recordActivity($, router, turn) {
|
|
|
486
656
|
const store = readStore(await $.store.get(STORE_KEY));
|
|
487
657
|
// Writing counts read before a reset would bring them back; this turn's are lost instead.
|
|
488
658
|
if (router.statsResets !== resets) return;
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
659
|
+
const written = recordStore(store, {
|
|
660
|
+
predicted: turn.predicted,
|
|
661
|
+
observed,
|
|
662
|
+
runEnded: ended,
|
|
663
|
+
lateral: turn.lateral,
|
|
664
|
+
wouldDiffer: turn.wouldDiffer,
|
|
665
|
+
wouldUsd: turn.wouldUsd,
|
|
666
|
+
});
|
|
667
|
+
await $.store.set(STORE_KEY, written);
|
|
668
|
+
// A reset during the write emptied the view; these counts are older than it.
|
|
669
|
+
if (router.statsResets === resets) await updateView($, router, { activityStore: written });
|
|
499
670
|
} catch {}
|
|
500
671
|
}
|
|
501
672
|
|
|
@@ -503,7 +674,7 @@ async function recordActivity($, router, turn) {
|
|
|
503
674
|
async function flushRun($, run) {
|
|
504
675
|
try {
|
|
505
676
|
const store = readStore(await $.store.get(STORE_KEY));
|
|
506
|
-
const ended = { predicted: null, observed: null, runEnded: run, lateral: null, wouldDiffer: null };
|
|
677
|
+
const ended = { predicted: null, observed: null, runEnded: run, lateral: null, wouldDiffer: null, wouldUsd: null };
|
|
507
678
|
await $.store.set(STORE_KEY, recordStore(store, ended));
|
|
508
679
|
} catch {}
|
|
509
680
|
}
|
|
@@ -657,16 +828,39 @@ function paneActions($, router, view) {
|
|
|
657
828
|
withActivities(file, { base: { activityRouting: from }, activityRouting: mode }),
|
|
658
829
|
);
|
|
659
830
|
},
|
|
660
|
-
|
|
831
|
+
// One reset for both readouts: activity stats and routing vs your model, this session's and the saved ones.
|
|
832
|
+
resetStats: async () => {
|
|
661
833
|
router.statsResets += 1;
|
|
662
834
|
router.run = null;
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
835
|
+
router.turnSavings.clear();
|
|
836
|
+
const clear = (key, value) =>
|
|
837
|
+
$.store
|
|
838
|
+
.set(key, value)
|
|
839
|
+
.then(() => true)
|
|
840
|
+
.catch(() => false);
|
|
841
|
+
const clearSavings = async () => {
|
|
842
|
+
// Written first, so a session still holding its record from before the reset drops it, unless that record's
|
|
843
|
+
// first reply has the reset's millisecond or the clock moved back across the reset.
|
|
844
|
+
await $.store.set(SAVINGS_RESET_KEY, Date.now());
|
|
845
|
+
const keys = (await $.store.keys()).filter((key) => key.startsWith(SAVINGS_PREFIX));
|
|
846
|
+
await Promise.all(keys.map((key) => $.store.delete(key)));
|
|
847
|
+
router.savingsOwn = null;
|
|
848
|
+
router.savingsOthers = emptySavingsStore();
|
|
849
|
+
return true;
|
|
850
|
+
};
|
|
851
|
+
const [activity, savings] = await Promise.all([
|
|
852
|
+
clear(STORE_KEY, emptyStore()),
|
|
853
|
+
clearSavings().catch(() => false),
|
|
854
|
+
]);
|
|
667
855
|
await updateView($, router, {
|
|
668
856
|
activityStats: null,
|
|
669
|
-
|
|
857
|
+
savings: null,
|
|
858
|
+
...(activity ? { activityStore: emptyStore() } : {}),
|
|
859
|
+
...(savings ? { savingsStore: emptySavingsStore() } : {}),
|
|
860
|
+
notice:
|
|
861
|
+
activity && savings
|
|
862
|
+
? 'Stats reset: activity and routing vs your model, this session and saved.'
|
|
863
|
+
: 'This session’s stats reset. Saved stats could not be cleared.',
|
|
670
864
|
});
|
|
671
865
|
},
|
|
672
866
|
discardRouting: () => updateView($, router, { routeDraft: null, tuning: null, tuningBase: null, notice: null }),
|
|
@@ -758,6 +952,8 @@ export function register(on, options) {
|
|
|
758
952
|
routeDraft: null,
|
|
759
953
|
lastWrite: null,
|
|
760
954
|
bandDetail: (await $.store.get(BAND_DETAIL).catch(() => false)) === true,
|
|
955
|
+
...(await storeView($, router)),
|
|
956
|
+
...savedView(router),
|
|
761
957
|
});
|
|
762
958
|
} catch (error) {
|
|
763
959
|
await updateView($, router, {
|
|
@@ -807,6 +1003,7 @@ export function register(on, options) {
|
|
|
807
1003
|
const saved = await rememberMode($, mode);
|
|
808
1004
|
await updateView($, router, {
|
|
809
1005
|
mode,
|
|
1006
|
+
nativeEffort: e.effort ?? null,
|
|
810
1007
|
...(!saved ? { notice: 'Resume preference could not be saved. Current mode remains active.' } : {}),
|
|
811
1008
|
});
|
|
812
1009
|
}
|
|
@@ -863,20 +1060,8 @@ export function register(on, options) {
|
|
|
863
1060
|
if (loop.decision.effort === null) delete request.effort;
|
|
864
1061
|
else request.effort = loop.decision.effort;
|
|
865
1062
|
const response = yield* next(request);
|
|
866
|
-
if (!next.signal.aborted && (await $.session.id()) === sessionId)
|
|
867
|
-
|
|
868
|
-
usage: response.usage,
|
|
869
|
-
requestedModel: request.model,
|
|
870
|
-
effort: request.effort ?? null,
|
|
871
|
-
stopReason: response.stopReason,
|
|
872
|
-
now: Date.now(),
|
|
873
|
-
});
|
|
874
|
-
if (response.stopReason === null) observed.suspended = true;
|
|
875
|
-
const written = await $.state.set(ref, observed, { ifVersion: loopVersion });
|
|
876
|
-
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
877
|
-
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
878
|
-
if (written.isSet) await publishReply($, router, e.turnId, response, tier);
|
|
879
|
-
}
|
|
1063
|
+
if (!next.signal.aborted && (await $.session.id()) === sessionId)
|
|
1064
|
+
await observeRouted($, router, { turnId: e.turnId, cfg, loop, loopVersion, request, response, nativeModel });
|
|
880
1065
|
return response;
|
|
881
1066
|
});
|
|
882
1067
|
|
|
@@ -895,6 +1080,9 @@ export function register(on, options) {
|
|
|
895
1080
|
const turn = router.turnActivities.get(e.turnId);
|
|
896
1081
|
router.turnActivities.delete(e.turnId);
|
|
897
1082
|
if (turn && turn.mode !== 'off') await recordActivity($, router, turn);
|
|
1083
|
+
const savings = router.turnSavings.get(e.turnId);
|
|
1084
|
+
router.turnSavings.delete(e.turnId);
|
|
1085
|
+
if (savings) await flushSavings($, router, savings);
|
|
898
1086
|
return next(e);
|
|
899
1087
|
});
|
|
900
1088
|
|
|
@@ -916,13 +1104,19 @@ export function register(on, options) {
|
|
|
916
1104
|
router.prompts.clear();
|
|
917
1105
|
router.turnConfigs.clear();
|
|
918
1106
|
router.turnActivities.clear();
|
|
1107
|
+
router.turnSavings.clear();
|
|
919
1108
|
const run = router.run;
|
|
920
1109
|
router.run = null;
|
|
921
1110
|
if (run) await flushRun($, run);
|
|
922
1111
|
router.view = {
|
|
923
1112
|
...initialView(router.view?.nativeModel ?? ''),
|
|
1113
|
+
// The effort carries over /clear until the next turn reads it again, so "your model" keeps its effort.
|
|
1114
|
+
nativeEffort: router.view?.nativeEffort ?? null,
|
|
924
1115
|
mode: 'auto',
|
|
925
1116
|
credentials: router.view?.credentials ?? null,
|
|
1117
|
+
// Counts kept across sessions: the next session starts from them.
|
|
1118
|
+
activityStore: router.view?.activityStore ?? null,
|
|
1119
|
+
savingsStore: router.view?.savingsStore ?? null,
|
|
926
1120
|
health: healthOf(clientOf(router, router.config.classifier), router.config.classifier),
|
|
927
1121
|
configPath: router.config.nativePath,
|
|
928
1122
|
phase: router.view?.phase === 'unavailable' ? 'unavailable' : 'ready',
|
|
@@ -989,6 +1183,9 @@ export function register(on, options) {
|
|
|
989
1183
|
const modelOptions = Object.keys(router.config.models).filter((alias) =>
|
|
990
1184
|
isModelAllowed(router.config.models[alias].id, settings.availableModels, view.nativeModel),
|
|
991
1185
|
);
|
|
992
|
-
return renderPanel(elements, router.config, view, usage, paneActions($, router, view), {
|
|
1186
|
+
return renderPanel(elements, router.config, view, usage, paneActions($, router, view), {
|
|
1187
|
+
modelOptions,
|
|
1188
|
+
columns: e.props.bodyColumns,
|
|
1189
|
+
});
|
|
993
1190
|
});
|
|
994
1191
|
}
|
package/lib/activity-stats.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
|
7
7
|
export const OBSERVED = ['code', 'docs', 'ops', 'read', 'talk'];
|
|
8
8
|
export const STORE_KEY = 'activity:stats:v1';
|
|
9
9
|
|
|
10
|
-
// The observed buckets each predicted activity allows. A weak oracle: it tells
|
|
10
|
+
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
11
11
|
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
12
12
|
export const ALLOWED = {
|
|
13
13
|
code: ['code'],
|
|
@@ -15,7 +15,8 @@ export const ALLOWED = {
|
|
|
15
15
|
explore: ['read', 'talk'],
|
|
16
16
|
plan: ['talk', 'read', 'docs'],
|
|
17
17
|
review: ['read', 'talk'],
|
|
18
|
-
|
|
18
|
+
// git status or a test listing is still running commands: a read-only Bash turn is ops too.
|
|
19
|
+
ops: ['ops', 'read'],
|
|
19
20
|
docs: ['docs'],
|
|
20
21
|
};
|
|
21
22
|
|
|
@@ -29,18 +30,34 @@ const CONFUSION_ROWS = [...ACTIVITY_VALUES, 'none'];
|
|
|
29
30
|
|
|
30
31
|
const count = (n) => (Number.isFinite(n) && n > 0 ? Math.floor(n) : 0);
|
|
31
32
|
|
|
33
|
+
// The shadow readout: turns in shadow, those `on` would have routed differently, and of those the ones with a
|
|
34
|
+
// next-request estimate (`estimated`) and the sum of their [min, max] ranges in USD at configured list prices.
|
|
35
|
+
const emptyShadow = () => ({ differs: 0, turns: 0, estimated: 0, minUsd: 0, maxUsd: 0 });
|
|
36
|
+
const isRange = (r) => Number.isFinite(r?.minUsd) && Number.isFinite(r?.maxUsd);
|
|
37
|
+
|
|
38
|
+
function addShadow(shadow, wouldDiffer, range) {
|
|
39
|
+
const priced = wouldDiffer && isRange(range);
|
|
40
|
+
return {
|
|
41
|
+
differs: shadow.differs + (wouldDiffer ? 1 : 0),
|
|
42
|
+
turns: shadow.turns + 1,
|
|
43
|
+
estimated: shadow.estimated + (priced ? 1 : 0),
|
|
44
|
+
minUsd: shadow.minUsd + (priced ? range.minUsd : 0),
|
|
45
|
+
maxUsd: shadow.maxUsd + (priced ? range.maxUsd : 0),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
32
49
|
export function emptySession() {
|
|
33
50
|
return {
|
|
34
51
|
byActivity: {},
|
|
35
52
|
switches: { tier: 0, activity: 0 },
|
|
36
53
|
agreement: { matched: 0, total: 0 },
|
|
37
|
-
shadow:
|
|
54
|
+
shadow: emptyShadow(),
|
|
38
55
|
};
|
|
39
56
|
}
|
|
40
57
|
|
|
41
58
|
// One finished turn. `activity` is the turn's label (null: none), `answer` the classifier's accepted activity, which
|
|
42
|
-
// agreement compares with the tools (null: not counted), `shadow` marks a turn in shadow mode
|
|
43
|
-
// whether 'on' would have routed it differently (null: no would-route).
|
|
59
|
+
// agreement compares with the tools (null: not counted), `shadow` marks a turn in shadow mode, `wouldDiffer` says
|
|
60
|
+
// whether 'on' would have routed it differently (null: no would-route) and `wouldUsd` prices that difference.
|
|
44
61
|
export function recordTurn(
|
|
45
62
|
session,
|
|
46
63
|
{
|
|
@@ -53,6 +70,7 @@ export function recordTurn(
|
|
|
53
70
|
tierSwitches,
|
|
54
71
|
activitySwitches,
|
|
55
72
|
wouldDiffer,
|
|
73
|
+
wouldUsd,
|
|
56
74
|
shadow,
|
|
57
75
|
},
|
|
58
76
|
) {
|
|
@@ -78,15 +96,15 @@ export function recordTurn(
|
|
|
78
96
|
total: session.agreement.total + 1,
|
|
79
97
|
}
|
|
80
98
|
: session.agreement,
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
99
|
+
// A view saved by 1.6.0 has no estimate fields.
|
|
100
|
+
shadow: shadow
|
|
101
|
+
? addShadow({ ...emptyShadow(), ...session.shadow }, wouldDiffer === true, wouldUsd)
|
|
102
|
+
: { ...emptyShadow(), ...session.shadow },
|
|
85
103
|
};
|
|
86
104
|
}
|
|
87
105
|
|
|
88
106
|
export function emptyStore() {
|
|
89
|
-
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow:
|
|
107
|
+
return { version: 1, confusion: {}, runs: {}, lateral: { taken: 0, refused: 0 }, shadow: emptyShadow() };
|
|
90
108
|
}
|
|
91
109
|
|
|
92
110
|
const isCount = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
@@ -96,7 +114,20 @@ const countsOf = (v, keys) => onlyKeys(v, keys) && Object.values(v).every(isCoun
|
|
|
96
114
|
const exactCountsOf = (v, keys) =>
|
|
97
115
|
isRecord(v) && Object.keys(v).length === keys.length && keys.every((k) => isCount(v[k]));
|
|
98
116
|
|
|
99
|
-
|
|
117
|
+
const SHADOW_COUNTS = ['differs', 'turns'];
|
|
118
|
+
const SHADOW_ESTIMATE = ['estimated', 'minUsd', 'maxUsd'];
|
|
119
|
+
// 1.6.0 kept the shadow counts only; 1.6.1 adds the estimate. Either shape, never a part of the estimate.
|
|
120
|
+
const isShadow = (v) =>
|
|
121
|
+
exactCountsOf(v, SHADOW_COUNTS) ||
|
|
122
|
+
(onlyKeys(v, [...SHADOW_COUNTS, ...SHADOW_ESTIMATE]) &&
|
|
123
|
+
Object.keys(v).length === 5 &&
|
|
124
|
+
SHADOW_COUNTS.every((k) => isCount(v[k])) &&
|
|
125
|
+
isCount(v.estimated) &&
|
|
126
|
+
Number.isFinite(v.minUsd) &&
|
|
127
|
+
Number.isFinite(v.maxUsd));
|
|
128
|
+
|
|
129
|
+
// The stored value if it has a known shape, filled to the current one, else an empty store: a hand-edited or
|
|
130
|
+
// unknown file must not break stats.
|
|
100
131
|
export function readStore(value) {
|
|
101
132
|
const ok =
|
|
102
133
|
onlyKeys(value, ['version', 'confusion', 'runs', 'lateral', 'shadow']) &&
|
|
@@ -108,14 +139,14 @@ export function readStore(value) {
|
|
|
108
139
|
(lengths) => Array.isArray(lengths) && lengths.length === RUN_BUCKETS && lengths.every(isCount),
|
|
109
140
|
) &&
|
|
110
141
|
exactCountsOf(value.lateral, ['taken', 'refused']) &&
|
|
111
|
-
|
|
112
|
-
return ok ? value : emptyStore();
|
|
142
|
+
isShadow(value.shadow);
|
|
143
|
+
return ok ? { ...value, shadow: { ...emptyShadow(), ...value.shadow } } : emptyStore();
|
|
113
144
|
}
|
|
114
145
|
|
|
115
146
|
// One finished turn. `predicted` is the classifier's answer (null: none), `runEnded` a run that closed this turn,
|
|
116
|
-
// `lateral` the outcome of a lateral switch decision
|
|
117
|
-
// Unknown labels are dropped, so the store keeps a fixed set of keys.
|
|
118
|
-
export function recordStore(store, { predicted, observed, runEnded, lateral, wouldDiffer }) {
|
|
147
|
+
// `lateral` the outcome of a lateral switch decision, `wouldDiffer` the shadow readout (null: not in shadow) and
|
|
148
|
+
// `wouldUsd` its estimate. Unknown labels are dropped, so the store keeps a fixed set of keys.
|
|
149
|
+
export function recordStore(store, { predicted, observed, runEnded, lateral, wouldDiffer, wouldUsd }) {
|
|
119
150
|
const row = predicted ?? 'none';
|
|
120
151
|
const next = {
|
|
121
152
|
version: 1,
|
|
@@ -132,10 +163,7 @@ export function recordStore(store, { predicted, observed, runEnded, lateral, wou
|
|
|
132
163
|
next.runs[runEnded.activity] = lengths;
|
|
133
164
|
}
|
|
134
165
|
if (lateral === 'taken' || lateral === 'refused') next.lateral[lateral] += 1;
|
|
135
|
-
if (typeof wouldDiffer === 'boolean')
|
|
136
|
-
next.shadow.turns += 1;
|
|
137
|
-
if (wouldDiffer) next.shadow.differs += 1;
|
|
138
|
-
}
|
|
166
|
+
if (typeof wouldDiffer === 'boolean') next.shadow = addShadow(next.shadow, wouldDiffer, wouldUsd);
|
|
139
167
|
return next;
|
|
140
168
|
}
|
|
141
169
|
|
|
@@ -166,6 +194,7 @@ export function turnActivity(config, advice, previous, decision) {
|
|
|
166
194
|
: 'tier'
|
|
167
195
|
: null;
|
|
168
196
|
const answer = mode === 'off' ? null : acceptActivity(config, advice);
|
|
197
|
+
const wouldDiffer = mode === 'shadow' ? Boolean(would && differs(would, decision)) : null;
|
|
169
198
|
return {
|
|
170
199
|
mode,
|
|
171
200
|
answer,
|
|
@@ -174,6 +203,43 @@ export function turnActivity(config, advice, previous, decision) {
|
|
|
174
203
|
switched,
|
|
175
204
|
lateral: LATERAL[(mode === 'shadow' ? would : mode === 'on' ? decision : null)?.reason] ?? null,
|
|
176
205
|
// A shadow turn always counts, so the session and store denominators agree.
|
|
177
|
-
wouldDiffer
|
|
206
|
+
wouldDiffer,
|
|
207
|
+
wouldUsd: wouldDiffer ? (would.difference ?? null) : null,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// The activities whose agreement plan §11 tracks: the costly mistake is a cheap route on a coding turn.
|
|
212
|
+
const FOCUS = ['code', 'ops', 'explore'];
|
|
213
|
+
|
|
214
|
+
// The counts kept across sessions as the Usage tab and /router show them. Agreement covers the rows that name an
|
|
215
|
+
// activity: the store holds the raw answer, and `uncertain` or no answer never agrees; `focus` is the same over the
|
|
216
|
+
// FOCUS rows only. The two largest disagreements come first; ties keep the matrix order, so the readout is stable.
|
|
217
|
+
export function storeSummary(store) {
|
|
218
|
+
let labelled = 0;
|
|
219
|
+
let matched = 0;
|
|
220
|
+
let total = 0;
|
|
221
|
+
const focus = { matched: 0, total: 0 };
|
|
222
|
+
const misses = [];
|
|
223
|
+
for (const predicted of CONFUSION_ROWS)
|
|
224
|
+
for (const observed of OBSERVED) {
|
|
225
|
+
const n = store.confusion[predicted]?.[observed] ?? 0;
|
|
226
|
+
labelled += n;
|
|
227
|
+
if (!n || !ALLOWED[predicted]) continue;
|
|
228
|
+
total += n;
|
|
229
|
+
const agreed = agrees(predicted, observed);
|
|
230
|
+
if (agreed) matched += n;
|
|
231
|
+
else misses.push({ predicted, observed, n });
|
|
232
|
+
if (FOCUS.includes(predicted)) {
|
|
233
|
+
focus.total += n;
|
|
234
|
+
if (agreed) focus.matched += n;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
labelled,
|
|
239
|
+
agreement: { matched, total },
|
|
240
|
+
focus,
|
|
241
|
+
disagreements: misses.sort((a, b) => b.n - a.n).slice(0, 2),
|
|
242
|
+
lateral: store.lateral,
|
|
243
|
+
shadow: store.shadow,
|
|
178
244
|
};
|
|
179
245
|
}
|