@alexeiled/claude-router 1.15.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +30 -0
- package/README.md +1 -1
- package/docs/architecture.md +9 -5
- package/docs/evaluation.md +15 -11
- package/docs/native-router.md +1 -0
- package/docs/routing-modes.md +8 -4
- package/docs/user-guide.md +14 -3
- package/hooks/native-router.mjs +286 -22
- package/lib/activity-stats.mjs +33 -11
- package/lib/activity.mjs +4 -4
- package/lib/band.mjs +2 -1
- package/lib/config.mjs +6 -3
- package/lib/decisions.mjs +252 -0
- package/lib/display.mjs +30 -4
- package/lib/facts.mjs +125 -4
- package/lib/panel.mjs +3 -0
- package/lib/policy.mjs +7 -4
- package/lib/route.mjs +4 -3
- package/lib/savings.mjs +17 -0
- package/package.json +1 -1
package/hooks/native-router.mjs
CHANGED
|
@@ -43,6 +43,24 @@ import {
|
|
|
43
43
|
withTuning,
|
|
44
44
|
} from '../lib/config.mjs';
|
|
45
45
|
import { changedLeaves, notSaved, restored, rewrittenConfig } from '../lib/config-file.mjs';
|
|
46
|
+
import {
|
|
47
|
+
addTurnRow,
|
|
48
|
+
appendDecisions,
|
|
49
|
+
DECISIONS_PREFIX,
|
|
50
|
+
decisionLines,
|
|
51
|
+
decisionRow,
|
|
52
|
+
decisionsAfterReset,
|
|
53
|
+
endRow,
|
|
54
|
+
isGoalOrigin,
|
|
55
|
+
isProgressOrigin,
|
|
56
|
+
MAX_ROWS,
|
|
57
|
+
originKind,
|
|
58
|
+
progressInput,
|
|
59
|
+
prunedDecisions,
|
|
60
|
+
queueOrigin,
|
|
61
|
+
readDecisions,
|
|
62
|
+
takeOrigin,
|
|
63
|
+
} from '../lib/decisions.mjs';
|
|
46
64
|
import {
|
|
47
65
|
activityDetailLines,
|
|
48
66
|
classifierStatus,
|
|
@@ -54,10 +72,12 @@ import {
|
|
|
54
72
|
savingsLine,
|
|
55
73
|
storeAgreementLine,
|
|
56
74
|
unavailableReason,
|
|
75
|
+
unpricedLine,
|
|
57
76
|
} from '../lib/display.mjs';
|
|
58
|
-
import { clip, observedActivity, promptIndex } from '../lib/facts.mjs';
|
|
77
|
+
import { clip, observedActivity, promptIndex, routingBucket } from '../lib/facts.mjs';
|
|
59
78
|
import { applyMode, modeLabel, readProjectMode, resolveRoutingMode } from '../lib/modes.mjs';
|
|
60
79
|
import { renderPanel, routeDraftOf, routingChanges } from '../lib/panel.mjs';
|
|
80
|
+
import { sameTaskOf } from '../lib/policy.mjs';
|
|
61
81
|
import {
|
|
62
82
|
canMoveUp,
|
|
63
83
|
cellForModel,
|
|
@@ -87,6 +107,7 @@ import {
|
|
|
87
107
|
SAVINGS_RESET_KEY,
|
|
88
108
|
sessionCacheTtl,
|
|
89
109
|
subagentCacheTtl,
|
|
110
|
+
unpricedCause,
|
|
90
111
|
} from '../lib/savings.mjs';
|
|
91
112
|
import {
|
|
92
113
|
addAgentReply,
|
|
@@ -209,6 +230,19 @@ function createRouter(options) {
|
|
|
209
230
|
clients: new Map(),
|
|
210
231
|
prompts: new Map(),
|
|
211
232
|
decisions: new Map(),
|
|
233
|
+
// Where each turn's prompt came from (originKind), matched at turn.start: the submission running its prompt.submit
|
|
234
|
+
// (`submitting`), else the oldest queued one with the turn's text (`origins`, queueOrigin).
|
|
235
|
+
submitting: null,
|
|
236
|
+
origins: [],
|
|
237
|
+
turnOrigins: new Map(),
|
|
238
|
+
// The session's active goal, the last prompt from a goal origin, clipped, `{ sessionId, text }`, in memory only;
|
|
239
|
+
// and each progress turn's report (`progressNotes`), which the classifier reads as dialogue (progressInput).
|
|
240
|
+
goal: null,
|
|
241
|
+
progressNotes: new Map(),
|
|
242
|
+
// Each turn's decision rows until turn.complete writes them, with the turn's end, to the session's record, held as
|
|
243
|
+
// `{ sessionId, record }` once read (`decisionRecord`).
|
|
244
|
+
turnDecisions: new Map(),
|
|
245
|
+
decisionRecord: null,
|
|
212
246
|
controllers: new Set(),
|
|
213
247
|
turnControllers: new Map(),
|
|
214
248
|
view: null,
|
|
@@ -544,6 +578,7 @@ async function routerCommand($, router, args) {
|
|
|
544
578
|
await changeMode($, router, action === 'auto' ? 'auto' : 'manual');
|
|
545
579
|
return { text: action === 'auto' ? 'Routing on.' : 'Routing off: Claude’s model is kept.' };
|
|
546
580
|
}
|
|
581
|
+
if (action === 'decisions') return { text: await decisionsText($, router, value) };
|
|
547
582
|
if (action === 'pin')
|
|
548
583
|
return { text: TIERS.includes(value) ? await setPin($, router, value) : `Choose ${TIERS.join(', ')}.` };
|
|
549
584
|
if (action === 'mode')
|
|
@@ -680,6 +715,7 @@ function detailText(config, view) {
|
|
|
680
715
|
`Reason: ${view.reason}`,
|
|
681
716
|
...activityDetailLines(config, view),
|
|
682
717
|
storeAgreementLine(view),
|
|
718
|
+
...[unpricedLine(view.activityMetrics)].filter(Boolean),
|
|
683
719
|
savingsLine(view),
|
|
684
720
|
view.error || missing ? classifierStatus(config, view.error ?? missing) : `${activeClassifier(config).label} ready`,
|
|
685
721
|
`Context: ${view.contextKnown ? `${view.contextTokens} tokens (estimate)` : 'unknown'}`,
|
|
@@ -759,13 +795,23 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
759
795
|
$.session.messages({ as: 'api' }),
|
|
760
796
|
]);
|
|
761
797
|
if (!(await live())) return null;
|
|
762
|
-
const
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
798
|
+
const origin = router.turnOrigins.get(e.turnId) ?? 'unknown';
|
|
799
|
+
// A progress turn is the same task (sameTaskOf).
|
|
800
|
+
const facts = {
|
|
801
|
+
...nativeFacts(cfg, loop, {
|
|
802
|
+
messages,
|
|
803
|
+
prompt: router.prompts.get(e.turnId),
|
|
804
|
+
effort: e.effort,
|
|
805
|
+
turnId: e.turnId,
|
|
806
|
+
contextTokens: context.tokens,
|
|
807
|
+
}),
|
|
808
|
+
progress: isProgressOrigin(origin),
|
|
809
|
+
};
|
|
810
|
+
const asked = progressInput(
|
|
811
|
+
facts,
|
|
812
|
+
{ goal: router.prompts.get(e.turnId), note: router.progressNotes.get(e.turnId) },
|
|
813
|
+
cfg.context.recentTurns,
|
|
814
|
+
);
|
|
769
815
|
await updateView($, router, {
|
|
770
816
|
phase: 'choosing',
|
|
771
817
|
activeTurnId: e.turnId,
|
|
@@ -781,8 +827,8 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
781
827
|
config: cfg,
|
|
782
828
|
apiKey: credentials.apiKey,
|
|
783
829
|
endpoint: credentials.endpoint,
|
|
784
|
-
prompt:
|
|
785
|
-
turns:
|
|
830
|
+
prompt: asked.prompt,
|
|
831
|
+
turns: asked.turns,
|
|
786
832
|
signal: controller.signal,
|
|
787
833
|
});
|
|
788
834
|
// The wait for advice alone, comparable with the activity probe's p95; the routing after it is not the classifier's.
|
|
@@ -811,11 +857,27 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
811
857
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
812
858
|
const activity = turnActivity(cfg, result.advice, previous, selected.decision);
|
|
813
859
|
const routingMode = step.cfg.routingMode;
|
|
860
|
+
addDecision(
|
|
861
|
+
router,
|
|
862
|
+
e.turnId,
|
|
863
|
+
decisionRow({
|
|
864
|
+
at: Date.now(),
|
|
865
|
+
kind: 'start',
|
|
866
|
+
origin,
|
|
867
|
+
mode: pin ? null : routingMode.mode,
|
|
868
|
+
advice: result.advice,
|
|
869
|
+
error: result.error,
|
|
870
|
+
decision: selected.decision,
|
|
871
|
+
facts,
|
|
872
|
+
sameTask: sameTaskOf(cfg, result.advice, facts),
|
|
873
|
+
}),
|
|
874
|
+
);
|
|
814
875
|
// One line per decision in Claude Code's debug log, so a later replay can compare modes. No prompt text.
|
|
815
876
|
$.ui.log(
|
|
816
877
|
`router decision ${JSON.stringify({
|
|
817
878
|
mode: routingMode.mode,
|
|
818
879
|
modeSource: routingMode.source,
|
|
880
|
+
origin,
|
|
819
881
|
classified: result.advice?.choice ?? null,
|
|
820
882
|
tier: selected.decision.tier,
|
|
821
883
|
reason: selected.decision.reason,
|
|
@@ -842,6 +904,7 @@ async function decideTurn($, router, e, signal, step) {
|
|
|
842
904
|
outputTokens: 0,
|
|
843
905
|
routedUsd: null,
|
|
844
906
|
pricedRequests: 0,
|
|
907
|
+
unpriced: {},
|
|
845
908
|
routes: {},
|
|
846
909
|
});
|
|
847
910
|
await updateView($, router, {
|
|
@@ -912,9 +975,23 @@ async function moveOnObserved($, router, cfg, loop, e, context) {
|
|
|
912
975
|
const messages = await transcript($);
|
|
913
976
|
if (!messages) return loop;
|
|
914
977
|
const facts = stepFacts(cfg, router, loop, e, messages, context);
|
|
915
|
-
const observed =
|
|
978
|
+
const observed = routingBucket(
|
|
979
|
+
observedActivity(messages, Math.max(promptIndex(messages), (loop.observedFrom ?? 0) - 1)),
|
|
980
|
+
);
|
|
916
981
|
const moved = observedMove(cfg, loop, { observed, facts, contextKnown: context.known, now: Date.now() });
|
|
917
982
|
if (moved === loop) return loop;
|
|
983
|
+
addDecision(
|
|
984
|
+
router,
|
|
985
|
+
e.turnId,
|
|
986
|
+
decisionRow({
|
|
987
|
+
at: Date.now(),
|
|
988
|
+
kind: 'observed',
|
|
989
|
+
origin: router.turnOrigins.get(e.turnId) ?? 'unknown',
|
|
990
|
+
mode: cfg.routingMode?.mode ?? null,
|
|
991
|
+
decision: moved.decision,
|
|
992
|
+
facts,
|
|
993
|
+
}),
|
|
994
|
+
);
|
|
918
995
|
$.ui.toast(switchToast(cfg, loop.decision, moved.decision, moved.decision.estimate));
|
|
919
996
|
// Replies from here on are labelled with the moved activity; the classifier's answer stays as recorded.
|
|
920
997
|
const turn = router.turnActivities.get(e.turnId);
|
|
@@ -930,7 +1007,7 @@ async function moveOnObserved($, router, cfg, loop, e, context) {
|
|
|
930
1007
|
// Inside a turn the classifier is asked again every `policy.reviewEveryRequests` requests, in the background: a step
|
|
931
1008
|
// never waits for it. The answer applies at the first step after it arrives, as a decision on the same task
|
|
932
1009
|
// (sameTaskOf), so the tier and the activity can move either way: up on the usual votes, down only past the dwell and a
|
|
933
|
-
//
|
|
1010
|
+
// checkpoint (completedSince). Pinned turns and a native fallback are left alone; a failed read or ask changes nothing.
|
|
934
1011
|
async function reviewMidTurn($, router, cfg, loop, e, { context, nativeModel, availableModels }) {
|
|
935
1012
|
const review = router.reviews.get(e.turnId);
|
|
936
1013
|
if (review?.answer) {
|
|
@@ -952,6 +1029,20 @@ async function reviewMidTurn($, router, cfg, loop, e, { context, nativeModel, av
|
|
|
952
1029
|
now: Date.now(),
|
|
953
1030
|
});
|
|
954
1031
|
const previous = loop.decision;
|
|
1032
|
+
addDecision(
|
|
1033
|
+
router,
|
|
1034
|
+
e.turnId,
|
|
1035
|
+
decisionRow({
|
|
1036
|
+
at: Date.now(),
|
|
1037
|
+
kind: 'review',
|
|
1038
|
+
origin: router.turnOrigins.get(e.turnId) ?? 'unknown',
|
|
1039
|
+
mode: cfg.routingMode?.mode ?? null,
|
|
1040
|
+
advice,
|
|
1041
|
+
decision: selected.decision,
|
|
1042
|
+
facts,
|
|
1043
|
+
sameTask: sameTaskOf(cfg, advice, facts),
|
|
1044
|
+
}),
|
|
1045
|
+
);
|
|
955
1046
|
if (previous && (previous.model !== selected.decision.model || previous.effort !== selected.decision.effort)) {
|
|
956
1047
|
$.ui.toast(switchToast(cfg, previous, selected.decision, selected.decision.estimate));
|
|
957
1048
|
const turn = router.turnActivities.get(e.turnId);
|
|
@@ -1012,7 +1103,7 @@ async function reviewMidTurn($, router, cfg, loop, e, { context, nativeModel, av
|
|
|
1012
1103
|
}
|
|
1013
1104
|
|
|
1014
1105
|
// A routed reply's metrics in the view, with the tier that served it, and added to the turn's activity record with its list price (`routedUsd`, null when unpriced) and, when its tier served it, its route.
|
|
1015
|
-
async function publishReply($, router, turnId, response, tier, route, routedUsd) {
|
|
1106
|
+
async function publishReply($, router, turnId, response, tier, route, routedUsd, cause = null) {
|
|
1016
1107
|
const turn = router.turnActivities.get(turnId);
|
|
1017
1108
|
const metrics = responseMetrics(router.view, response, tier, Date.now());
|
|
1018
1109
|
await updateView($, router, metrics);
|
|
@@ -1024,6 +1115,7 @@ async function publishReply($, router, turnId, response, tier, route, routedUsd)
|
|
|
1024
1115
|
outputTokens: turn.outputTokens + (metrics.outputTokens ?? 0),
|
|
1025
1116
|
routedUsd: Number.isFinite(routedUsd) ? (turn.routedUsd ?? 0) + routedUsd : turn.routedUsd,
|
|
1026
1117
|
pricedRequests: turn.pricedRequests + (Number.isFinite(routedUsd) ? 1 : 0),
|
|
1118
|
+
unpriced: cause ? { ...turn.unpriced, [cause]: (turn.unpriced?.[cause] ?? 0) + 1 } : turn.unpriced,
|
|
1027
1119
|
routes: tier ? { ...turn.routes, [route]: (turn.routes[route] ?? 0) + 1 } : turn.routes,
|
|
1028
1120
|
});
|
|
1029
1121
|
}
|
|
@@ -1048,7 +1140,20 @@ async function observeRouted($, router, { turnId, cfg, loop, loopVersion, reques
|
|
|
1048
1140
|
// A substituted reply is not the tier's: the strip and trend must not count it as one.
|
|
1049
1141
|
const tier = isSameModel(request.model, response.usage?.model) ? loop.decision.tier : null;
|
|
1050
1142
|
const route = `${request.model}@${request.effort ?? 'session'}`;
|
|
1051
|
-
|
|
1143
|
+
// Why the usage record leaves this reply out, when it does (UNPRICED_CAUSES).
|
|
1144
|
+
const cause = !compared
|
|
1145
|
+
? response.usage?.model
|
|
1146
|
+
? 'error'
|
|
1147
|
+
: 'usage'
|
|
1148
|
+
: compared.reply
|
|
1149
|
+
? null
|
|
1150
|
+
: (unpricedCause(cfg, {
|
|
1151
|
+
usage: response.usage,
|
|
1152
|
+
served: { model: response.usage.model, effort: request.effort ?? null },
|
|
1153
|
+
yours: { model: nativeModel },
|
|
1154
|
+
ttl: compared.ttl,
|
|
1155
|
+
}) ?? 'error');
|
|
1156
|
+
await publishReply($, router, turnId, response, tier, route, compared?.reply?.routedUsd ?? null, cause);
|
|
1052
1157
|
// Kept by model id, under the model that served it: a substitute's cost is not the requested model's, and a models
|
|
1053
1158
|
// key can name another model later.
|
|
1054
1159
|
const idOf = (model) => modelSpec(cfg, model)?.id ?? model;
|
|
@@ -1092,14 +1197,16 @@ async function compareYours($, router, loop, response, servedEffort, nativeModel
|
|
|
1092
1197
|
try {
|
|
1093
1198
|
if (!response.usage?.model) return null;
|
|
1094
1199
|
const view = router.view ?? (await readView($, router));
|
|
1095
|
-
|
|
1200
|
+
const ttl = await cacheTtlOf($);
|
|
1201
|
+
const compared = compareModels(cfg, loop.yoursBy ?? {}, {
|
|
1096
1202
|
usage: response.usage,
|
|
1097
1203
|
served: { model: response.usage.model, effort: servedEffort },
|
|
1098
1204
|
effort: view.nativeEffort ?? null,
|
|
1099
|
-
ttl
|
|
1205
|
+
ttl,
|
|
1100
1206
|
now: Date.now(),
|
|
1101
1207
|
yours: nativeModel,
|
|
1102
1208
|
});
|
|
1209
|
+
return { ...compared, ttl };
|
|
1103
1210
|
} catch {
|
|
1104
1211
|
return null;
|
|
1105
1212
|
}
|
|
@@ -1138,6 +1245,140 @@ async function flushUsage($, router, entries, switched) {
|
|
|
1138
1245
|
} catch {}
|
|
1139
1246
|
}
|
|
1140
1247
|
|
|
1248
|
+
// A turn's prompt, its origin and the classifier's request text. A progress turn asks about the goal it serves, with
|
|
1249
|
+
// its report as dialogue (progressInput); without a goal from this session it reads as before. A goal origin's prompt
|
|
1250
|
+
// becomes the goal.
|
|
1251
|
+
async function startTurn($, router, e) {
|
|
1252
|
+
const text = clip(e.text, router.config.context.maxTextChars);
|
|
1253
|
+
const origin = originOfTurn(router, e.text);
|
|
1254
|
+
router.turnOrigins.set(e.turnId, origin);
|
|
1255
|
+
const sessionId = await $.session.id();
|
|
1256
|
+
if (isProgressOrigin(origin) && router.goal?.sessionId === sessionId) {
|
|
1257
|
+
router.prompts.set(e.turnId, router.goal.text);
|
|
1258
|
+
router.progressNotes.set(e.turnId, text);
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
router.prompts.set(e.turnId, text);
|
|
1262
|
+
if (isGoalOrigin(origin)) router.goal = { sessionId, text };
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// The engine stamps each prompt with where it came from (PromptOrigin); turn.start does not carry it. An idle
|
|
1266
|
+
// submission starts its turn inside next(e), so the turn takes `submitting`; one queued behind a running turn starts
|
|
1267
|
+
// later, or is delivered into it, and waits in `origins` under the text that entered. Bookkeeping only: the prompt
|
|
1268
|
+
// passes on unchanged, and nothing here throws after next(e).
|
|
1269
|
+
async function submitPrompt(router, e, next) {
|
|
1270
|
+
const entry = { kind: originKind(e.origin), started: false };
|
|
1271
|
+
const outer = router.submitting;
|
|
1272
|
+
router.submitting = entry;
|
|
1273
|
+
try {
|
|
1274
|
+
const result = await next(e);
|
|
1275
|
+
if (!entry.started && typeof result?.text === 'string' && !result.drop)
|
|
1276
|
+
router.origins = queueOrigin(router.origins, { kind: entry.kind, text: result.text, at: Date.now() });
|
|
1277
|
+
return result;
|
|
1278
|
+
} finally {
|
|
1279
|
+
router.submitting = outer;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
// One decision row added to the turn's, bounded per turn (addTurnRow).
|
|
1284
|
+
function addDecision(router, turnId, row) {
|
|
1285
|
+
router.turnDecisions.set(turnId, addTurnRow(router.turnDecisions.get(turnId) ?? [], row));
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
// The origin of a turn starting with `text`: the submission whose next(e) is running, once, else the oldest queued one
|
|
1289
|
+
// with that text, else 'unknown'.
|
|
1290
|
+
function originOfTurn(router, text) {
|
|
1291
|
+
if (router.submitting && !router.submitting.started) {
|
|
1292
|
+
router.submitting.started = true;
|
|
1293
|
+
return router.submitting.kind;
|
|
1294
|
+
}
|
|
1295
|
+
const taken = takeOrigin(router.origins, text, Date.now());
|
|
1296
|
+
router.origins = taken.queue;
|
|
1297
|
+
return taken.kind;
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
// A finished turn's decision rows and its end row, added to this session's record. Telemetry never breaks a turn: a
|
|
1301
|
+
// failed read or write loses this turn's rows. A record from before the last reset reads as empty, and rows read
|
|
1302
|
+
// before a Reset in this session are not written back.
|
|
1303
|
+
async function flushDecisions($, router, turnId, turn) {
|
|
1304
|
+
const rows = router.turnDecisions.get(turnId) ?? [];
|
|
1305
|
+
router.turnDecisions.delete(turnId);
|
|
1306
|
+
router.turnOrigins.delete(turnId);
|
|
1307
|
+
router.progressNotes.delete(turnId);
|
|
1308
|
+
if (!rows.length) return;
|
|
1309
|
+
try {
|
|
1310
|
+
const resets = router.statsResets;
|
|
1311
|
+
const sessionId = await $.session.id();
|
|
1312
|
+
if (turn && turn.sessionId !== sessionId) return;
|
|
1313
|
+
const key = `${DECISIONS_PREFIX}${sessionId}`;
|
|
1314
|
+
const resetAt = await $.store.get(SAVINGS_RESET_KEY);
|
|
1315
|
+
if (router.decisionRecord?.sessionId !== sessionId)
|
|
1316
|
+
router.decisionRecord = { sessionId, record: readDecisions(await $.store.get(key)) };
|
|
1317
|
+
const held = router.decisionRecord;
|
|
1318
|
+
const now = Date.now();
|
|
1319
|
+
const end = turn
|
|
1320
|
+
? [
|
|
1321
|
+
endRow({
|
|
1322
|
+
at: now,
|
|
1323
|
+
// With activity routing off recordActivity does not run, so the bucket is read here.
|
|
1324
|
+
observed: turn.observed ?? (await turnBucket($)),
|
|
1325
|
+
requests: turn.requests,
|
|
1326
|
+
priced: turn.pricedRequests,
|
|
1327
|
+
unpriced: turn.unpriced,
|
|
1328
|
+
}),
|
|
1329
|
+
]
|
|
1330
|
+
: [];
|
|
1331
|
+
const record = appendDecisions(decisionsAfterReset(held.record, resetAt), [...rows, ...end], now);
|
|
1332
|
+
if (router.statsResets !== resets || router.decisionRecord !== held) return;
|
|
1333
|
+
held.record = record;
|
|
1334
|
+
await $.store.set(key, record);
|
|
1335
|
+
} catch {}
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
// The bucket of the turn the transcript ends with (observedActivity), or null when it cannot be read.
|
|
1339
|
+
async function turnBucket($) {
|
|
1340
|
+
const messages = await transcript($);
|
|
1341
|
+
const start = messages ? promptIndex(messages) : -1;
|
|
1342
|
+
return start < 0 ? null : observedActivity(messages, start);
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
// Other sessions' decision records past their retention or the record bound (prunedDecisions), each read again just
|
|
1346
|
+
// before its delete, so a session that wrote meanwhile keeps its record.
|
|
1347
|
+
async function pruneDecisionRecords($) {
|
|
1348
|
+
try {
|
|
1349
|
+
const own = `${DECISIONS_PREFIX}${await $.session.id()}`;
|
|
1350
|
+
const keys = (await $.store.keys()).filter((key) => key.startsWith(DECISIONS_PREFIX) && key !== own);
|
|
1351
|
+
const values = await Promise.all(keys.map((key) => $.store.get(key)));
|
|
1352
|
+
const gone = prunedDecisions(
|
|
1353
|
+
keys.map((key, i) => [key, values[i]]),
|
|
1354
|
+
Date.now(),
|
|
1355
|
+
);
|
|
1356
|
+
await Promise.allSettled(
|
|
1357
|
+
[...gone].map(async (key) => {
|
|
1358
|
+
const current = await $.store.get(key);
|
|
1359
|
+
if (JSON.stringify(current) === JSON.stringify(values[keys.indexOf(key)])) await $.store.delete(key);
|
|
1360
|
+
}),
|
|
1361
|
+
);
|
|
1362
|
+
} catch {}
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
// `/router decisions [n]`: this session's newest routing decisions, the running turn's included.
|
|
1366
|
+
async function decisionsText($, router, value) {
|
|
1367
|
+
const n = Math.min(MAX_ROWS, Math.max(1, Number.parseInt(value, 10) || 20));
|
|
1368
|
+
try {
|
|
1369
|
+
const sessionId = await $.session.id();
|
|
1370
|
+
const stored =
|
|
1371
|
+
router.decisionRecord?.sessionId === sessionId
|
|
1372
|
+
? router.decisionRecord.record
|
|
1373
|
+
: readDecisions(await $.store.get(`${DECISIONS_PREFIX}${sessionId}`));
|
|
1374
|
+
const resetAt = await $.store.get(SAVINGS_RESET_KEY);
|
|
1375
|
+
const pending = [...router.turnDecisions.values()].flat();
|
|
1376
|
+
return decisionLines([...decisionsAfterReset(stored, resetAt).rows, ...pending], n).join('\n');
|
|
1377
|
+
} catch {
|
|
1378
|
+
return 'Routing decisions could not be read.';
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1141
1382
|
// A finished turn's activity stats: the session counts in the view and the counts kept across sessions. Stats never
|
|
1142
1383
|
// break a turn: a failed read or write loses this turn's counts. Readings already in the session's metrics record stay
|
|
1143
1384
|
// in memory after a failed write and go out with the next turn that adds a reading.
|
|
@@ -1148,6 +1389,8 @@ async function recordActivity($, router, turn) {
|
|
|
1148
1389
|
const start = promptIndex(messages);
|
|
1149
1390
|
if (start < 0) return;
|
|
1150
1391
|
const observed = observedActivity(messages, start);
|
|
1392
|
+
// The turn's end row reads it (flushDecisions).
|
|
1393
|
+
turn.observed = observed;
|
|
1151
1394
|
const view = router.view ?? (await readView($, router));
|
|
1152
1395
|
const manual = (await modeOf($, router)) === 'manual';
|
|
1153
1396
|
// The session may have changed during the reads; this turn is not the new session's.
|
|
@@ -1180,7 +1423,12 @@ async function recordActivity($, router, turn) {
|
|
|
1180
1423
|
// empties the view; these counts are older than it. Each record is written on its own: a failed write of one
|
|
1181
1424
|
// keeps the other.
|
|
1182
1425
|
const stats = { predicted: turn.predicted, observed, runEnded: ended, lateral: turn.lateral };
|
|
1183
|
-
const metrics = {
|
|
1426
|
+
const metrics = {
|
|
1427
|
+
classifier: turn.classifier,
|
|
1428
|
+
adviceMs: turn.adviceMs,
|
|
1429
|
+
downMove: down.event,
|
|
1430
|
+
unpriced: turn.unpriced,
|
|
1431
|
+
};
|
|
1184
1432
|
const shown = await Promise.all([
|
|
1185
1433
|
addToLedger($, router, 'activity', (record, now) => recordStore(record, stats, now)).catch(() => false),
|
|
1186
1434
|
addToLedger($, router, 'metrics', (record, now) => recordMetrics(record, { ...metrics, now })).catch(() => false),
|
|
@@ -1568,6 +1816,8 @@ function paneActions($, router, view) {
|
|
|
1568
1816
|
router.run = null;
|
|
1569
1817
|
router.afterDown = null;
|
|
1570
1818
|
router.turnUsage.clear();
|
|
1819
|
+
router.turnDecisions.clear();
|
|
1820
|
+
router.decisionRecord = null;
|
|
1571
1821
|
for (const [id, agent] of router.agents) router.agents.set(id, flushedAgent(agent));
|
|
1572
1822
|
// Every session's records. The marker is written first, so a session still holding its record from before the
|
|
1573
1823
|
// reset drops it, unless that record's first count has the reset's millisecond or the clock moved back across
|
|
@@ -1575,8 +1825,10 @@ function paneActions($, router, view) {
|
|
|
1575
1825
|
const clearRecords = async () => {
|
|
1576
1826
|
const resetAt = Date.now();
|
|
1577
1827
|
await $.store.set(SAVINGS_RESET_KEY, resetAt);
|
|
1578
|
-
const keys = (await $.store.keys()).filter(
|
|
1579
|
-
|
|
1828
|
+
const keys = (await $.store.keys()).filter(
|
|
1829
|
+
(key) =>
|
|
1830
|
+
key.startsWith(DECISIONS_PREFIX) ||
|
|
1831
|
+
Object.values(LEDGERS).some(({ prefix, legacy }) => key.startsWith(prefix) || key === legacy),
|
|
1580
1832
|
);
|
|
1581
1833
|
await Promise.all(keys.map((key) => $.store.delete(key)));
|
|
1582
1834
|
for (const kind of LEDGER_KINDS)
|
|
@@ -1658,6 +1910,13 @@ async function endSession($, router) {
|
|
|
1658
1910
|
for (const controller of router.controllers) controller.abort();
|
|
1659
1911
|
router.decisions.clear();
|
|
1660
1912
|
router.prompts.clear();
|
|
1913
|
+
router.turnOrigins.clear();
|
|
1914
|
+
router.progressNotes.clear();
|
|
1915
|
+
router.turnDecisions.clear();
|
|
1916
|
+
router.origins = [];
|
|
1917
|
+
router.submitting = null;
|
|
1918
|
+
router.goal = null;
|
|
1919
|
+
router.decisionRecord = null;
|
|
1661
1920
|
router.turnConfigs.clear();
|
|
1662
1921
|
router.turnPins.clear();
|
|
1663
1922
|
router.turnActivities.clear();
|
|
@@ -1700,8 +1959,9 @@ export function register(on, options) {
|
|
|
1700
1959
|
router.definitions.clear();
|
|
1701
1960
|
await $.command.register({
|
|
1702
1961
|
name: 'router',
|
|
1703
|
-
description:
|
|
1704
|
-
|
|
1962
|
+
description:
|
|
1963
|
+
'Open the Router pane, or switch routing: auto, off, pin <tier>, activities <mode>, mode <mode>; decisions [n] lists the last routing decisions.',
|
|
1964
|
+
argumentHint: '[auto|off|pin <tier>|activities off|on|mode cost|balanced|quality|default|decisions [n]]',
|
|
1705
1965
|
immediate: true,
|
|
1706
1966
|
});
|
|
1707
1967
|
const model = await $.session.model();
|
|
@@ -1738,6 +1998,7 @@ export function register(on, options) {
|
|
|
1738
1998
|
usagePeriod: await periodOf($),
|
|
1739
1999
|
...(await storeView($, router)),
|
|
1740
2000
|
});
|
|
2001
|
+
await pruneDecisionRecords($);
|
|
1741
2002
|
} catch (error) {
|
|
1742
2003
|
await updateView($, router, {
|
|
1743
2004
|
phase: 'unavailable',
|
|
@@ -1771,10 +2032,12 @@ export function register(on, options) {
|
|
|
1771
2032
|
|
|
1772
2033
|
on('turn.start', async ($, e, next) => {
|
|
1773
2034
|
await fixTurnConfig($, router, e.turnId);
|
|
1774
|
-
router
|
|
2035
|
+
await startTurn($, router, e);
|
|
1775
2036
|
return next(e);
|
|
1776
2037
|
});
|
|
1777
2038
|
|
|
2039
|
+
on('prompt.submit', (_$, e, next) => submitPrompt(router, e, next));
|
|
2040
|
+
|
|
1778
2041
|
// A hook failure spawns the agent as Claude Code would; next(e) replays a spawn already made. agent.spawn and a
|
|
1779
2042
|
// catchable on() are in the 2.1.293 build types, not checked on 2.1.289: an engine without them still loads routing.
|
|
1780
2043
|
try {
|
|
@@ -1889,6 +2152,7 @@ export function register(on, options) {
|
|
|
1889
2152
|
const entries = router.turnUsage.get(e.turnId);
|
|
1890
2153
|
router.turnUsage.delete(e.turnId);
|
|
1891
2154
|
if (entries) await flushUsage($, router, entries, turn?.switched ?? null);
|
|
2155
|
+
await flushDecisions($, router, e.turnId, turn);
|
|
1892
2156
|
return next(e);
|
|
1893
2157
|
});
|
|
1894
2158
|
|
package/lib/activity-stats.mjs
CHANGED
|
@@ -6,7 +6,9 @@ import { acceptActivity } from './activity.mjs';
|
|
|
6
6
|
import { ACTIVITIES, ACTIVITY_VALUES } from './config.mjs';
|
|
7
7
|
import { afterResetOf, earlierSince, isRecord } from './records.mjs';
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
// A turn's bucket from its tools (observedActivity in facts.mjs). `shell` and `check` came in 1.16: a version before
|
|
10
|
+
// reads a store that holds them as empty.
|
|
11
|
+
export const OBSERVED = ['code', 'docs', 'shell', 'ops', 'check', 'read', 'talk'];
|
|
10
12
|
// The one stats key every session shared before 1.10.3: it reads as another session's record until a reset deletes it.
|
|
11
13
|
export const STORE_KEY = 'activity:stats:v1';
|
|
12
14
|
// Each session keeps its own stats under this prefix and its session id, as the metrics do.
|
|
@@ -16,16 +18,17 @@ export const STORE_PREFIX = 'activity:stats:v1:';
|
|
|
16
18
|
export const METRICS_PREFIX = 'activity:metrics:v1:';
|
|
17
19
|
|
|
18
20
|
// The observed buckets each predicted activity allows. A weak oracle: it tells code from ops and explore, and cannot
|
|
19
|
-
// separate plan from review. That is enough to catch a cheap route on a coding turn.
|
|
21
|
+
// separate plan from review. That is enough to catch a cheap route on a coding turn. A `shell` write is a candidate
|
|
22
|
+
// for code, docs or routine work, so each of them allows it; a `check` run fits a review, a debug session or ops.
|
|
20
23
|
export const ALLOWED = {
|
|
21
|
-
code: ['code'],
|
|
22
|
-
debug: ['code', 'ops', 'read'],
|
|
24
|
+
code: ['code', 'shell'],
|
|
25
|
+
debug: ['code', 'shell', 'ops', 'check', 'read'],
|
|
23
26
|
explore: ['read', 'talk'],
|
|
24
27
|
plan: ['talk', 'read', 'docs'],
|
|
25
|
-
review: ['read', 'talk'],
|
|
28
|
+
review: ['read', 'talk', 'check'],
|
|
26
29
|
// git status or a test listing is still running commands: a read-only Bash turn is ops too.
|
|
27
|
-
ops: ['ops', 'read'],
|
|
28
|
-
docs: ['docs'],
|
|
30
|
+
ops: ['ops', 'shell', 'check', 'read'],
|
|
31
|
+
docs: ['docs', 'shell'],
|
|
29
32
|
};
|
|
30
33
|
|
|
31
34
|
// An answer without an allowed set (uncertain, none) never agrees; callers count agreement only for activities.
|
|
@@ -224,9 +227,15 @@ export function mergeStores(values, resetAt = null) {
|
|
|
224
227
|
// classifier id, and the cheaper activity moves taken in `on` with the escalations that followed one, since `since`
|
|
225
228
|
// (ms), the first reading after a reset.
|
|
226
229
|
export function emptyMetrics() {
|
|
227
|
-
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 } };
|
|
230
|
+
return { version: 1, since: null, latency: {}, downMoves: { moves: 0, escalations: 0 }, unpriced: emptyUnpriced() };
|
|
228
231
|
}
|
|
229
232
|
|
|
233
|
+
// Why a routed main reply was left out of the usage record (unpricedCause in savings.mjs): its usage did not read
|
|
234
|
+
// (`usage`), a model in the comparison had no price (`model`), the cache lifetime could not be read (`ttl`), or the
|
|
235
|
+
// comparison failed (`error`). Kept beside the stats from 1.16; a record from before reads as none.
|
|
236
|
+
export const UNPRICED_CAUSES = ['usage', 'model', 'ttl', 'error'];
|
|
237
|
+
const emptyUnpriced = () => Object.fromEntries(UNPRICED_CAUSES.map((cause) => [cause, 0]));
|
|
238
|
+
|
|
230
239
|
const isLatency = (v) =>
|
|
231
240
|
isRecord(v) &&
|
|
232
241
|
Object.keys(v).length <= LATENCY_CLASSIFIERS &&
|
|
@@ -242,19 +251,31 @@ export function readMetrics(value) {
|
|
|
242
251
|
since: Number.isFinite(value.since) ? value.since : null,
|
|
243
252
|
latency: isLatency(value.latency) ? value.latency : empty.latency,
|
|
244
253
|
downMoves: exactCountsOf(value.downMoves, ['moves', 'escalations']) ? value.downMoves : empty.downMoves,
|
|
254
|
+
unpriced: exactCountsOf(value.unpriced, UNPRICED_CAUSES) ? value.unpriced : empty.unpriced,
|
|
245
255
|
};
|
|
246
256
|
}
|
|
247
257
|
|
|
248
|
-
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed),
|
|
249
|
-
// 'escalated' (advanceDown). A turn that adds nothing
|
|
250
|
-
|
|
258
|
+
// One finished turn at `now`: `adviceMs` how long `classifier` took (null: not timed), `downMove` 'moved' or
|
|
259
|
+
// 'escalated' (advanceDown), and `unpriced` the turn's routed replies left unpriced, by cause. A turn that adds nothing
|
|
260
|
+
// returns `metrics` itself.
|
|
261
|
+
export function recordMetrics(
|
|
262
|
+
metrics,
|
|
263
|
+
{ classifier = null, adviceMs = null, downMove = null, unpriced = {}, now = null },
|
|
264
|
+
) {
|
|
251
265
|
const next = {
|
|
252
266
|
version: 1,
|
|
253
267
|
since: metrics.since ?? now,
|
|
254
268
|
latency: { ...metrics.latency },
|
|
255
269
|
downMoves: { ...metrics.downMoves },
|
|
270
|
+
unpriced: { ...emptyUnpriced(), ...metrics.unpriced },
|
|
256
271
|
};
|
|
257
272
|
let counted = downMove === 'moved' || downMove === 'escalated';
|
|
273
|
+
for (const cause of UNPRICED_CAUSES) {
|
|
274
|
+
const n = count(unpriced?.[cause]);
|
|
275
|
+
if (!n) continue;
|
|
276
|
+
next.unpriced[cause] += n;
|
|
277
|
+
counted = true;
|
|
278
|
+
}
|
|
258
279
|
const known = Object.hasOwn(next.latency, classifier);
|
|
259
280
|
if (
|
|
260
281
|
typeof classifier === 'string' &&
|
|
@@ -293,6 +314,7 @@ export function mergeMetrics(values, resetAt = null) {
|
|
|
293
314
|
moves: merged.downMoves.moves + one.downMoves.moves,
|
|
294
315
|
escalations: merged.downMoves.escalations + one.downMoves.escalations,
|
|
295
316
|
};
|
|
317
|
+
for (const cause of UNPRICED_CAUSES) merged.unpriced[cause] += one.unpriced[cause];
|
|
296
318
|
}
|
|
297
319
|
return merged;
|
|
298
320
|
}
|
package/lib/activity.mjs
CHANGED
|
@@ -34,10 +34,10 @@ export const sameRequest = (config, a, b, sentEffort) =>
|
|
|
34
34
|
|
|
35
35
|
// The activity a turn runs with, given the incumbent cell `{ tier, activity }` and its resolved `route`. Without
|
|
36
36
|
// advice, or without an activity answer (a skipped, failed or malformed activity step), the incumbent's activity
|
|
37
|
-
// stays: a flaky step must not move the route every turn. On the same task (sameTaskOf: a review inside a turn,
|
|
38
|
-
// continuation prompt) a stronger label is taken at once (explore, then code), and a weaker
|
|
39
|
-
// `downgradeMass` probability once facts.downAllowed holds: the dwell has passed and a
|
|
40
|
-
// last model change. An unclear label keeps the running activity.
|
|
37
|
+
// stays: a flaky step must not move the route every turn. On the same task (sameTaskOf: a review inside a turn, a turn
|
|
38
|
+
// with a progress origin, or a continuation prompt) a stronger label is taken at once (explore, then code), and a weaker
|
|
39
|
+
// one only with at least `downgradeMass` probability once facts.downAllowed holds: the dwell has passed and a checkpoint
|
|
40
|
+
// (completedSince) since the last model change. An unclear label keeps the running activity.
|
|
41
41
|
export function chooseActivity(config, advice, incumbent, sentEffort, facts = {}) {
|
|
42
42
|
if (!advice?.activity) return incumbent.activity;
|
|
43
43
|
if (!sameTaskOf(config, advice, facts)) return acceptActivity(config, advice);
|
package/lib/band.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
classifierStatus,
|
|
6
6
|
classifierWarns,
|
|
7
7
|
differenceShare,
|
|
8
|
+
modeLever,
|
|
8
9
|
modelColor,
|
|
9
10
|
modelName,
|
|
10
11
|
NOT_A_TIER_REASON,
|
|
@@ -198,7 +199,7 @@ export function bandSegments(config, view, usage, actions) {
|
|
|
198
199
|
main,
|
|
199
200
|
pin,
|
|
200
201
|
segment(1, part(shortReason(view.reason, reasonActivity(config, view)), { dimColor: true })),
|
|
201
|
-
segment(5, part(routingModeText(view), { dimColor: true })),
|
|
202
|
+
segment(5, part([routingModeText(view), modeLever(config, view)].filter(Boolean).join(' · '), { dimColor: true })),
|
|
202
203
|
support ? segment(4, part(support, { dimColor: true })) : null,
|
|
203
204
|
Number.isFinite(switchUsd) && switchUsd >= 0.005
|
|
204
205
|
? segment(4, part(`switch cost ≈ ${usd(switchUsd)}`, { dimColor: true }))
|
package/lib/config.mjs
CHANGED
|
@@ -134,13 +134,16 @@ export const DEFAULTS = {
|
|
|
134
134
|
},
|
|
135
135
|
// `cost` uses a cheaper route where the turn allows it, `quality` a stronger one; `balanced` routes as 1.14. The
|
|
136
136
|
// routes stay as configured: the mode moves a tier's route one step (lib/modes.mjs). Each mode's `policy` replaces
|
|
137
|
-
// those fields of `policy` while it is active. The values
|
|
137
|
+
// those fields of `policy` while it is active. The values started as codex-router's presets. 1.16 sets cost's upgrade
|
|
138
|
+
// bar to balanced's (2 votes from 0.75): from 3 votes at 0.85 the bar, raised by the cache write of a cold stronger
|
|
139
|
+
// model, sat at 0.92 from 60k tokens and 0.97 at 254k, so a cost turn on Haiku kept a task the classifier scored
|
|
140
|
+
// hard at 90%. Cost still moves down sooner. Not tuned on outcomes yet.
|
|
138
141
|
mode: 'balanced',
|
|
139
142
|
modes: {
|
|
140
143
|
cost: {
|
|
141
144
|
policy: {
|
|
142
|
-
upgradeVotes:
|
|
143
|
-
upgradeBase: 0.
|
|
145
|
+
upgradeVotes: 2,
|
|
146
|
+
upgradeBase: 0.75,
|
|
144
147
|
jumpConfidence: 0.98,
|
|
145
148
|
downgradeVotes: 1,
|
|
146
149
|
downgradeMass: 0.8,
|