@alexeiled/claude-router 1.23.0 → 1.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/replay.mjs ADDED
@@ -0,0 +1,712 @@
1
+ // Offline only: all inputs are frozen, text-free recorder projections.
2
+ import { acceptActivity, acceptPhase, routingActivity } from './activity.mjs';
3
+ import { assessCandidates, decideMove, initialIncumbentState, proposeAutoRoute, selectTarget } from './auto-engine.mjs';
4
+ import { autoCatalogKey, externalIncumbent } from './auto-route.mjs';
5
+ import { TIERS } from './config.mjs';
6
+ import * as costs from './cost.mjs';
7
+ import {
8
+ completedFromTrace,
9
+ mergeTraceExport,
10
+ observedFromTrace,
11
+ traceDecision,
12
+ traceProblems,
13
+ } from './replay-trace.mjs';
14
+ import {
15
+ chooseRoute,
16
+ continueRoute,
17
+ economicConfig,
18
+ hasRoutedDecision,
19
+ isNativeFallback,
20
+ isSameModel,
21
+ observedMove,
22
+ observeResponse,
23
+ ownRoute,
24
+ prepareLoop,
25
+ resetHistory,
26
+ } from './route.mjs';
27
+
28
+ const samePair = (a, b) => a?.model === b?.model && (a?.effort ?? null) === (b?.effort ?? null);
29
+ const fail = (message) => {
30
+ throw new Error(`Invalid replay input: ${message}`);
31
+ };
32
+ const countTurns = (events) => new Set(events.filter((e) => e.type === 'request').map((e) => e.turn)).size;
33
+
34
+ // Validate before projecting: malformed data must not quietly turn into a successful scenario.
35
+ export function mergeExports(exports) {
36
+ let merged = null;
37
+ for (const value of exports) {
38
+ if (value?.version !== 1 || !value.sessions || Array.isArray(value.sessions) || typeof value.sessions !== 'object')
39
+ fail('expected version 1 sessions object');
40
+ for (const record of Object.values(value.sessions)) {
41
+ if (!Array.isArray(record?.events)) fail('expected events array');
42
+ for (const event of record.events) {
43
+ if (!Number.isFinite(event.at)) fail('event timestamp required');
44
+ if (event.type === 'request') {
45
+ if (
46
+ !Array.isArray(event.operations) ||
47
+ !['auto', 'manual'].includes(event.mode) ||
48
+ typeof event.model !== 'string' ||
49
+ !Number.isSafeInteger(event.messageCount)
50
+ )
51
+ fail('invalid request');
52
+ for (const op of event.operations) {
53
+ if (!Number.isFinite(op.at) || !['choose', 'observed', 'continue'].includes(op.kind))
54
+ fail('operation timestamp/kind required');
55
+ }
56
+ }
57
+ if (
58
+ event.type === 'response' &&
59
+ event.usage &&
60
+ Object.values(event.usage).some((n) => !Number.isFinite(n) || n < 0)
61
+ )
62
+ fail('invalid token counter');
63
+ }
64
+ }
65
+ try {
66
+ merged = mergeTraceExport(merged, Object.entries(value.sessions));
67
+ } catch {
68
+ fail('invalid trace export or record');
69
+ }
70
+ }
71
+ return merged ?? { version: 1, sessions: {} };
72
+ }
73
+
74
+ function factsFor(config, loop, op, turn) {
75
+ const input = op.facts ?? { effort: op.effort, contextTokens: op.contextTokens, hasImages: op.hasImages };
76
+ const since = loop.switchedAt ?? { requests: 0, messageCount: 0 };
77
+ const dwellAllowed = !loop.switchedAt || loop.requests - since.requests >= config.policy.minDwellRequests;
78
+ return {
79
+ ...input,
80
+ turnKey: turn,
81
+ lastRoute: loop.lastRoute,
82
+ models: loop.models,
83
+ lastRequest: loop.lastRequest,
84
+ historyMeasured: loop.historyMeasured,
85
+ ineligible: loop.ineligible,
86
+ dwellAllowed,
87
+ downAllowed:
88
+ !input.progress &&
89
+ loop.requests - since.requests >= config.policy.minDwellRequests &&
90
+ completedFromTrace(
91
+ op.taskProgress,
92
+ since.messageCount,
93
+ config.activityRouting === 'on' ? loop.lastActivity : null,
94
+ ),
95
+ };
96
+ }
97
+
98
+ function resetAt(loop, event, events, index) {
99
+ if (event.cause === 'history' || event.cause === 'rewind') {
100
+ const request = events.slice(index + 1).find((e) => e.type === 'request');
101
+ return request ? prepareLoop(loop, request.messageCount) : resetHistory(loop);
102
+ }
103
+ return resetHistory(loop);
104
+ }
105
+
106
+ const physical = (config, route, effort) =>
107
+ route && {
108
+ model: config.models[route.model]?.id ?? route.model,
109
+ effort:
110
+ typeof route.effort === 'number'
111
+ ? route.effort
112
+ : config.models[route.model]
113
+ ? costs.routeEffort(config, route, effort)
114
+ : (route.effort ?? effort ?? null),
115
+ };
116
+ const aliasRoute = (config, pair) => ({
117
+ model: Object.keys(config.models).find((k) => isSameModel(config.models[k].id, pair.model)) ?? pair.model,
118
+ effort: pair.effort ?? null,
119
+ });
120
+
121
+ export function replayTable(record) {
122
+ let config = record.events[0].config;
123
+ let loop = structuredClone(record.events[0].loop);
124
+ loop.turnId = null;
125
+ const requests = new Map();
126
+ const rows = [];
127
+ const mismatches = [];
128
+ const mismatch = (seq, field) => mismatches.push({ seq, field });
129
+ for (const [index, event] of record.events.entries()) {
130
+ if (event.type === 'config') config = event.config;
131
+ if (event.type === 'reset') loop = resetAt(loop, event, record.events, index);
132
+ if (event.type === 'request') {
133
+ loop = prepareLoop(loop, event.messageCount);
134
+ const fallback =
135
+ event.mode !== 'manual' && loop.turnId === event.turn && isNativeFallback(loop, event.engineModel);
136
+ if (event.mode === 'manual' || fallback) {
137
+ if (event.operations.length) mismatch(event.seq, 'unexpected-operations');
138
+ } else {
139
+ for (const op of event.operations) {
140
+ const input = { ...op, now: op.at, facts: factsFor(config, loop, op, event.turn) };
141
+ if (op.kind === 'choose') {
142
+ const fresh = loop.turnId !== event.turn;
143
+ loop = chooseRoute(config, loop, input);
144
+ if (fresh) loop.engineModel = event.engineModel;
145
+ } else if (op.kind === 'observed') loop = observedMove(config, loop, input);
146
+ else loop = continueRoute(config, loop, input);
147
+ if (JSON.stringify(traceDecision(loop.decision)) !== JSON.stringify(op.expected))
148
+ mismatch(event.seq, op.kind);
149
+ }
150
+ }
151
+ const pair =
152
+ event.mode === 'manual' || fallback ? { model: event.engineModel, effort: event.engineEffort } : loop.decision;
153
+ if (!samePair(pair, event)) mismatch(event.seq, 'request');
154
+ const row = {
155
+ seq: event.seq,
156
+ turn: event.turn,
157
+ generation: loop.generation,
158
+ at: event.at,
159
+ pair: { model: pair?.model, effort: pair?.effort ?? null },
160
+ mode: config.routingMode?.mode ?? config.mode ?? 'balanced',
161
+ manual: event.mode === 'manual',
162
+ fallback,
163
+ pinned: Boolean(loop.decision?.pinned) && event.mode !== 'manual',
164
+ activity: loop.decision?.activity ?? null,
165
+ config,
166
+ };
167
+ rows.push(row);
168
+ requests.set(event.seq, row);
169
+ }
170
+ if (event.type === 'response') {
171
+ const request = requests.get(event.requestSeq);
172
+ if (!request || !samePair({ model: event.requestedModel, effort: event.effort }, request.pair))
173
+ mismatch(event.seq, 'response-request');
174
+ loop = observeResponse(config, loop, {
175
+ usage: event.usage && { ...event.usage, model: event.servedModel },
176
+ requestedModel: event.requestedModel,
177
+ effort: event.effort,
178
+ stopReason: event.stopReason,
179
+ now: event.at,
180
+ });
181
+ if (!request?.manual && !request?.fallback && event.stopReason === null) loop.suspended = true;
182
+ if (request?.manual && !loop.decision) loop.route = ownRoute(config, event.requestedModel);
183
+ }
184
+ }
185
+ return { rows, mismatches };
186
+ }
187
+
188
+ // Cost and cache on a frozen transcript: total prompt and output lengths are observed; hit tokens are NOT copied.
189
+ function simulatedReply(config, loop, pair, event) {
190
+ const usage = event.usage;
191
+ const counts = ['input_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens', 'output_tokens'];
192
+ if (!usage || !counts.every((k) => Number.isFinite(usage[k]) && usage[k] >= 0)) return { range: null, usage: null };
193
+ const tokens = usage.input_tokens + usage.cache_read_input_tokens + usage.cache_creation_input_tokens;
194
+ const route = aliasRoute(config, pair);
195
+ const model = config.models[route.model];
196
+ if (!model || !Number.isFinite(model.input) || !Number.isFinite(model.cacheRead) || !Number.isFinite(model.output))
197
+ return { range: null, usage: null };
198
+ // A shortened transcript invalidates the replay's own warmth, independently of the table cache.
199
+ const models = loop.lastRequest && tokens < loop.lastRequest.tokens * 0.8 ? {} : loop.models;
200
+ const pricedRoute = typeof route.effort === 'number' ? { ...route, effort: null } : route;
201
+ const bounds = costs.inputBounds(config, pricedRoute, tokens, { models, effort: pair.effort }, event.at);
202
+ const rates = costs.ratesAt(model, tokens, event.at);
203
+ const output = (usage.output_tokens * rates.output) / 1e6;
204
+ const key = costs.cacheKey(config, pair.model, pair.effort);
205
+ const prefix =
206
+ config.cache.mode === 'explicit' && costs.isWarm(models[key], event.at, config.cache)
207
+ ? Math.min(models[key].prefixTokens, tokens)
208
+ : 0;
209
+ // Explicit caches build a full-prefix scenario; automatic caches retain uncertainty in pricing.
210
+ return {
211
+ range: { minUsd: bounds.min + output, maxUsd: bounds.max + output },
212
+ usage: {
213
+ model: pair.model,
214
+ input_tokens: config.cache.mode === 'disabled' ? tokens : 0,
215
+ cache_read_input_tokens: prefix,
216
+ cache_creation_input_tokens: config.cache.mode === 'disabled' ? 0 : tokens - prefix,
217
+ output_tokens: usage.output_tokens,
218
+ },
219
+ };
220
+ }
221
+
222
+ function acceptAutoAdvice(replay, event, op) {
223
+ const { config } = replay;
224
+ replay.advice = op.advice;
225
+ const accepted = config.activityRouting === 'on' ? acceptActivity(config, op.advice) : null;
226
+ const signal = routingActivity(config, op.advice);
227
+ if (accepted) replay.activity = signal?.mass >= config.policy.activityMass ? signal.choice : accepted;
228
+ replay.boundedExecute = accepted !== null && acceptPhase(config, op.advice) === 'execute';
229
+ replay.observedFrom = op.facts?.midTurn
230
+ ? Math.max(op.taskProgress?.prompt ?? 0, (event.observedFrom ?? 0) - 1)
231
+ : (op.taskProgress?.prompt ?? event.messageCount - 1);
232
+ }
233
+
234
+ function autoOperation(replay, event, op, mode) {
235
+ const { config, loop } = replay;
236
+ if (op.kind === 'observed') return null;
237
+ if (op.kind === 'continue' && !replay.savedPin) {
238
+ const observed = observedFromTrace(op.taskProgress, replay.observedFrom);
239
+ if (['code', 'docs', 'ops'].includes(observed)) replay.activity = observed;
240
+ if (['code', 'docs'].includes(observed)) replay.boundedExecute = false;
241
+ }
242
+ const facts = factsFor(config, { ...loop, lastActivity: replay.activity }, op, event.turn);
243
+ if (op.kind !== 'choose') facts.midTurn = true;
244
+ if (op.kind === 'continue') facts.hasImages = op.actualHasImages;
245
+ const projected = economicConfig(config, loop);
246
+ if (!replay.state) replay.activeMode = mode;
247
+ replay.state ??= initialIncumbentState({ config: projected, incumbent: replay.incumbent, facts });
248
+ const catalog = autoCatalogKey(config, {
249
+ models: projected.models,
250
+ availableModels: op.availableModels,
251
+ nativeModel: op.nativeModel,
252
+ });
253
+ const changeCause =
254
+ mode !== replay.activeMode ? 'mode' : replay.catalog !== undefined && replay.catalog !== catalog ? 'catalog' : null;
255
+ const input = {
256
+ config: projected,
257
+ state: replay.state,
258
+ incumbent: replay.incumbent,
259
+ advice: replay.advice,
260
+ facts: {
261
+ ...facts,
262
+ ...(replay.incumbentOrigin === 'external-fallback' && replay.incumbent.effort === null ? { effort: null } : {}),
263
+ },
264
+ now: op.at,
265
+ mode,
266
+ contextKnown: op.contextKnown,
267
+ availableModels: op.availableModels,
268
+ nativeModel: op.nativeModel,
269
+ changeCause,
270
+ incumbentOrigin: op.kind === 'choose' && !op.facts?.midTurn ? replay.incumbentOrigin : null,
271
+ };
272
+ let route;
273
+ if (op.kind === 'choose') {
274
+ if (!op.pin) acceptAutoAdvice(replay, event, op);
275
+ const proposal = proposeAutoRoute({
276
+ ...input,
277
+ startup: !hasRoutedDecision(loop),
278
+ advice: op.advice,
279
+ pin: op.pin,
280
+ boundedExecute: replay.boundedExecute,
281
+ });
282
+ replay.reason = proposal.move.reason;
283
+ route = proposal.move.route;
284
+ if (op.pin) {
285
+ replay.savedPin ??= {
286
+ incumbent: replay.incumbent,
287
+ pin: op.pin,
288
+ switchedAt: replay.loop.switchedAt,
289
+ servedRoute: replay.loop.servedRoute,
290
+ };
291
+ } else {
292
+ replay.demandResult = proposal.demand;
293
+ replay.state = proposal.state;
294
+ if (['reentry', 'same-route'].includes(proposal.move.reason) && input.incumbentOrigin)
295
+ replay.incumbentOrigin = null;
296
+ replay.incumbent = route ?? replay.incumbent;
297
+ }
298
+ } else if (replay.savedPin) {
299
+ const proposal = proposeAutoRoute({
300
+ ...input,
301
+ incumbent: aliasRoute(config, loop.decision),
302
+ pin: replay.savedPin.pin,
303
+ });
304
+ route = proposal.move.route;
305
+ replay.reason = proposal.move.reason;
306
+ } else {
307
+ const level = Math.max(
308
+ 0,
309
+ replay.state.demand -
310
+ (replay.boundedExecute && !replay.state.headroom && replay.state.turn > replay.state.holdUntilTurn ? 0.5 : 0),
311
+ );
312
+ const demand = {
313
+ ...replay.demandResult,
314
+ level,
315
+ state: replay.state,
316
+ escalated: false,
317
+ headroom: replay.state.headroom,
318
+ };
319
+ const candidates = assessCandidates(input);
320
+ const target = selectTarget({ candidates, demand: level, mode, headroom: replay.state.headroom });
321
+ const state = replay.state.headroom ? { ...replay.state, headroom: target.headroom } : replay.state;
322
+ const move = decideMove({ ...input, state, demandResult: demand, candidates, target });
323
+ replay.state = move.state;
324
+ replay.incumbent = move.route ?? replay.incumbent;
325
+ replay.reason = move.reason;
326
+ replay.demandResult = demand;
327
+ route = move.route;
328
+ }
329
+ if (!replay.savedPin) {
330
+ replay.catalog = catalog;
331
+ replay.activeMode = mode;
332
+ }
333
+ if (!route) {
334
+ replay.reason = 'no-route-native';
335
+ return { model: event.engineModel, effort: event.engineEffort ?? null };
336
+ }
337
+ return physical(config, route, facts.effort);
338
+ }
339
+
340
+ function autoRequest(replay, event, table) {
341
+ const generation = replay.loop.generation;
342
+ replay.loop = prepareLoop(replay.loop, event.messageCount);
343
+ if (replay.loop.generation !== generation) {
344
+ replay.state = null;
345
+ replay.demandResult = null;
346
+ }
347
+ replay.incumbent ??=
348
+ replay.loop.route ?? aliasRoute(replay.config, { model: event.engineModel, effort: event.engineEffort });
349
+ if (event.turn !== replay.lastTurn && replay.savedPin) {
350
+ replay.incumbent = replay.savedPin.incumbent;
351
+ if (!replay.pendingFallback) {
352
+ replay.loop.switchedAt = replay.savedPin.switchedAt;
353
+ replay.loop.servedRoute = replay.savedPin.servedRoute;
354
+ }
355
+ replay.savedPin = null;
356
+ }
357
+ const continuedFallback = event.turn === replay.lastTurn && replay.loop.suspended;
358
+ if (event.turn !== replay.lastTurn) replay.loop.suspended = false;
359
+ if (!replay.savedPin && replay.pendingFallback) {
360
+ Object.assign(replay, replay.pendingFallback);
361
+ replay.pendingFallback = null;
362
+ }
363
+ replay.lastTurn = event.turn;
364
+ let pair;
365
+ if (table.manual || table.fallback || continuedFallback) {
366
+ pair = { model: event.engineModel, effort: event.engineEffort };
367
+ replay.reason = table.manual ? 'manual' : 'native-fallback';
368
+ if (table.manual) replay.incumbent = aliasRoute(replay.config, pair);
369
+ } else {
370
+ for (const op of event.operations) {
371
+ pair = autoOperation(replay, event, op, table.mode) ?? pair;
372
+ if (op.kind !== 'observed') replay.loop.hasRoutedDecision = true;
373
+ }
374
+ pair ??= replay.savedPin ? replay.loop.decision : physical(replay.config, replay.incumbent, event.engineEffort);
375
+ }
376
+ replay.loop.decision = { model: pair.model, effort: pair.effort, pinned: Boolean(replay.savedPin) };
377
+ replay.loop.lastActivity = replay.activity;
378
+ return {
379
+ ...table,
380
+ pair,
381
+ fallback: table.fallback || continuedFallback,
382
+ pinned: Boolean(replay.savedPin) && !table.manual,
383
+ activity: replay.activity,
384
+ demand: replay.savedPin
385
+ ? TIERS.indexOf(replay.savedPin.pin)
386
+ : (replay.demandResult?.level ?? replay.state?.demand ?? TIERS.indexOf(replay.config.baselineTier)),
387
+ reason: replay.reason,
388
+ generation: replay.loop.generation,
389
+ config: replay.config,
390
+ };
391
+ }
392
+
393
+ export function replayAuto(record, tableRows) {
394
+ const header = record.events[0];
395
+ const replay = {
396
+ config: header.configured,
397
+ activeMode: header.config.routingMode?.mode ?? header.config.mode,
398
+ loop: { ...structuredClone(header.loop), turnId: null },
399
+ state: null,
400
+ incumbent: null,
401
+ savedPin: null,
402
+ demandResult: null,
403
+ advice: null,
404
+ boundedExecute: false,
405
+ activity: header.loop.lastActivity,
406
+ lastTurn: null,
407
+ reason: null,
408
+ observedFrom: 0,
409
+ };
410
+ const rows = [];
411
+ const requests = new Map();
412
+ const tables = new Map(tableRows.map((row) => [row.seq, row]));
413
+ for (const [index, event] of record.events.entries()) {
414
+ // Evidence comes from the last unpinned accepted mode/catalog, not table-only pin snapshots.
415
+ if (event.type === 'config') replay.config = event.configured;
416
+ if (event.type === 'control' && !event.enabled) {
417
+ replay.state = null;
418
+ replay.demandResult = null;
419
+ replay.savedPin = null;
420
+ }
421
+ if (event.type === 'reset') {
422
+ replay.loop = resetAt(replay.loop, event, record.events, index);
423
+ replay.state = null;
424
+ replay.demandResult = null;
425
+ replay.boundedExecute = false;
426
+ replay.advice = null;
427
+ replay.observedFrom = Math.min(replay.observedFrom, replay.loop.lastMessageCount ?? 0);
428
+ }
429
+ if (event.type === 'request') {
430
+ const row = autoRequest(replay, event, tables.get(event.seq));
431
+ rows.push(row);
432
+ requests.set(event.seq, row);
433
+ }
434
+ if (event.type === 'response') {
435
+ const row = requests.get(event.requestSeq);
436
+ const substituted = event.servedModel && !isSameModel(event.requestedModel, event.servedModel);
437
+ const simulated = simulatedReply(replay.config, replay.loop, row.pair, event);
438
+ row.external = substituted || event.stopReason === null || row.reason === 'no-route-native' || row.fallback;
439
+ row.cost = substituted ? null : simulated.range;
440
+ if (!row.manual && (substituted || row.reason === 'no-route-native' || row.fallback)) {
441
+ const external = externalIncumbent(replay.config, {
442
+ requestedModel: row.pair.model,
443
+ effort: row.pair.effort,
444
+ usage: substituted ? { model: event.servedModel } : simulated.usage,
445
+ });
446
+ if (replay.savedPin) replay.pendingFallback = external;
447
+ else Object.assign(replay, external);
448
+ }
449
+ replay.loop = observeResponse(replay.config, replay.loop, {
450
+ requestedModel: row.pair.model,
451
+ effort: row.pair.effort,
452
+ usage: substituted ? null : simulated.usage,
453
+ stopReason: event.stopReason,
454
+ now: event.at,
455
+ });
456
+ if (
457
+ substituted ||
458
+ row.reason === 'no-route-native' ||
459
+ (event.stopReason === null && !row.manual && !row.fallback)
460
+ )
461
+ replay.loop.suspended = true;
462
+ }
463
+ }
464
+ return rows;
465
+ }
466
+
467
+ function markSwitches(rows, window) {
468
+ let prior = null;
469
+ let previous = null;
470
+ for (const row of rows) {
471
+ if (row.manual || row.fallback || row.external) {
472
+ previous = null;
473
+ prior = null;
474
+ continue;
475
+ }
476
+ if (previous && row.generation !== previous.generation) {
477
+ previous = null;
478
+ prior = null;
479
+ }
480
+ if (row.pinned) continue;
481
+ if (previous && !samePair(previous.pair, row.pair)) {
482
+ const move = { from: previous.pair, to: row.pair, turn: row.turn, reversal: false, paired: false };
483
+ if (prior && samePair(prior.from, move.to) && samePair(prior.to, move.from) && move.turn - prior.turn <= window) {
484
+ move.reversal = true;
485
+ move.paired = true;
486
+ prior.paired = true;
487
+ }
488
+ row.switchEvent = move;
489
+ prior = move;
490
+ }
491
+ previous = row;
492
+ }
493
+ }
494
+
495
+ function summary(rows) {
496
+ const priced = rows.filter((row) => row.cost);
497
+ const knownRange = {
498
+ minUsd: priced.reduce((n, r) => n + r.cost.minUsd, 0),
499
+ maxUsd: priced.reduce((n, r) => n + r.cost.maxUsd, 0),
500
+ };
501
+ const moves = rows.map((row) => row.switchEvent).filter(Boolean);
502
+ const turns = new Set(rows.map((row) => row.turn)).size;
503
+ const reversals = moves.filter((move) => move.reversal).length;
504
+ return {
505
+ requests: rows.length,
506
+ externalRequests: rows.filter((row) => row.external || row.fallback).length,
507
+ turns,
508
+ conditionalCost: {
509
+ minUsd: priced.length === rows.length ? knownRange.minUsd : null,
510
+ maxUsd: priced.length === rows.length ? knownRange.maxUsd : null,
511
+ knownPricedRange: knownRange,
512
+ pricedRequests: priced.length,
513
+ unpricedRequests: rows.length - priced.length,
514
+ },
515
+ switches: {
516
+ moves: moves.length,
517
+ reversals,
518
+ reversalMoves: moves.filter((m) => m.paired).length,
519
+ oneWay: moves.filter((m) => !m.paired).length,
520
+ per100Turns: turns ? (reversals * 100) / turns : 0,
521
+ },
522
+ effortMix: Object.fromEntries(
523
+ [...new Set(rows.map((r) => String(r.pair.effort ?? 'session')))].map((e) => [
524
+ e,
525
+ rows.filter((r) => String(r.pair.effort ?? 'session') === e).length,
526
+ ]),
527
+ ),
528
+ };
529
+ }
530
+
531
+ function priceTable(record, rows) {
532
+ let loop = structuredClone(record.events[0].loop);
533
+ let config = record.events[0].config;
534
+ const bySeq = new Map(rows.map((row) => [row.seq, row]));
535
+ for (const [index, event] of record.events.entries()) {
536
+ if (event.type === 'config') config = event.config;
537
+ if (event.type === 'reset') loop = resetAt(loop, event, record.events, index);
538
+ if (event.type === 'response') {
539
+ const row = bySeq.get(event.requestSeq);
540
+ const substituted = event.servedModel && !isSameModel(event.requestedModel, event.servedModel);
541
+ row.external = substituted || event.stopReason === null;
542
+ row.cost = substituted ? null : simulatedReply(config, loop, row.pair, event).range;
543
+ loop = observeResponse(config, loop, {
544
+ requestedModel: row.pair.model,
545
+ effort: row.pair.effort,
546
+ usage: event.usage && { ...event.usage, model: event.servedModel },
547
+ now: event.at,
548
+ stopReason: event.stopReason,
549
+ });
550
+ }
551
+ }
552
+ }
553
+
554
+ function sessionReport(item, window) {
555
+ const auto = replayAuto(item.record, item.table);
556
+ priceTable(item.record, item.table);
557
+ markSwitches(item.table, window);
558
+ markSwitches(auto, window);
559
+ const perMode = {};
560
+ const tableRows = new Map(item.table.map((row) => [row.seq, row]));
561
+ for (const mode of ['cost', 'balanced', 'quality']) {
562
+ const table = item.table.filter((row) => row.mode === mode);
563
+ const alternative = auto.filter((row) => row.mode === mode);
564
+ if (!table.length) continue;
565
+ const disagreements = {};
566
+ for (const row of alternative) {
567
+ const original = tableRows.get(row.seq);
568
+ const key = `${row.demand}/${row.activity ?? 'none'}`;
569
+ const bucket = disagreements[key] ?? { requests: 0, different: 0 };
570
+ bucket.requests++;
571
+ if (!samePair(row.pair, original.pair)) bucket.different++;
572
+ disagreements[key] = bucket;
573
+ }
574
+ perMode[mode] = { table: summary(table), auto: summary(alternative), disagreementByDemandActivity: disagreements };
575
+ }
576
+ return {
577
+ cohort: item.cohort,
578
+ modes: perMode,
579
+ unknownCapabilities: Object.values(item.record.events[0].configured.models).filter(
580
+ (model) => !Object.keys(model.capability).length,
581
+ ).length,
582
+ };
583
+ }
584
+
585
+ function missingEvidence(op) {
586
+ const progress = op.taskProgress;
587
+ const index = (value) => Number.isInteger(value) && value >= -1;
588
+ if (!progress || !['delegate', 'anyCheck', 'nonDocEdit', 'prompt', 'lastTool'].every((key) => index(progress[key])))
589
+ return true;
590
+ for (const key of ['check', 'edit', 'run'])
591
+ if (progress[key] !== null && (!index(progress[key]?.index) || typeof progress[key]?.ok !== 'boolean')) return true;
592
+ if (progress.result !== null && (!index(progress.result?.index) || typeof progress.result?.error !== 'boolean'))
593
+ return true;
594
+ if (typeof op.contextKnown !== 'boolean') return true;
595
+ if (op.kind === 'continue') return typeof op.actualHasImages !== 'boolean' || typeof op.hasImages !== 'boolean';
596
+ return (
597
+ !op.facts ||
598
+ !['hasImages', 'continuation', 'progress', 'midTurn'].every((key) => typeof op.facts[key] === 'boolean')
599
+ );
600
+ }
601
+
602
+ function evidenceProblems(exports) {
603
+ const problems = new Map();
604
+ for (const value of exports)
605
+ for (const [key, record] of Object.entries(value.sessions))
606
+ for (const event of record.events)
607
+ for (const op of event.operations ?? [])
608
+ if (missingEvidence(op)) {
609
+ const seqs = problems.get(key) ?? new Set();
610
+ seqs.add(event.seq);
611
+ problems.set(key, seqs);
612
+ }
613
+ return problems;
614
+ }
615
+
616
+ export function evaluateReplay(exports, { reversalWindow = 3 } = {}) {
617
+ if (!Number.isSafeInteger(reversalWindow) || reversalWindow < 1) fail('reversal window must be a positive integer');
618
+ const merged = mergeExports(exports);
619
+ const records = Object.entries(merged.sessions);
620
+ const usable = [];
621
+ const explained = {};
622
+ const mismatches = [];
623
+ let capturedTurns = 0;
624
+ let cappedTurns = 0;
625
+ let cappedEvents = 0;
626
+ for (const [index, [key, record]] of records.entries()) {
627
+ capturedTurns += countTurns(record.events);
628
+ cappedTurns += record.capped?.turns ?? 0;
629
+ cappedEvents += record.capped?.events ?? 0;
630
+ const problems = traceProblems(record);
631
+ if (problems.length) {
632
+ for (const reason of problems) explained[reason] = (explained[reason] ?? 0) + 1;
633
+ continue;
634
+ }
635
+ const table = replayTable(record);
636
+ mismatches.push(...table.mismatches.map((m) => ({ cohort: index + 1, ...m })));
637
+ usable.push({ key, record, table: table.rows, cohort: index + 1 });
638
+ }
639
+ const usableTurns = usable.reduce((n, item) => n + countTurns(item.record.events), 0);
640
+ const coverage = {
641
+ exportCohort: {
642
+ sessions: records.length,
643
+ usableSessions: usable.length,
644
+ capturedTurns,
645
+ usableTurns,
646
+ usableSessionRatio: records.length ? usable.length / records.length : null,
647
+ usableTurnRatio: capturedTurns ? usableTurns / capturedTurns : null,
648
+ },
649
+ allSessions: {
650
+ sessions: null,
651
+ turns: null,
652
+ usableSessionRatio: null,
653
+ gate80Percent: 'unavailable',
654
+ reason: 'Exports omit untraced and evicted sessions; captured/export cohort is a lower bound.',
655
+ },
656
+ capped: { turns: cappedTurns, events: cappedEvents },
657
+ explainedExclusions: explained,
658
+ };
659
+ const fidelity = { checkedSessions: usable.length, mismatches, passed: mismatches.length === 0 };
660
+ if (mismatches.length) return { status: 'blocked', fidelity, coverage };
661
+ const problems = evidenceProblems(exports);
662
+ const autoInputIncomplete = usable.flatMap((item) =>
663
+ [...(problems.get(item.key) ?? [])].map((seq) => ({ cohort: item.cohort, seq, reason: 'auto-input-incomplete' })),
664
+ );
665
+ if (autoInputIncomplete.length) return { status: 'blocked', fidelity, coverage, autoInputIncomplete };
666
+ const sessions = usable.map((item) => sessionReport(item, reversalWindow));
667
+ const modes = {};
668
+ for (const item of usable) {
669
+ for (const mode of ['cost', 'balanced', 'quality']) {
670
+ const turns = new Set(
671
+ item.record.events
672
+ .filter(
673
+ (event) =>
674
+ event.type === 'request' &&
675
+ item.table.find((row) => row.seq === event.seq)?.mode === mode &&
676
+ event.operations.some((op) => op.kind === 'choose' && !op.pin),
677
+ )
678
+ .map((event) => event.turn),
679
+ ).size;
680
+ if (!turns) continue;
681
+ const exposure = modes[mode] ?? { sessions: 0, turns: 0 };
682
+ exposure.sessions++;
683
+ exposure.turns += turns;
684
+ modes[mode] = exposure;
685
+ }
686
+ }
687
+ return {
688
+ status: 'ok',
689
+ fidelity,
690
+ coverage,
691
+ reversalWindow,
692
+ semantics:
693
+ 'Recorded mode history; independent stateful auto; frozen transcript conditional estimates, no savings or quality result.',
694
+ exposureDiagnostic: {
695
+ completeSessions: usable.filter(
696
+ (item) =>
697
+ !item.record.capped &&
698
+ item.record.events.some(
699
+ (event) => event.type === 'request' && event.operations.some((op) => op.kind === 'choose' && !op.pin),
700
+ ),
701
+ ).length,
702
+ usablePrefixSessions: usable.length,
703
+ capturedMainTurns: usableTurns,
704
+ eligibleRoutedTurns: Object.values(modes).reduce((n, mode) => n + mode.turns, 0),
705
+ modes,
706
+ thresholds: { sessions: 30, turns: 1000, turnsPerMode: 200 },
707
+ freshness: 'unverified',
708
+ gate: 'not-established',
709
+ },
710
+ sessions,
711
+ };
712
+ }