@genesislcap/ai-assistant 15.12.0 → 15.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/ai-assistant.api.json +245 -3
  2. package/dist/ai-assistant.d.ts +127 -7
  3. package/dist/chat-driver.cjs +79 -13
  4. package/dist/chat-driver.cjs.map +2 -2
  5. package/dist/chat-driver.mjs +76 -12
  6. package/dist/chat-driver.mjs.map +2 -2
  7. package/dist/custom-elements.json +452 -359
  8. package/dist/dts/chat-driver-node.d.ts +2 -2
  9. package/dist/dts/chat-driver-node.d.ts.map +1 -1
  10. package/dist/dts/components/chat-driver/chat-driver.d.ts +45 -1
  11. package/dist/dts/components/chat-driver/chat-driver.d.ts.map +1 -1
  12. package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts +2 -0
  13. package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts.map +1 -0
  14. package/dist/dts/main/main.d.ts +20 -1
  15. package/dist/dts/main/main.d.ts.map +1 -1
  16. package/dist/dts/state/debug-event-log.d.ts.map +1 -1
  17. package/dist/dts/state/persistence/diagnostics.d.ts +65 -8
  18. package/dist/dts/state/persistence/diagnostics.d.ts.map +1 -1
  19. package/dist/dts/state/persistence/index.d.ts +1 -1
  20. package/dist/dts/state/persistence/index.d.ts.map +1 -1
  21. package/dist/dts/state/persistence/session-persister.d.ts +4 -3
  22. package/dist/dts/state/persistence/session-persister.d.ts.map +1 -1
  23. package/dist/dts/utils/sum-usage.d.ts +20 -0
  24. package/dist/dts/utils/sum-usage.d.ts.map +1 -1
  25. package/dist/esm/chat-driver-node.js +12 -2
  26. package/dist/esm/components/chat-driver/chat-driver.js +60 -5
  27. package/dist/esm/components/chat-driver/chat-driver.turn-usage.test.js +268 -0
  28. package/dist/esm/main/main.js +53 -28
  29. package/dist/esm/state/debug-event-log.js +7 -2
  30. package/dist/esm/state/persistence/diagnostics.js +79 -16
  31. package/dist/esm/state/persistence/diagnostics.test.js +174 -1
  32. package/dist/esm/state/persistence/index.js +1 -1
  33. package/dist/esm/state/persistence/session-persister.js +13 -5
  34. package/dist/esm/state/persistence/session-persister.test.js +31 -0
  35. package/dist/esm/utils/sum-usage.js +43 -0
  36. package/dist/esm/utils/sum-usage.test.js +45 -1
  37. package/dist/tsconfig.tsbuildinfo +1 -1
  38. package/package.json +17 -17
  39. package/src/chat-driver-node.ts +12 -2
  40. package/src/components/chat-driver/chat-driver.ts +107 -6
  41. package/src/components/chat-driver/chat-driver.turn-usage.test.ts +362 -0
  42. package/src/main/main.ts +52 -23
  43. package/src/state/debug-event-log.ts +7 -2
  44. package/src/state/persistence/diagnostics.test.ts +208 -1
  45. package/src/state/persistence/diagnostics.ts +117 -15
  46. package/src/state/persistence/index.ts +1 -1
  47. package/src/state/persistence/session-persister.test.ts +37 -0
  48. package/src/state/persistence/session-persister.ts +13 -5
  49. package/src/utils/sum-usage.test.ts +52 -1
  50. package/src/utils/sum-usage.ts +45 -0
@@ -105,7 +105,13 @@ export type {
105
105
  // `addUsage`/`emptyUsage` are the composition primitives for a conversation that CONTINUES across
106
106
  // rounds — see the guidance on `sumUsage` itself. `totalTokens` exists because the four buckets of
107
107
  // an `AggregateUsage` are disjoint and safe to add, which the per-message fields are NOT.
108
- export { addUsage, emptyUsage, sumUsage, totalTokens } from './utils/sum-usage';
108
+ //
109
+ // `messageUsage` is the single-message projection of the same arithmetic — what ONE request
110
+ // cost, in the same four-bucket shape. It is what prices a turn snapshot (see
111
+ // `TurnSnapshot.usage`), and the reason a consumer needs it separately is that the response
112
+ // to a *failed* attempt never reaches history, so `sumUsage` over the transcript cannot see
113
+ // that spend at all.
114
+ export { addUsage, emptyUsage, messageUsage, sumUsage, totalTokens } from './utils/sum-usage';
109
115
 
110
116
  // Per-call projection, for the cases an aggregate cannot serve: usage rows for
111
117
  // per-project attribution, a cost dashboard, or auditing which turns came back
@@ -128,5 +134,9 @@ export { clearSession, getMetaEvents } from './state/debug-event-log';
128
134
  export type { MetaEvent } from './state/debug-event-log';
129
135
  export { buildTimelineEntries } from './state/persistence/build-timeline-entries';
130
136
  export type { TurnSnapshotLike } from './state/persistence/build-timeline-entries';
131
- export { assembleDebugLog } from './state/persistence/diagnostics';
137
+ // `withFreshMetaSnapshot` matters to a headless consumer that stitches a STORED stream: the
138
+ // newest stored `meta-snapshot` is frozen at the last config-signature change, so its context
139
+ // and cost figures are as old as that. Pass a fresh snapshot of your own through this before
140
+ // `assembleDebugLog`, or state in your own output that the totals are historical.
141
+ export { assembleDebugLog, withFreshMetaSnapshot } from './state/persistence/diagnostics';
132
142
  export type { DebugLog, DiagnosticEntry } from './state/persistence/diagnostics';
@@ -1,4 +1,5 @@
1
1
  import type {
2
+ AggregateUsage,
2
3
  AIProvider,
3
4
  AIProviderRegistry,
4
5
  AIProviderType,
@@ -70,7 +71,7 @@ import {
70
71
  normalizeForProvider,
71
72
  } from '../../utils/history-transform';
72
73
  import { logger } from '../../utils/logger';
73
- import { sumUsage } from '../../utils/sum-usage';
74
+ import { messageUsage, sumUsage } from '../../utils/sum-usage';
74
75
  import { TOOL_FOLD_SYMBOL, type ToolFold } from '../../utils/tool-fold';
75
76
  import type { AiDriver, AllAgentSummary } from '../ai-driver/ai-driver';
76
77
 
@@ -241,6 +242,46 @@ export interface TurnSnapshot {
241
242
  agentLabel?: string;
242
243
  /** Agent-supplied snapshot — machine state/context for stateful agents, undefined otherwise. */
243
244
  agentSnapshot?: unknown;
245
+ /**
246
+ * Concrete model that ran this call (e.g. `'claude-sonnet-4-6'`) — the serving model
247
+ * where the provider reports one, else the model the resolved tier was configured
248
+ * with. Undefined when the provider exposes no `getStatus` and the transport stamped
249
+ * nothing.
250
+ *
251
+ * Recorded per call, so an agent whose `provider` selector varies by state (a tier
252
+ * switch between steps of a flow) has each step attributed to the model that actually
253
+ * ran it — without the reader having to join the turn to the message after it, which
254
+ * is impossible for a call that produced no message.
255
+ */
256
+ model?: string;
257
+ /**
258
+ * Registry slot the provider resolved under for this call — a tier name like
259
+ * `'high'`/`'low'`, or the registry default's name. Kept alongside {@link
260
+ * TurnSnapshot.model} because they answer different questions: the slot is what the
261
+ * agent asked for, the model is what served it, and repointing a slot at a new model
262
+ * mid-session is only visible when both are recorded.
263
+ */
264
+ providerName?: string;
265
+ /** Vendor behind the resolved slot (`'anthropic'`, `'gemini'`, …), when the provider reports it. */
266
+ provider?: AIProviderType;
267
+ /**
268
+ * What this one LLM call cost — the four disjoint token buckets plus USD, derived
269
+ * from the response's usage by `messageUsage`. Back-filled when the response lands
270
+ * (the rest of the snapshot is captured *before* the call), so it is `undefined`
271
+ * while the call is in flight, on providers that report no usage, and on a call that
272
+ * threw rather than returning — a malformed-call/truncation/402 error carries no usage
273
+ * block, so any tokens the provider billed for it are not recoverable here.
274
+ *
275
+ * A snapshot is one **model call**, not one user turn: every tool-loop iteration and
276
+ * every retried attempt records its own. That makes this the only record of spend on
277
+ * an attempt that produced no message — a blank or refused response is billed and
278
+ * then discarded (see the empty-response retries), so summing the transcript alone
279
+ * under-reports the turn.
280
+ *
281
+ * For a call that DID produce a message, this is the same money as that message's
282
+ * `cost`/token fields, not additional money — never add turn usage to message usage.
283
+ */
284
+ usage?: AggregateUsage;
244
285
  }
245
286
 
246
287
  interface FoldStackFrame {
@@ -1140,7 +1181,14 @@ export class ChatDriver extends EventTarget implements AiDriver {
1140
1181
  if (resolvedName !== this.lastDispatchedProviderName) {
1141
1182
  this.lastDispatchedProviderName = resolvedName;
1142
1183
  recordMetaEvent(this.sessionKey, 'provider.selected', {
1184
+ // `provider` is the registry SLOT (a tier name like 'high'), kept under that key
1185
+ // for compatibility; `model` and `vendor` are what it resolved to. Recording all
1186
+ // three is the difference between "the agent switched to its high tier" and
1187
+ // knowing which model that actually was — a tier can be repointed mid-session,
1188
+ // and a slot name alone cannot distinguish anthropic from gemini.
1143
1189
  provider: resolvedName,
1190
+ model: status.model,
1191
+ vendor: status.provider,
1144
1192
  agent: this.activeAgentName,
1145
1193
  });
1146
1194
  this.dispatchEvent(
@@ -1304,13 +1352,17 @@ export class ChatDriver extends EventTarget implements AiDriver {
1304
1352
  * Push one snapshot to the ring buffer. Called inside `runToolLoop` just
1305
1353
  * before each LLM call — that's the latest point where the prompt, tool
1306
1354
  * surface, and agent state line up with what the model is about to see.
1355
+ *
1356
+ * Returns the pushed object so the caller can back-fill what only the response
1357
+ * knows (`usage`). Mutating it after the fact is safe whether or not the ring
1358
+ * buffer has since evicted it — an evicted snapshot is simply no longer exported.
1307
1359
  */
1308
1360
  private recordTurnSnapshot(
1309
1361
  resolvedSystemPrompt: string | undefined,
1310
1362
  temperature: number | undefined,
1311
1363
  toolChoice: ChatToolChoice | undefined,
1312
1364
  tailContext: string | undefined,
1313
- ): void {
1365
+ ): TurnSnapshot {
1314
1366
  let agentSnapshot: unknown;
1315
1367
  if (this.debugSnapshotter) {
1316
1368
  try {
@@ -1324,7 +1376,7 @@ export class ChatDriver extends EventTarget implements AiDriver {
1324
1376
  }
1325
1377
  const turnIndex = String(this.globalTurnIndex);
1326
1378
  this.globalTurnIndex += 1;
1327
- this.turnSnapshots.push({
1379
+ const snapshot: TurnSnapshot = {
1328
1380
  turnIndex,
1329
1381
  timestamp: new Date().toISOString(),
1330
1382
  agentName: this.activeAgentName,
@@ -1335,10 +1387,12 @@ export class ChatDriver extends EventTarget implements AiDriver {
1335
1387
  temperature,
1336
1388
  toolChoice,
1337
1389
  agentSnapshot,
1338
- });
1390
+ };
1391
+ this.turnSnapshots.push(snapshot);
1339
1392
  if (this.turnSnapshots.length > this.maxTurnSnapshots) {
1340
1393
  this.turnSnapshots.shift();
1341
1394
  }
1395
+ return snapshot;
1342
1396
  }
1343
1397
 
1344
1398
  /**
@@ -2759,7 +2813,12 @@ export class ChatDriver extends EventTarget implements AiDriver {
2759
2813
  // force when no tools are advertised.)
2760
2814
  const effectiveToolChoice = resolvedToolChoice ?? (this.isSubAgent ? 'required' : undefined);
2761
2815
 
2762
- this.recordTurnSnapshot(systemPrompt, resolvedTemperature, effectiveToolChoice, tailContext);
2816
+ const turnSnapshot = this.recordTurnSnapshot(
2817
+ systemPrompt,
2818
+ resolvedTemperature,
2819
+ effectiveToolChoice,
2820
+ tailContext,
2821
+ );
2763
2822
 
2764
2823
  // Capture the pending user input, then clear the slots BEFORE the chat
2765
2824
  // call. `sendMessage` already appended the user message to `this.history`,
@@ -2809,6 +2868,20 @@ export class ChatDriver extends EventTarget implements AiDriver {
2809
2868
  // oxlint-disable-next-line no-await-in-loop
2810
2869
  const activeProvider = await this.resolveProviderForTurn(promptCtx);
2811
2870
 
2871
+ // Attribute the turn to the tier/model it resolved. Stamped HERE, not inside
2872
+ // `recordTurnSnapshot`: the snapshot is taken before this line runs, so reading
2873
+ // `lastResolved*` there yields the PREVIOUS call's model — wrong on precisely the
2874
+ // turn where an agent's per-state `provider` selector switches tier, which is the
2875
+ // turn a reader is looking for. `model` is refined to the serving model once the
2876
+ // response lands (see below); until then — and on a call that throws — it is the
2877
+ // model we ASKED for, which is the only thing knowable at that point.
2878
+ if (this.lastResolvedProviderName !== undefined) {
2879
+ turnSnapshot.providerName = this.lastResolvedProviderName;
2880
+ }
2881
+ if (this.lastResolvedProvider !== undefined)
2882
+ turnSnapshot.provider = this.lastResolvedProvider;
2883
+ if (this.lastResolvedModel !== undefined) turnSnapshot.model = this.lastResolvedModel;
2884
+
2812
2885
  let response: ChatMessage;
2813
2886
  try {
2814
2887
  // oxlint-disable-next-line no-await-in-loop
@@ -2995,6 +3068,19 @@ export class ChatDriver extends EventTarget implements AiDriver {
2995
3068
  response.providerName = this.lastResolvedProviderName;
2996
3069
  }
2997
3070
 
3071
+ // Back-fill what this call cost onto the snapshot taken just before it, so the
3072
+ // exported debug log prices each model call next to the prompt/tools/state that
3073
+ // produced it (GENC-1480 follow-up). Stamped BEFORE the empty-response branch
3074
+ // below deliberately: a blank or refused response is billed and then thrown away,
3075
+ // so the snapshot is the only place that spend is ever recorded.
3076
+ turnSnapshot.usage = messageUsage(response);
3077
+ // Take the SERVING model over the requested one, now that it is known. `response.model`
3078
+ // was just filled from `lastResolvedModel` if the transport left it unset, so this is
3079
+ // the same rule the message gets — which is the point: a turn and the message it
3080
+ // produced must never disagree about which model ran, including when a server-side
3081
+ // fallback chain answered on a different model than the one we asked for.
3082
+ if (response.model !== undefined) turnSnapshot.model = response.model;
3083
+
2998
3084
  const isThinkingStep = response.content && response.toolCalls?.length;
2999
3085
  const isEmptyResponse = !response.content?.trim() && !response.toolCalls?.length;
3000
3086
  // A pre-output refusal (safety-classifier decline, e.g. Fable 5 `stop_reason: 'refusal'`)
@@ -3060,8 +3146,23 @@ export class ChatDriver extends EventTarget implements AiDriver {
3060
3146
  // `sumCosts`/`sumTokens` don't double-count and `contextTokens` reads it. Reasoning/narration are
3061
3147
  // display-only (usage undefined) and are skipped when building the provider request. `model` /
3062
3148
  // `provider` / `providerName` stay on every split message so each is still attributed.
3149
+ //
3150
+ // EVERY usage field has to be cleared here, not just the three the invariant was
3151
+ // originally written against — keep this list in step with the usage fields on
3152
+ // `ChatMessage`. The cache buckets arrived later (GENC-1475) and were left riding
3153
+ // along on the copies, so a response carrying reasoning AND narration counted its
3154
+ // cache read/write volume three times in `sumUsage`/`usageRows` — invisible in the
3155
+ // cost total (which comes from `cost`) but wrong in every bucket display and in the
3156
+ // exported log.
3063
3157
  const { reasoning, ...rest } = response;
3064
- const displayOnly = { cost: undefined, inputTokens: undefined, outputTokens: undefined };
3158
+ const displayOnly = {
3159
+ cost: undefined,
3160
+ externalCostUsd: undefined,
3161
+ inputTokens: undefined,
3162
+ outputTokens: undefined,
3163
+ cacheReadTokens: undefined,
3164
+ cacheWriteTokens: undefined,
3165
+ };
3065
3166
  if (reasoning) {
3066
3167
  this.appendToHistory({
3067
3168
  ...rest,
@@ -0,0 +1,362 @@
1
+ import type {
2
+ AIProvider,
3
+ AIProviderRegistry,
4
+ ChatMessage,
5
+ ChatToolDefinition,
6
+ } from '@genesislcap/foundation-ai';
7
+ import { assert, createLogicSuite } from '@genesislcap/foundation-testing';
8
+ import type { AgentConfig } from '../../config/config';
9
+ import { clearMetaEventRegistry, getMetaEvents } from '../../state/debug-event-log';
10
+ import { messageUsage, sumUsage } from '../../utils/sum-usage';
11
+ // Side-effect import — MUST come before `./chat-driver` so the driver subclasses
12
+ // jsdom's EventTarget rather than Node's native one. Mirrors chat-driver.test.ts.
13
+ import './align-event-globals';
14
+ import { ChatDriver } from './chat-driver';
15
+
16
+ // ---------------------------------------------------------------------------
17
+ // Per-call usage on the turn snapshots — the four token buckets plus USD priced
18
+ // onto each `kind: 'turn'` entry of the exported debug log.
19
+ //
20
+ // The snapshot is captured BEFORE the model call (that is where the prompt, tool
21
+ // surface and agent state line up with what the model saw), so usage has to be
22
+ // back-filled when the response lands. Two things follow, and both are asserted
23
+ // here: a completed call's turn agrees with the message it produced, and a call
24
+ // that produced NO message (a blank response, retried) is still priced — that
25
+ // spend appears nowhere in the transcript.
26
+ // ---------------------------------------------------------------------------
27
+
28
+ const makeRegistry = (provider: AIProvider): AIProviderRegistry => ({
29
+ get: () => provider,
30
+ default: () => provider,
31
+ defaultName: () => 'test',
32
+ names: () => ['test'],
33
+ getStatus: async () => null,
34
+ listStatuses: async () => [],
35
+ });
36
+
37
+ /** Answers from a queue, then ends the turn with a plain unpriced reply. */
38
+ const scriptedProvider = (responses: ChatMessage[]): AIProvider => {
39
+ const queue = [...responses];
40
+ return {
41
+ chat: async (): Promise<ChatMessage> => queue.shift() ?? { role: 'assistant', content: 'done' },
42
+ };
43
+ };
44
+
45
+ const agent = (overrides: Partial<AgentConfig> & { name: string }): AgentConfig =>
46
+ ({ description: 'test agent', ...overrides }) as AgentConfig;
47
+
48
+ const makeDriver = (config: AgentConfig, provider: AIProvider): ChatDriver => {
49
+ const driver = new ChatDriver(makeRegistry(provider), {
50
+ maxToolIterations: 50,
51
+ maxFoldOperations: 5,
52
+ sessionKey: '',
53
+ });
54
+ driver.applyAgent(config);
55
+ return driver;
56
+ };
57
+
58
+ const def = (name: string): ChatToolDefinition => ({
59
+ name,
60
+ description: `${name} tool`,
61
+ parameters: { type: 'object', properties: {} },
62
+ });
63
+
64
+ /** One priced response: a 1000-token prompt that was mostly a cache hit. */
65
+ const priced = (over: Partial<ChatMessage> = {}): ChatMessage => ({
66
+ role: 'assistant',
67
+ content: 'answer',
68
+ cost: 0.1,
69
+ inputTokens: 1000,
70
+ cacheReadTokens: 900,
71
+ cacheWriteTokens: 50,
72
+ outputTokens: 20,
73
+ ...over,
74
+ });
75
+
76
+ const suite = createLogicSuite('ChatDriver per-turn usage');
77
+
78
+ suite('prices each turn snapshot with the four buckets and USD of its own call', async () => {
79
+ const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([priced()]));
80
+ await driver.sendMessage('go');
81
+
82
+ const [turn] = driver.getTurnSnapshots();
83
+ assert.equal(
84
+ turn.usage,
85
+ {
86
+ costUsd: 0.1,
87
+ // The prompt split into disjoint buckets: 1000 total, 900 read + 50 written,
88
+ // so 50 uncached. A reader of the log gets this without re-deriving it.
89
+ uncachedInputTokens: 50,
90
+ cacheReadTokens: 900,
91
+ cacheWriteTokens: 50,
92
+ outputTokens: 20,
93
+ },
94
+ 'the turn carries its own call’s usage',
95
+ );
96
+ // The turn and the message it produced are the SAME charge, so they must agree
97
+ // exactly — a log whose two views of one call disagree is worse than one view.
98
+ assert.equal(turn.usage, sumUsage(driver.getHistory()), 'turn agrees with the transcript total');
99
+ });
100
+
101
+ suite('prices every model call in a tool loop separately', async () => {
102
+ const driver = makeDriver(
103
+ agent({
104
+ name: 'a',
105
+ toolDefinitions: [def('work')],
106
+ toolHandlers: { work: async () => 'worked' },
107
+ }),
108
+ scriptedProvider([
109
+ priced({ content: '', cost: 0.02, toolCalls: [{ id: 'w1', name: 'work', args: {} }] }),
110
+ priced({ cost: 0.03 }),
111
+ ]),
112
+ );
113
+ await driver.sendMessage('go');
114
+
115
+ const snapshots = driver.getTurnSnapshots();
116
+ assert.is(snapshots.length, 2, 'one snapshot per model call, not per user turn');
117
+ assert.equal(
118
+ snapshots.map((s) => s.usage?.costUsd),
119
+ [0.02, 0.03],
120
+ );
121
+ });
122
+
123
+ suite('prices a blank response that produced no message', async () => {
124
+ // A blank turn is billed and then discarded before the retry, so the transcript can
125
+ // never account for it. The snapshot is the only record — which is the whole reason
126
+ // usage is stamped before the empty-response branch runs.
127
+ const driver = makeDriver(
128
+ agent({ name: 'a' }),
129
+ scriptedProvider([
130
+ priced({ content: '', cost: 0.07, outputTokens: 0 }),
131
+ priced({ cost: 0.03 }),
132
+ ]),
133
+ );
134
+ await driver.sendMessage('go');
135
+
136
+ const snapshots = driver.getTurnSnapshots();
137
+ assert.is(snapshots.length, 2, 'the retried attempt has its own snapshot');
138
+ assert.is(snapshots[0].usage?.costUsd, 0.07, 'the discarded attempt is still priced');
139
+ assert.is(
140
+ sumUsage(driver.getHistory()).costUsd,
141
+ 0.03,
142
+ 'and is genuinely absent from the transcript — that is the gap the snapshot fills',
143
+ );
144
+ });
145
+
146
+ suite('leaves usage undefined when the provider reports none', async () => {
147
+ // Absent, not zeroed: "this provider reports no usage" (e.g. Chrome's built-in model)
148
+ // must not read as a free call.
149
+ const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([]));
150
+ await driver.sendMessage('go');
151
+ assert.is(driver.getTurnSnapshots()[0].usage, undefined);
152
+ });
153
+
154
+ suite.run();
155
+
156
+ // ---------------------------------------------------------------------------
157
+ // The reasoning/narration/answer split must not multiply the cache buckets.
158
+ //
159
+ // One model response becomes up to three messages; only the last carries usage.
160
+ // The clearing list was written when usage meant cost + input + output, so when the
161
+ // cache buckets arrived they kept riding along on the display-only copies — and every
162
+ // bucket aggregate (session totals, usage rows, the exported log) counted a split
163
+ // turn's cache volume two or three times. The cost total hid it, since that comes
164
+ // from `cost`, which WAS cleared.
165
+ // ---------------------------------------------------------------------------
166
+
167
+ const split = createLogicSuite('ChatDriver response split usage');
168
+
169
+ split('counts a reasoning + answer split once, not twice', async () => {
170
+ const response = priced({ reasoning: 'thinking…' });
171
+ const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([response]));
172
+ await driver.sendMessage('go');
173
+
174
+ const history = driver.getHistory();
175
+ assert.is(
176
+ history.filter((m) => m.role === 'assistant').length,
177
+ 2,
178
+ 'the response did split into a reasoning message and an answer',
179
+ );
180
+ assert.equal(
181
+ sumUsage(history),
182
+ messageUsage(response),
183
+ 'the split totals exactly one call’s usage',
184
+ );
185
+ const reasoning = history.find((m) => m.category === 'reasoning')!;
186
+ assert.is(reasoning.cacheReadTokens, undefined, 'the display-only copy carries no cache volume');
187
+ assert.is(reasoning.cacheWriteTokens, undefined);
188
+ });
189
+
190
+ split('counts a reasoning + narration + answer split once', async () => {
191
+ // The three-way case: content alongside a tool call is interstitial narration, so this
192
+ // response produces reasoning + narration + the tool-call message. Worst case for the
193
+ // old behaviour — cache volume counted three times.
194
+ const first = priced({
195
+ content: 'let me look that up',
196
+ reasoning: 'thinking…',
197
+ cost: 0.02,
198
+ toolCalls: [{ id: 'w1', name: 'work', args: {} }],
199
+ });
200
+ const driver = makeDriver(
201
+ agent({
202
+ name: 'a',
203
+ toolDefinitions: [def('work')],
204
+ toolHandlers: { work: async () => 'worked' },
205
+ }),
206
+ scriptedProvider([first, priced({ cost: 0.03 })]),
207
+ );
208
+ await driver.sendMessage('go');
209
+
210
+ const total = sumUsage(driver.getHistory());
211
+ assert.is(total.costUsd, 0.05, 'two calls, each counted once');
212
+ assert.is(total.cacheReadTokens, 1800, '900 per call — not 2700 with the narration copies');
213
+ assert.is(total.cacheWriteTokens, 100);
214
+ assert.is(total.uncachedInputTokens, 100);
215
+ });
216
+
217
+ split.run();
218
+
219
+ // ---------------------------------------------------------------------------
220
+ // Per-turn model attribution across a tier switch.
221
+ //
222
+ // The shape this exists for is an agent whose `provider` selector varies by state —
223
+ // e.g. the showcase trade-operations agent: `({state}) => isPlanning(state) ? High : Low`,
224
+ // planning on sonnet and executing on haiku. Each call has to name the model that ran
225
+ // it, on the turn itself: joining a turn to the message after it works for a normal
226
+ // step but not for a call that produced no message, and `provider.selected` fires only
227
+ // when the slot CHANGES, so the nearest event can be many turns back.
228
+ //
229
+ // The ordering trap: the snapshot is recorded BEFORE the provider is resolved for that
230
+ // call, so stamping `lastResolvedModel` at creation time yields the previous call's
231
+ // model — wrong on exactly the turn the tier changes.
232
+ // ---------------------------------------------------------------------------
233
+
234
+ const tiers = createLogicSuite('ChatDriver per-turn model attribution');
235
+
236
+ /** A tiered registry: named slots, each reporting its own model via `getStatus`. */
237
+ const tieredRegistry = (
238
+ slots: Record<string, AIProvider>,
239
+ defaultName: string,
240
+ ): AIProviderRegistry =>
241
+ ({
242
+ get: (name: string) => slots[name],
243
+ default: () => slots[defaultName],
244
+ defaultName: () => defaultName,
245
+ names: () => Object.keys(slots),
246
+ getStatus: async () => null,
247
+ listStatuses: async () => [],
248
+ }) as AIProviderRegistry;
249
+
250
+ const tierProvider = (model: string, responses: ChatMessage[]): AIProvider => {
251
+ const queue = [...responses];
252
+ return {
253
+ getStatus: async () => ({ model, provider: 'anthropic' }) as never,
254
+ chat: async () => queue.shift() ?? { role: 'assistant', content: 'done' },
255
+ };
256
+ };
257
+
258
+ tiers('attributes each step to the tier that ran it when the agent switches mid-flow', async () => {
259
+ // Responses deliberately carry NO `model` of their own — the common case for a
260
+ // transport that leaves attribution to the driver, and the one where a stale
261
+ // `lastResolved*` read would go unnoticed.
262
+ const slots = {
263
+ high: tierProvider('claude-sonnet-4-6', [
264
+ priced({
265
+ content: '',
266
+ cost: 0.05,
267
+ toolCalls: [{ id: 'p1', name: 'finish_planning', args: {} }],
268
+ }),
269
+ ]),
270
+ low: tierProvider('claude-haiku-4-5-20251001', [priced({ content: 'Booked.', cost: 0.001 })]),
271
+ };
272
+ // The "state" the selector reads, advanced by the tool — as a flow agent's machine does.
273
+ let planning = true;
274
+ const driver = new ChatDriver(tieredRegistry(slots, 'low'), {
275
+ maxToolIterations: 20,
276
+ maxFoldOperations: 5,
277
+ sessionKey: 'tiers',
278
+ });
279
+ driver.applyAgent(
280
+ agent({
281
+ name: 'Trade Operations',
282
+ provider: () => (planning ? 'high' : 'low'),
283
+ toolDefinitions: [def('finish_planning')],
284
+ toolHandlers: {
285
+ finish_planning: async () => {
286
+ planning = false;
287
+ return 'planned';
288
+ },
289
+ },
290
+ }),
291
+ );
292
+ await driver.sendMessage('book me a trade');
293
+
294
+ const snapshots = driver.getTurnSnapshots();
295
+ assert.equal(
296
+ snapshots.map((s) => [s.providerName, s.model, s.usage?.costUsd]),
297
+ [
298
+ ['high', 'claude-sonnet-4-6', 0.05],
299
+ ['low', 'claude-haiku-4-5-20251001', 0.001],
300
+ ],
301
+ 'the planning call is attributed to the high tier and the execution call to the low one',
302
+ );
303
+ assert.equal(
304
+ snapshots.map((s) => s.provider),
305
+ ['anthropic', 'anthropic'],
306
+ 'the vendor behind each slot is recorded too',
307
+ );
308
+
309
+ // A turn and the message it produced must never disagree about which model ran.
310
+ const assistantModels = driver
311
+ .getHistory()
312
+ .filter((m) => m.role === 'assistant' && m.cost != null)
313
+ .map((m) => m.model);
314
+ assert.equal(
315
+ assistantModels,
316
+ snapshots.map((s) => s.model),
317
+ );
318
+ });
319
+
320
+ tiers('records the SERVING model when a fallback answers on another model', async () => {
321
+ // A server-side fallback chain answers on a different model than the one requested.
322
+ // The transport stamps the real one; the turn must follow it rather than relabel the
323
+ // call as the tier's configured model — that would misattribute the spend.
324
+ const slots = {
325
+ high: tierProvider('claude-fable-5', [
326
+ priced({ content: 'answered by the fallback', model: 'claude-opus-4-8' }),
327
+ ]),
328
+ };
329
+ const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
330
+ maxToolIterations: 20,
331
+ maxFoldOperations: 5,
332
+ sessionKey: 'fallback',
333
+ });
334
+ driver.applyAgent(agent({ name: 'a', provider: 'high' }));
335
+ await driver.sendMessage('go');
336
+
337
+ const [turn] = driver.getTurnSnapshots();
338
+ assert.is(turn.model, 'claude-opus-4-8', 'the model that answered, not the one asked for');
339
+ assert.is(turn.providerName, 'high', 'the slot asked for is still recorded');
340
+ });
341
+
342
+ tiers('names the model and vendor on the provider.selected event, not just the slot', async () => {
343
+ clearMetaEventRegistry();
344
+ const slots = {
345
+ high: tierProvider('claude-sonnet-4-6', [priced({ content: 'hi' })]),
346
+ };
347
+ const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
348
+ maxToolIterations: 20,
349
+ maxFoldOperations: 5,
350
+ sessionKey: 'selected',
351
+ });
352
+ driver.applyAgent(agent({ name: 'a', provider: 'high' }));
353
+ await driver.sendMessage('go');
354
+
355
+ const selected = getMetaEvents('selected').find((e) => e.type === 'provider.selected');
356
+ assert.ok(selected, 'a provider.selected event is recorded');
357
+ assert.is(selected!.detail?.provider, 'high', 'the registry slot');
358
+ assert.is(selected!.detail?.model, 'claude-sonnet-4-6', 'and the model behind it');
359
+ assert.is(selected!.detail?.vendor, 'anthropic');
360
+ });
361
+
362
+ tiers.run();