@genesislcap/ai-assistant 15.12.0 → 15.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-assistant.api.json +245 -3
- package/dist/ai-assistant.d.ts +127 -7
- package/dist/chat-driver.cjs +79 -13
- package/dist/chat-driver.cjs.map +2 -2
- package/dist/chat-driver.mjs +76 -12
- package/dist/chat-driver.mjs.map +2 -2
- package/dist/custom-elements.json +452 -359
- package/dist/dts/chat-driver-node.d.ts +2 -2
- package/dist/dts/chat-driver-node.d.ts.map +1 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts +45 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts.map +1 -1
- package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts +2 -0
- package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts.map +1 -0
- package/dist/dts/main/main.d.ts +20 -1
- package/dist/dts/main/main.d.ts.map +1 -1
- package/dist/dts/state/debug-event-log.d.ts.map +1 -1
- package/dist/dts/state/persistence/diagnostics.d.ts +65 -8
- package/dist/dts/state/persistence/diagnostics.d.ts.map +1 -1
- package/dist/dts/state/persistence/index.d.ts +1 -1
- package/dist/dts/state/persistence/index.d.ts.map +1 -1
- package/dist/dts/state/persistence/session-persister.d.ts +4 -3
- package/dist/dts/state/persistence/session-persister.d.ts.map +1 -1
- package/dist/dts/utils/sum-usage.d.ts +20 -0
- package/dist/dts/utils/sum-usage.d.ts.map +1 -1
- package/dist/esm/chat-driver-node.js +12 -2
- package/dist/esm/components/chat-driver/chat-driver.js +60 -5
- package/dist/esm/components/chat-driver/chat-driver.turn-usage.test.js +268 -0
- package/dist/esm/main/main.js +53 -28
- package/dist/esm/state/debug-event-log.js +7 -2
- package/dist/esm/state/persistence/diagnostics.js +79 -16
- package/dist/esm/state/persistence/diagnostics.test.js +174 -1
- package/dist/esm/state/persistence/index.js +1 -1
- package/dist/esm/state/persistence/session-persister.js +13 -5
- package/dist/esm/state/persistence/session-persister.test.js +31 -0
- package/dist/esm/utils/sum-usage.js +43 -0
- package/dist/esm/utils/sum-usage.test.js +45 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/chat-driver-node.ts +12 -2
- package/src/components/chat-driver/chat-driver.ts +107 -6
- package/src/components/chat-driver/chat-driver.turn-usage.test.ts +362 -0
- package/src/main/main.ts +52 -23
- package/src/state/debug-event-log.ts +7 -2
- package/src/state/persistence/diagnostics.test.ts +208 -1
- package/src/state/persistence/diagnostics.ts +117 -15
- package/src/state/persistence/index.ts +1 -1
- package/src/state/persistence/session-persister.test.ts +37 -0
- package/src/state/persistence/session-persister.ts +13 -5
- package/src/utils/sum-usage.test.ts +52 -1
- package/src/utils/sum-usage.ts +45 -0
package/src/chat-driver-node.ts
CHANGED
|
@@ -105,7 +105,13 @@ export type {
|
|
|
105
105
|
// `addUsage`/`emptyUsage` are the composition primitives for a conversation that CONTINUES across
|
|
106
106
|
// rounds — see the guidance on `sumUsage` itself. `totalTokens` exists because the four buckets of
|
|
107
107
|
// an `AggregateUsage` are disjoint and safe to add, which the per-message fields are NOT.
|
|
108
|
-
|
|
108
|
+
//
|
|
109
|
+
// `messageUsage` is the single-message projection of the same arithmetic — what ONE request
|
|
110
|
+
// cost, in the same four-bucket shape. It is what prices a turn snapshot (see
|
|
111
|
+
// `TurnSnapshot.usage`), and the reason a consumer needs it separately is that the response
|
|
112
|
+
// to a *failed* attempt never reaches history, so `sumUsage` over the transcript cannot see
|
|
113
|
+
// that spend at all.
|
|
114
|
+
export { addUsage, emptyUsage, messageUsage, sumUsage, totalTokens } from './utils/sum-usage';
|
|
109
115
|
|
|
110
116
|
// Per-call projection, for the cases an aggregate cannot serve: usage rows for
|
|
111
117
|
// per-project attribution, a cost dashboard, or auditing which turns came back
|
|
@@ -128,5 +134,9 @@ export { clearSession, getMetaEvents } from './state/debug-event-log';
|
|
|
128
134
|
export type { MetaEvent } from './state/debug-event-log';
|
|
129
135
|
export { buildTimelineEntries } from './state/persistence/build-timeline-entries';
|
|
130
136
|
export type { TurnSnapshotLike } from './state/persistence/build-timeline-entries';
|
|
131
|
-
|
|
137
|
+
// `withFreshMetaSnapshot` matters to a headless consumer that stitches a STORED stream: the
|
|
138
|
+
// newest stored `meta-snapshot` is frozen at the last config-signature change, so its context
|
|
139
|
+
// and cost figures are as old as that. Pass a fresh snapshot of your own through this before
|
|
140
|
+
// `assembleDebugLog`, or state in your own output that the totals are historical.
|
|
141
|
+
export { assembleDebugLog, withFreshMetaSnapshot } from './state/persistence/diagnostics';
|
|
132
142
|
export type { DebugLog, DiagnosticEntry } from './state/persistence/diagnostics';
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
AggregateUsage,
|
|
2
3
|
AIProvider,
|
|
3
4
|
AIProviderRegistry,
|
|
4
5
|
AIProviderType,
|
|
@@ -70,7 +71,7 @@ import {
|
|
|
70
71
|
normalizeForProvider,
|
|
71
72
|
} from '../../utils/history-transform';
|
|
72
73
|
import { logger } from '../../utils/logger';
|
|
73
|
-
import { sumUsage } from '../../utils/sum-usage';
|
|
74
|
+
import { messageUsage, sumUsage } from '../../utils/sum-usage';
|
|
74
75
|
import { TOOL_FOLD_SYMBOL, type ToolFold } from '../../utils/tool-fold';
|
|
75
76
|
import type { AiDriver, AllAgentSummary } from '../ai-driver/ai-driver';
|
|
76
77
|
|
|
@@ -241,6 +242,46 @@ export interface TurnSnapshot {
|
|
|
241
242
|
agentLabel?: string;
|
|
242
243
|
/** Agent-supplied snapshot — machine state/context for stateful agents, undefined otherwise. */
|
|
243
244
|
agentSnapshot?: unknown;
|
|
245
|
+
/**
|
|
246
|
+
* Concrete model that ran this call (e.g. `'claude-sonnet-4-6'`) — the serving model
|
|
247
|
+
* where the provider reports one, else the model the resolved tier was configured
|
|
248
|
+
* with. Undefined when the provider exposes no `getStatus` and the transport stamped
|
|
249
|
+
* nothing.
|
|
250
|
+
*
|
|
251
|
+
* Recorded per call, so an agent whose `provider` selector varies by state (a tier
|
|
252
|
+
* switch between steps of a flow) has each step attributed to the model that actually
|
|
253
|
+
* ran it — without the reader having to join the turn to the message after it, which
|
|
254
|
+
* is impossible for a call that produced no message.
|
|
255
|
+
*/
|
|
256
|
+
model?: string;
|
|
257
|
+
/**
|
|
258
|
+
* Registry slot the provider resolved under for this call — a tier name like
|
|
259
|
+
* `'high'`/`'low'`, or the registry default's name. Kept alongside {@link
|
|
260
|
+
* TurnSnapshot.model} because they answer different questions: the slot is what the
|
|
261
|
+
* agent asked for, the model is what served it, and repointing a slot at a new model
|
|
262
|
+
* mid-session is only visible when both are recorded.
|
|
263
|
+
*/
|
|
264
|
+
providerName?: string;
|
|
265
|
+
/** Vendor behind the resolved slot (`'anthropic'`, `'gemini'`, …), when the provider reports it. */
|
|
266
|
+
provider?: AIProviderType;
|
|
267
|
+
/**
|
|
268
|
+
* What this one LLM call cost — the four disjoint token buckets plus USD, derived
|
|
269
|
+
* from the response's usage by `messageUsage`. Back-filled when the response lands
|
|
270
|
+
* (the rest of the snapshot is captured *before* the call), so it is `undefined`
|
|
271
|
+
* while the call is in flight, on providers that report no usage, and on a call that
|
|
272
|
+
* threw rather than returning — a malformed-call/truncation/402 error carries no usage
|
|
273
|
+
* block, so any tokens the provider billed for it are not recoverable here.
|
|
274
|
+
*
|
|
275
|
+
* A snapshot is one **model call**, not one user turn: every tool-loop iteration and
|
|
276
|
+
* every retried attempt records its own. That makes this the only record of spend on
|
|
277
|
+
* an attempt that produced no message — a blank or refused response is billed and
|
|
278
|
+
* then discarded (see the empty-response retries), so summing the transcript alone
|
|
279
|
+
* under-reports the turn.
|
|
280
|
+
*
|
|
281
|
+
* For a call that DID produce a message, this is the same money as that message's
|
|
282
|
+
* `cost`/token fields, not additional money — never add turn usage to message usage.
|
|
283
|
+
*/
|
|
284
|
+
usage?: AggregateUsage;
|
|
244
285
|
}
|
|
245
286
|
|
|
246
287
|
interface FoldStackFrame {
|
|
@@ -1140,7 +1181,14 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
1140
1181
|
if (resolvedName !== this.lastDispatchedProviderName) {
|
|
1141
1182
|
this.lastDispatchedProviderName = resolvedName;
|
|
1142
1183
|
recordMetaEvent(this.sessionKey, 'provider.selected', {
|
|
1184
|
+
// `provider` is the registry SLOT (a tier name like 'high'), kept under that key
|
|
1185
|
+
// for compatibility; `model` and `vendor` are what it resolved to. Recording all
|
|
1186
|
+
// three is the difference between "the agent switched to its high tier" and
|
|
1187
|
+
// knowing which model that actually was — a tier can be repointed mid-session,
|
|
1188
|
+
// and a slot name alone cannot distinguish anthropic from gemini.
|
|
1143
1189
|
provider: resolvedName,
|
|
1190
|
+
model: status.model,
|
|
1191
|
+
vendor: status.provider,
|
|
1144
1192
|
agent: this.activeAgentName,
|
|
1145
1193
|
});
|
|
1146
1194
|
this.dispatchEvent(
|
|
@@ -1304,13 +1352,17 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
1304
1352
|
* Push one snapshot to the ring buffer. Called inside `runToolLoop` just
|
|
1305
1353
|
* before each LLM call — that's the latest point where the prompt, tool
|
|
1306
1354
|
* surface, and agent state line up with what the model is about to see.
|
|
1355
|
+
*
|
|
1356
|
+
* Returns the pushed object so the caller can back-fill what only the response
|
|
1357
|
+
* knows (`usage`). Mutating it after the fact is safe whether or not the ring
|
|
1358
|
+
* buffer has since evicted it — an evicted snapshot is simply no longer exported.
|
|
1307
1359
|
*/
|
|
1308
1360
|
private recordTurnSnapshot(
|
|
1309
1361
|
resolvedSystemPrompt: string | undefined,
|
|
1310
1362
|
temperature: number | undefined,
|
|
1311
1363
|
toolChoice: ChatToolChoice | undefined,
|
|
1312
1364
|
tailContext: string | undefined,
|
|
1313
|
-
):
|
|
1365
|
+
): TurnSnapshot {
|
|
1314
1366
|
let agentSnapshot: unknown;
|
|
1315
1367
|
if (this.debugSnapshotter) {
|
|
1316
1368
|
try {
|
|
@@ -1324,7 +1376,7 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
1324
1376
|
}
|
|
1325
1377
|
const turnIndex = String(this.globalTurnIndex);
|
|
1326
1378
|
this.globalTurnIndex += 1;
|
|
1327
|
-
|
|
1379
|
+
const snapshot: TurnSnapshot = {
|
|
1328
1380
|
turnIndex,
|
|
1329
1381
|
timestamp: new Date().toISOString(),
|
|
1330
1382
|
agentName: this.activeAgentName,
|
|
@@ -1335,10 +1387,12 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
1335
1387
|
temperature,
|
|
1336
1388
|
toolChoice,
|
|
1337
1389
|
agentSnapshot,
|
|
1338
|
-
}
|
|
1390
|
+
};
|
|
1391
|
+
this.turnSnapshots.push(snapshot);
|
|
1339
1392
|
if (this.turnSnapshots.length > this.maxTurnSnapshots) {
|
|
1340
1393
|
this.turnSnapshots.shift();
|
|
1341
1394
|
}
|
|
1395
|
+
return snapshot;
|
|
1342
1396
|
}
|
|
1343
1397
|
|
|
1344
1398
|
/**
|
|
@@ -2759,7 +2813,12 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
2759
2813
|
// force when no tools are advertised.)
|
|
2760
2814
|
const effectiveToolChoice = resolvedToolChoice ?? (this.isSubAgent ? 'required' : undefined);
|
|
2761
2815
|
|
|
2762
|
-
this.recordTurnSnapshot(
|
|
2816
|
+
const turnSnapshot = this.recordTurnSnapshot(
|
|
2817
|
+
systemPrompt,
|
|
2818
|
+
resolvedTemperature,
|
|
2819
|
+
effectiveToolChoice,
|
|
2820
|
+
tailContext,
|
|
2821
|
+
);
|
|
2763
2822
|
|
|
2764
2823
|
// Capture the pending user input, then clear the slots BEFORE the chat
|
|
2765
2824
|
// call. `sendMessage` already appended the user message to `this.history`,
|
|
@@ -2809,6 +2868,20 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
2809
2868
|
// oxlint-disable-next-line no-await-in-loop
|
|
2810
2869
|
const activeProvider = await this.resolveProviderForTurn(promptCtx);
|
|
2811
2870
|
|
|
2871
|
+
// Attribute the turn to the tier/model it resolved. Stamped HERE, not inside
|
|
2872
|
+
// `recordTurnSnapshot`: the snapshot is taken before this line runs, so reading
|
|
2873
|
+
// `lastResolved*` there yields the PREVIOUS call's model — wrong on precisely the
|
|
2874
|
+
// turn where an agent's per-state `provider` selector switches tier, which is the
|
|
2875
|
+
// turn a reader is looking for. `model` is refined to the serving model once the
|
|
2876
|
+
// response lands (see below); until then — and on a call that throws — it is the
|
|
2877
|
+
// model we ASKED for, which is the only thing knowable at that point.
|
|
2878
|
+
if (this.lastResolvedProviderName !== undefined) {
|
|
2879
|
+
turnSnapshot.providerName = this.lastResolvedProviderName;
|
|
2880
|
+
}
|
|
2881
|
+
if (this.lastResolvedProvider !== undefined)
|
|
2882
|
+
turnSnapshot.provider = this.lastResolvedProvider;
|
|
2883
|
+
if (this.lastResolvedModel !== undefined) turnSnapshot.model = this.lastResolvedModel;
|
|
2884
|
+
|
|
2812
2885
|
let response: ChatMessage;
|
|
2813
2886
|
try {
|
|
2814
2887
|
// oxlint-disable-next-line no-await-in-loop
|
|
@@ -2995,6 +3068,19 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
2995
3068
|
response.providerName = this.lastResolvedProviderName;
|
|
2996
3069
|
}
|
|
2997
3070
|
|
|
3071
|
+
// Back-fill what this call cost onto the snapshot taken just before it, so the
|
|
3072
|
+
// exported debug log prices each model call next to the prompt/tools/state that
|
|
3073
|
+
// produced it (GENC-1480 follow-up). Stamped BEFORE the empty-response branch
|
|
3074
|
+
// below deliberately: a blank or refused response is billed and then thrown away,
|
|
3075
|
+
// so the snapshot is the only place that spend is ever recorded.
|
|
3076
|
+
turnSnapshot.usage = messageUsage(response);
|
|
3077
|
+
// Take the SERVING model over the requested one, now that it is known. `response.model`
|
|
3078
|
+
// was just filled from `lastResolvedModel` if the transport left it unset, so this is
|
|
3079
|
+
// the same rule the message gets — which is the point: a turn and the message it
|
|
3080
|
+
// produced must never disagree about which model ran, including when a server-side
|
|
3081
|
+
// fallback chain answered on a different model than the one we asked for.
|
|
3082
|
+
if (response.model !== undefined) turnSnapshot.model = response.model;
|
|
3083
|
+
|
|
2998
3084
|
const isThinkingStep = response.content && response.toolCalls?.length;
|
|
2999
3085
|
const isEmptyResponse = !response.content?.trim() && !response.toolCalls?.length;
|
|
3000
3086
|
// A pre-output refusal (safety-classifier decline, e.g. Fable 5 `stop_reason: 'refusal'`)
|
|
@@ -3060,8 +3146,23 @@ export class ChatDriver extends EventTarget implements AiDriver {
|
|
|
3060
3146
|
// `sumCosts`/`sumTokens` don't double-count and `contextTokens` reads it. Reasoning/narration are
|
|
3061
3147
|
// display-only (usage undefined) and are skipped when building the provider request. `model` /
|
|
3062
3148
|
// `provider` / `providerName` stay on every split message so each is still attributed.
|
|
3149
|
+
//
|
|
3150
|
+
// EVERY usage field has to be cleared here, not just the three the invariant was
|
|
3151
|
+
// originally written against — keep this list in step with the usage fields on
|
|
3152
|
+
// `ChatMessage`. The cache buckets arrived later (GENC-1475) and were left riding
|
|
3153
|
+
// along on the copies, so a response carrying reasoning AND narration counted its
|
|
3154
|
+
// cache read/write volume three times in `sumUsage`/`usageRows` — invisible in the
|
|
3155
|
+
// cost total (which comes from `cost`) but wrong in every bucket display and in the
|
|
3156
|
+
// exported log.
|
|
3063
3157
|
const { reasoning, ...rest } = response;
|
|
3064
|
-
const displayOnly = {
|
|
3158
|
+
const displayOnly = {
|
|
3159
|
+
cost: undefined,
|
|
3160
|
+
externalCostUsd: undefined,
|
|
3161
|
+
inputTokens: undefined,
|
|
3162
|
+
outputTokens: undefined,
|
|
3163
|
+
cacheReadTokens: undefined,
|
|
3164
|
+
cacheWriteTokens: undefined,
|
|
3165
|
+
};
|
|
3065
3166
|
if (reasoning) {
|
|
3066
3167
|
this.appendToHistory({
|
|
3067
3168
|
...rest,
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AIProvider,
|
|
3
|
+
AIProviderRegistry,
|
|
4
|
+
ChatMessage,
|
|
5
|
+
ChatToolDefinition,
|
|
6
|
+
} from '@genesislcap/foundation-ai';
|
|
7
|
+
import { assert, createLogicSuite } from '@genesislcap/foundation-testing';
|
|
8
|
+
import type { AgentConfig } from '../../config/config';
|
|
9
|
+
import { clearMetaEventRegistry, getMetaEvents } from '../../state/debug-event-log';
|
|
10
|
+
import { messageUsage, sumUsage } from '../../utils/sum-usage';
|
|
11
|
+
// Side-effect import — MUST come before `./chat-driver` so the driver subclasses
|
|
12
|
+
// jsdom's EventTarget rather than Node's native one. Mirrors chat-driver.test.ts.
|
|
13
|
+
import './align-event-globals';
|
|
14
|
+
import { ChatDriver } from './chat-driver';
|
|
15
|
+
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Per-call usage on the turn snapshots — the four token buckets plus USD priced
|
|
18
|
+
// onto each `kind: 'turn'` entry of the exported debug log.
|
|
19
|
+
//
|
|
20
|
+
// The snapshot is captured BEFORE the model call (that is where the prompt, tool
|
|
21
|
+
// surface and agent state line up with what the model saw), so usage has to be
|
|
22
|
+
// back-filled when the response lands. Two things follow, and both are asserted
|
|
23
|
+
// here: a completed call's turn agrees with the message it produced, and a call
|
|
24
|
+
// that produced NO message (a blank response, retried) is still priced — that
|
|
25
|
+
// spend appears nowhere in the transcript.
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
const makeRegistry = (provider: AIProvider): AIProviderRegistry => ({
|
|
29
|
+
get: () => provider,
|
|
30
|
+
default: () => provider,
|
|
31
|
+
defaultName: () => 'test',
|
|
32
|
+
names: () => ['test'],
|
|
33
|
+
getStatus: async () => null,
|
|
34
|
+
listStatuses: async () => [],
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
/** Answers from a queue, then ends the turn with a plain unpriced reply. */
|
|
38
|
+
const scriptedProvider = (responses: ChatMessage[]): AIProvider => {
|
|
39
|
+
const queue = [...responses];
|
|
40
|
+
return {
|
|
41
|
+
chat: async (): Promise<ChatMessage> => queue.shift() ?? { role: 'assistant', content: 'done' },
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const agent = (overrides: Partial<AgentConfig> & { name: string }): AgentConfig =>
|
|
46
|
+
({ description: 'test agent', ...overrides }) as AgentConfig;
|
|
47
|
+
|
|
48
|
+
const makeDriver = (config: AgentConfig, provider: AIProvider): ChatDriver => {
|
|
49
|
+
const driver = new ChatDriver(makeRegistry(provider), {
|
|
50
|
+
maxToolIterations: 50,
|
|
51
|
+
maxFoldOperations: 5,
|
|
52
|
+
sessionKey: '',
|
|
53
|
+
});
|
|
54
|
+
driver.applyAgent(config);
|
|
55
|
+
return driver;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const def = (name: string): ChatToolDefinition => ({
|
|
59
|
+
name,
|
|
60
|
+
description: `${name} tool`,
|
|
61
|
+
parameters: { type: 'object', properties: {} },
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
/** One priced response: a 1000-token prompt that was mostly a cache hit. */
|
|
65
|
+
const priced = (over: Partial<ChatMessage> = {}): ChatMessage => ({
|
|
66
|
+
role: 'assistant',
|
|
67
|
+
content: 'answer',
|
|
68
|
+
cost: 0.1,
|
|
69
|
+
inputTokens: 1000,
|
|
70
|
+
cacheReadTokens: 900,
|
|
71
|
+
cacheWriteTokens: 50,
|
|
72
|
+
outputTokens: 20,
|
|
73
|
+
...over,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const suite = createLogicSuite('ChatDriver per-turn usage');
|
|
77
|
+
|
|
78
|
+
suite('prices each turn snapshot with the four buckets and USD of its own call', async () => {
|
|
79
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([priced()]));
|
|
80
|
+
await driver.sendMessage('go');
|
|
81
|
+
|
|
82
|
+
const [turn] = driver.getTurnSnapshots();
|
|
83
|
+
assert.equal(
|
|
84
|
+
turn.usage,
|
|
85
|
+
{
|
|
86
|
+
costUsd: 0.1,
|
|
87
|
+
// The prompt split into disjoint buckets: 1000 total, 900 read + 50 written,
|
|
88
|
+
// so 50 uncached. A reader of the log gets this without re-deriving it.
|
|
89
|
+
uncachedInputTokens: 50,
|
|
90
|
+
cacheReadTokens: 900,
|
|
91
|
+
cacheWriteTokens: 50,
|
|
92
|
+
outputTokens: 20,
|
|
93
|
+
},
|
|
94
|
+
'the turn carries its own call’s usage',
|
|
95
|
+
);
|
|
96
|
+
// The turn and the message it produced are the SAME charge, so they must agree
|
|
97
|
+
// exactly — a log whose two views of one call disagree is worse than one view.
|
|
98
|
+
assert.equal(turn.usage, sumUsage(driver.getHistory()), 'turn agrees with the transcript total');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
suite('prices every model call in a tool loop separately', async () => {
|
|
102
|
+
const driver = makeDriver(
|
|
103
|
+
agent({
|
|
104
|
+
name: 'a',
|
|
105
|
+
toolDefinitions: [def('work')],
|
|
106
|
+
toolHandlers: { work: async () => 'worked' },
|
|
107
|
+
}),
|
|
108
|
+
scriptedProvider([
|
|
109
|
+
priced({ content: '', cost: 0.02, toolCalls: [{ id: 'w1', name: 'work', args: {} }] }),
|
|
110
|
+
priced({ cost: 0.03 }),
|
|
111
|
+
]),
|
|
112
|
+
);
|
|
113
|
+
await driver.sendMessage('go');
|
|
114
|
+
|
|
115
|
+
const snapshots = driver.getTurnSnapshots();
|
|
116
|
+
assert.is(snapshots.length, 2, 'one snapshot per model call, not per user turn');
|
|
117
|
+
assert.equal(
|
|
118
|
+
snapshots.map((s) => s.usage?.costUsd),
|
|
119
|
+
[0.02, 0.03],
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
suite('prices a blank response that produced no message', async () => {
|
|
124
|
+
// A blank turn is billed and then discarded before the retry, so the transcript can
|
|
125
|
+
// never account for it. The snapshot is the only record — which is the whole reason
|
|
126
|
+
// usage is stamped before the empty-response branch runs.
|
|
127
|
+
const driver = makeDriver(
|
|
128
|
+
agent({ name: 'a' }),
|
|
129
|
+
scriptedProvider([
|
|
130
|
+
priced({ content: '', cost: 0.07, outputTokens: 0 }),
|
|
131
|
+
priced({ cost: 0.03 }),
|
|
132
|
+
]),
|
|
133
|
+
);
|
|
134
|
+
await driver.sendMessage('go');
|
|
135
|
+
|
|
136
|
+
const snapshots = driver.getTurnSnapshots();
|
|
137
|
+
assert.is(snapshots.length, 2, 'the retried attempt has its own snapshot');
|
|
138
|
+
assert.is(snapshots[0].usage?.costUsd, 0.07, 'the discarded attempt is still priced');
|
|
139
|
+
assert.is(
|
|
140
|
+
sumUsage(driver.getHistory()).costUsd,
|
|
141
|
+
0.03,
|
|
142
|
+
'and is genuinely absent from the transcript — that is the gap the snapshot fills',
|
|
143
|
+
);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
suite('leaves usage undefined when the provider reports none', async () => {
|
|
147
|
+
// Absent, not zeroed: "this provider reports no usage" (e.g. Chrome's built-in model)
|
|
148
|
+
// must not read as a free call.
|
|
149
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([]));
|
|
150
|
+
await driver.sendMessage('go');
|
|
151
|
+
assert.is(driver.getTurnSnapshots()[0].usage, undefined);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
suite.run();
|
|
155
|
+
|
|
156
|
+
// ---------------------------------------------------------------------------
|
|
157
|
+
// The reasoning/narration/answer split must not multiply the cache buckets.
|
|
158
|
+
//
|
|
159
|
+
// One model response becomes up to three messages; only the last carries usage.
|
|
160
|
+
// The clearing list was written when usage meant cost + input + output, so when the
|
|
161
|
+
// cache buckets arrived they kept riding along on the display-only copies — and every
|
|
162
|
+
// bucket aggregate (session totals, usage rows, the exported log) counted a split
|
|
163
|
+
// turn's cache volume two or three times. The cost total hid it, since that comes
|
|
164
|
+
// from `cost`, which WAS cleared.
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
|
|
167
|
+
const split = createLogicSuite('ChatDriver response split usage');
|
|
168
|
+
|
|
169
|
+
split('counts a reasoning + answer split once, not twice', async () => {
|
|
170
|
+
const response = priced({ reasoning: 'thinking…' });
|
|
171
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([response]));
|
|
172
|
+
await driver.sendMessage('go');
|
|
173
|
+
|
|
174
|
+
const history = driver.getHistory();
|
|
175
|
+
assert.is(
|
|
176
|
+
history.filter((m) => m.role === 'assistant').length,
|
|
177
|
+
2,
|
|
178
|
+
'the response did split into a reasoning message and an answer',
|
|
179
|
+
);
|
|
180
|
+
assert.equal(
|
|
181
|
+
sumUsage(history),
|
|
182
|
+
messageUsage(response),
|
|
183
|
+
'the split totals exactly one call’s usage',
|
|
184
|
+
);
|
|
185
|
+
const reasoning = history.find((m) => m.category === 'reasoning')!;
|
|
186
|
+
assert.is(reasoning.cacheReadTokens, undefined, 'the display-only copy carries no cache volume');
|
|
187
|
+
assert.is(reasoning.cacheWriteTokens, undefined);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
split('counts a reasoning + narration + answer split once', async () => {
|
|
191
|
+
// The three-way case: content alongside a tool call is interstitial narration, so this
|
|
192
|
+
// response produces reasoning + narration + the tool-call message. Worst case for the
|
|
193
|
+
// old behaviour — cache volume counted three times.
|
|
194
|
+
const first = priced({
|
|
195
|
+
content: 'let me look that up',
|
|
196
|
+
reasoning: 'thinking…',
|
|
197
|
+
cost: 0.02,
|
|
198
|
+
toolCalls: [{ id: 'w1', name: 'work', args: {} }],
|
|
199
|
+
});
|
|
200
|
+
const driver = makeDriver(
|
|
201
|
+
agent({
|
|
202
|
+
name: 'a',
|
|
203
|
+
toolDefinitions: [def('work')],
|
|
204
|
+
toolHandlers: { work: async () => 'worked' },
|
|
205
|
+
}),
|
|
206
|
+
scriptedProvider([first, priced({ cost: 0.03 })]),
|
|
207
|
+
);
|
|
208
|
+
await driver.sendMessage('go');
|
|
209
|
+
|
|
210
|
+
const total = sumUsage(driver.getHistory());
|
|
211
|
+
assert.is(total.costUsd, 0.05, 'two calls, each counted once');
|
|
212
|
+
assert.is(total.cacheReadTokens, 1800, '900 per call — not 2700 with the narration copies');
|
|
213
|
+
assert.is(total.cacheWriteTokens, 100);
|
|
214
|
+
assert.is(total.uncachedInputTokens, 100);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
split.run();
|
|
218
|
+
|
|
219
|
+
// ---------------------------------------------------------------------------
|
|
220
|
+
// Per-turn model attribution across a tier switch.
|
|
221
|
+
//
|
|
222
|
+
// The shape this exists for is an agent whose `provider` selector varies by state —
|
|
223
|
+
// e.g. the showcase trade-operations agent: `({state}) => isPlanning(state) ? High : Low`,
|
|
224
|
+
// planning on sonnet and executing on haiku. Each call has to name the model that ran
|
|
225
|
+
// it, on the turn itself: joining a turn to the message after it works for a normal
|
|
226
|
+
// step but not for a call that produced no message, and `provider.selected` fires only
|
|
227
|
+
// when the slot CHANGES, so the nearest event can be many turns back.
|
|
228
|
+
//
|
|
229
|
+
// The ordering trap: the snapshot is recorded BEFORE the provider is resolved for that
|
|
230
|
+
// call, so stamping `lastResolvedModel` at creation time yields the previous call's
|
|
231
|
+
// model — wrong on exactly the turn the tier changes.
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
|
|
234
|
+
const tiers = createLogicSuite('ChatDriver per-turn model attribution');
|
|
235
|
+
|
|
236
|
+
/** A tiered registry: named slots, each reporting its own model via `getStatus`. */
|
|
237
|
+
const tieredRegistry = (
|
|
238
|
+
slots: Record<string, AIProvider>,
|
|
239
|
+
defaultName: string,
|
|
240
|
+
): AIProviderRegistry =>
|
|
241
|
+
({
|
|
242
|
+
get: (name: string) => slots[name],
|
|
243
|
+
default: () => slots[defaultName],
|
|
244
|
+
defaultName: () => defaultName,
|
|
245
|
+
names: () => Object.keys(slots),
|
|
246
|
+
getStatus: async () => null,
|
|
247
|
+
listStatuses: async () => [],
|
|
248
|
+
}) as AIProviderRegistry;
|
|
249
|
+
|
|
250
|
+
const tierProvider = (model: string, responses: ChatMessage[]): AIProvider => {
|
|
251
|
+
const queue = [...responses];
|
|
252
|
+
return {
|
|
253
|
+
getStatus: async () => ({ model, provider: 'anthropic' }) as never,
|
|
254
|
+
chat: async () => queue.shift() ?? { role: 'assistant', content: 'done' },
|
|
255
|
+
};
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
tiers('attributes each step to the tier that ran it when the agent switches mid-flow', async () => {
|
|
259
|
+
// Responses deliberately carry NO `model` of their own — the common case for a
|
|
260
|
+
// transport that leaves attribution to the driver, and the one where a stale
|
|
261
|
+
// `lastResolved*` read would go unnoticed.
|
|
262
|
+
const slots = {
|
|
263
|
+
high: tierProvider('claude-sonnet-4-6', [
|
|
264
|
+
priced({
|
|
265
|
+
content: '',
|
|
266
|
+
cost: 0.05,
|
|
267
|
+
toolCalls: [{ id: 'p1', name: 'finish_planning', args: {} }],
|
|
268
|
+
}),
|
|
269
|
+
]),
|
|
270
|
+
low: tierProvider('claude-haiku-4-5-20251001', [priced({ content: 'Booked.', cost: 0.001 })]),
|
|
271
|
+
};
|
|
272
|
+
// The "state" the selector reads, advanced by the tool — as a flow agent's machine does.
|
|
273
|
+
let planning = true;
|
|
274
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'low'), {
|
|
275
|
+
maxToolIterations: 20,
|
|
276
|
+
maxFoldOperations: 5,
|
|
277
|
+
sessionKey: 'tiers',
|
|
278
|
+
});
|
|
279
|
+
driver.applyAgent(
|
|
280
|
+
agent({
|
|
281
|
+
name: 'Trade Operations',
|
|
282
|
+
provider: () => (planning ? 'high' : 'low'),
|
|
283
|
+
toolDefinitions: [def('finish_planning')],
|
|
284
|
+
toolHandlers: {
|
|
285
|
+
finish_planning: async () => {
|
|
286
|
+
planning = false;
|
|
287
|
+
return 'planned';
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
}),
|
|
291
|
+
);
|
|
292
|
+
await driver.sendMessage('book me a trade');
|
|
293
|
+
|
|
294
|
+
const snapshots = driver.getTurnSnapshots();
|
|
295
|
+
assert.equal(
|
|
296
|
+
snapshots.map((s) => [s.providerName, s.model, s.usage?.costUsd]),
|
|
297
|
+
[
|
|
298
|
+
['high', 'claude-sonnet-4-6', 0.05],
|
|
299
|
+
['low', 'claude-haiku-4-5-20251001', 0.001],
|
|
300
|
+
],
|
|
301
|
+
'the planning call is attributed to the high tier and the execution call to the low one',
|
|
302
|
+
);
|
|
303
|
+
assert.equal(
|
|
304
|
+
snapshots.map((s) => s.provider),
|
|
305
|
+
['anthropic', 'anthropic'],
|
|
306
|
+
'the vendor behind each slot is recorded too',
|
|
307
|
+
);
|
|
308
|
+
|
|
309
|
+
// A turn and the message it produced must never disagree about which model ran.
|
|
310
|
+
const assistantModels = driver
|
|
311
|
+
.getHistory()
|
|
312
|
+
.filter((m) => m.role === 'assistant' && m.cost != null)
|
|
313
|
+
.map((m) => m.model);
|
|
314
|
+
assert.equal(
|
|
315
|
+
assistantModels,
|
|
316
|
+
snapshots.map((s) => s.model),
|
|
317
|
+
);
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
tiers('records the SERVING model when a fallback answers on another model', async () => {
|
|
321
|
+
// A server-side fallback chain answers on a different model than the one requested.
|
|
322
|
+
// The transport stamps the real one; the turn must follow it rather than relabel the
|
|
323
|
+
// call as the tier's configured model — that would misattribute the spend.
|
|
324
|
+
const slots = {
|
|
325
|
+
high: tierProvider('claude-fable-5', [
|
|
326
|
+
priced({ content: 'answered by the fallback', model: 'claude-opus-4-8' }),
|
|
327
|
+
]),
|
|
328
|
+
};
|
|
329
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
|
|
330
|
+
maxToolIterations: 20,
|
|
331
|
+
maxFoldOperations: 5,
|
|
332
|
+
sessionKey: 'fallback',
|
|
333
|
+
});
|
|
334
|
+
driver.applyAgent(agent({ name: 'a', provider: 'high' }));
|
|
335
|
+
await driver.sendMessage('go');
|
|
336
|
+
|
|
337
|
+
const [turn] = driver.getTurnSnapshots();
|
|
338
|
+
assert.is(turn.model, 'claude-opus-4-8', 'the model that answered, not the one asked for');
|
|
339
|
+
assert.is(turn.providerName, 'high', 'the slot asked for is still recorded');
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
tiers('names the model and vendor on the provider.selected event, not just the slot', async () => {
|
|
343
|
+
clearMetaEventRegistry();
|
|
344
|
+
const slots = {
|
|
345
|
+
high: tierProvider('claude-sonnet-4-6', [priced({ content: 'hi' })]),
|
|
346
|
+
};
|
|
347
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
|
|
348
|
+
maxToolIterations: 20,
|
|
349
|
+
maxFoldOperations: 5,
|
|
350
|
+
sessionKey: 'selected',
|
|
351
|
+
});
|
|
352
|
+
driver.applyAgent(agent({ name: 'a', provider: 'high' }));
|
|
353
|
+
await driver.sendMessage('go');
|
|
354
|
+
|
|
355
|
+
const selected = getMetaEvents('selected').find((e) => e.type === 'provider.selected');
|
|
356
|
+
assert.ok(selected, 'a provider.selected event is recorded');
|
|
357
|
+
assert.is(selected!.detail?.provider, 'high', 'the registry slot');
|
|
358
|
+
assert.is(selected!.detail?.model, 'claude-sonnet-4-6', 'and the model behind it');
|
|
359
|
+
assert.is(selected!.detail?.vendor, 'anthropic');
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
tiers.run();
|