@genesislcap/ai-assistant 15.12.0 → 15.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-assistant.api.json +245 -3
- package/dist/ai-assistant.d.ts +127 -7
- package/dist/chat-driver.cjs +79 -13
- package/dist/chat-driver.cjs.map +2 -2
- package/dist/chat-driver.mjs +76 -12
- package/dist/chat-driver.mjs.map +2 -2
- package/dist/custom-elements.json +99 -6
- package/dist/dts/chat-driver-node.d.ts +2 -2
- package/dist/dts/chat-driver-node.d.ts.map +1 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts +45 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts.map +1 -1
- package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts +2 -0
- package/dist/dts/components/chat-driver/chat-driver.turn-usage.test.d.ts.map +1 -0
- package/dist/dts/main/main.d.ts +20 -1
- package/dist/dts/main/main.d.ts.map +1 -1
- package/dist/dts/state/debug-event-log.d.ts.map +1 -1
- package/dist/dts/state/persistence/diagnostics.d.ts +65 -8
- package/dist/dts/state/persistence/diagnostics.d.ts.map +1 -1
- package/dist/dts/state/persistence/index.d.ts +1 -1
- package/dist/dts/state/persistence/index.d.ts.map +1 -1
- package/dist/dts/state/persistence/session-persister.d.ts +4 -3
- package/dist/dts/state/persistence/session-persister.d.ts.map +1 -1
- package/dist/dts/utils/sum-usage.d.ts +20 -0
- package/dist/dts/utils/sum-usage.d.ts.map +1 -1
- package/dist/esm/chat-driver-node.js +12 -2
- package/dist/esm/components/chat-driver/chat-driver.js +60 -5
- package/dist/esm/components/chat-driver/chat-driver.turn-usage.test.js +268 -0
- package/dist/esm/main/main.js +53 -28
- package/dist/esm/state/debug-event-log.js +7 -2
- package/dist/esm/state/persistence/diagnostics.js +79 -16
- package/dist/esm/state/persistence/diagnostics.test.js +174 -1
- package/dist/esm/state/persistence/index.js +1 -1
- package/dist/esm/state/persistence/session-persister.js +13 -5
- package/dist/esm/state/persistence/session-persister.test.js +31 -0
- package/dist/esm/utils/sum-usage.js +43 -0
- package/dist/esm/utils/sum-usage.test.js +45 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/chat-driver-node.ts +12 -2
- package/src/components/chat-driver/chat-driver.ts +107 -6
- package/src/components/chat-driver/chat-driver.turn-usage.test.ts +362 -0
- package/src/main/main.ts +52 -23
- package/src/state/debug-event-log.ts +7 -2
- package/src/state/persistence/diagnostics.test.ts +208 -1
- package/src/state/persistence/diagnostics.ts +117 -15
- package/src/state/persistence/index.ts +1 -1
- package/src/state/persistence/session-persister.test.ts +37 -0
- package/src/state/persistence/session-persister.ts +13 -5
- package/src/utils/sum-usage.test.ts +52 -1
- package/src/utils/sum-usage.ts +45 -0
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
* Server-saved diagnostics (GENC-1351 §5.8) — the forward-only stream that lets
|
|
3
3
|
* the debug log persist across a whole session lifetime, not just the current
|
|
4
4
|
* page load. This module owns the *pure* pieces: the entry type and the
|
|
5
|
-
* reassembly back into the `{ readme, timeline, meta }` debug-log
|
|
6
|
-
* element owns capture/cadence; the provider owns storage layout.
|
|
5
|
+
* reassembly back into the `{ readme, sessionUsage, timeline, meta }` debug-log
|
|
6
|
+
* shape. The element owns capture/cadence; the provider owns storage layout.
|
|
7
7
|
*
|
|
8
8
|
* @packageDocumentation
|
|
9
9
|
*/
|
|
10
|
+
import type { AggregateUsage } from '@genesislcap/foundation-ai';
|
|
10
11
|
/**
|
|
11
12
|
* One entry in the diagnostics stream — the debug-log timeline-entry union
|
|
12
13
|
* (`message` / `turn` / `event`, as `getDebugLog()` builds) plus a point-in-time
|
|
13
14
|
* **`meta-snapshot`** carrying the export-time `meta` block (agent summary,
|
|
14
15
|
* context, active debug snapshot, …). Kept permissive: a provider stores and
|
|
15
16
|
* returns these opaquely, and reassembly reads only `kind`, `timestamp` (for
|
|
16
|
-
* ordering), and `meta` (on a snapshot).
|
|
17
|
+
* ordering), `usage` (on a turn) and `meta` (on a snapshot).
|
|
17
18
|
*
|
|
18
19
|
* @public
|
|
19
20
|
*/
|
|
@@ -23,6 +24,16 @@ export interface DiagnosticEntry {
|
|
|
23
24
|
timestamp?: string;
|
|
24
25
|
/** Present on `meta-snapshot` entries: the export-time `meta` block. */
|
|
25
26
|
meta?: unknown;
|
|
27
|
+
/**
|
|
28
|
+
* Present on a `turn` entry once its model call has returned and reported usage —
|
|
29
|
+
* the four token buckets plus USD for that one call (`TurnSnapshot.usage`).
|
|
30
|
+
*
|
|
31
|
+
* Read by the forward-capture delta as well as by readers: the turn entry is
|
|
32
|
+
* created *before* the call, so a flush that lands mid-call persists it without
|
|
33
|
+
* usage, and the delta re-emits it once (see `collectDiagnosticsDelta`) so the
|
|
34
|
+
* priced copy reaches the stream. {@link assembleDebugLog} then collapses the pair.
|
|
35
|
+
*/
|
|
36
|
+
usage?: AggregateUsage;
|
|
26
37
|
/**
|
|
27
38
|
* Present on `meta-snapshot` entries: a stable signature of the near-static config
|
|
28
39
|
* (excludes volatile timestamp/context/debug-snapshot), used by the forward-capture
|
|
@@ -35,6 +46,27 @@ export interface DiagnosticEntry {
|
|
|
35
46
|
/** The reassembled debug log — the exact shape `getDebugLog()` returns. @public */
|
|
36
47
|
export interface DebugLog {
|
|
37
48
|
readme: readonly string[];
|
|
49
|
+
/**
|
|
50
|
+
* Session usage — the four token buckets plus USD cost — lifted out of the newest
|
|
51
|
+
* `meta-snapshot`'s `meta.context.sessionUsage` so the headline spend figures sit at
|
|
52
|
+
* the top of an exported log rather than buried under the (large) `meta` block. The
|
|
53
|
+
* same object, not a second derivation.
|
|
54
|
+
*
|
|
55
|
+
* `undefined` when the stream carries no `meta-snapshot`, or one written before the
|
|
56
|
+
* field existed. Never re-derived from the timeline: the per-request costs the
|
|
57
|
+
* transports stamped are authoritative, and a ring-buffered timeline can have lost
|
|
58
|
+
* entries the total still legitimately counts.
|
|
59
|
+
*
|
|
60
|
+
* **As current as the snapshot it came from, which is not automatically "now".** The
|
|
61
|
+
* forward-capture delta only re-appends a `meta-snapshot` when the near-static config
|
|
62
|
+
* signature changes, so the newest STORED snapshot of a session whose config never
|
|
63
|
+
* changed is its first — with the totals frozen there. A caller reassembling a stored
|
|
64
|
+
* stream on its own therefore gets stale figures unless it appends an export-time
|
|
65
|
+
* snapshot first: see {@link withFreshMetaSnapshot}, which is what the assistant's own
|
|
66
|
+
* download path does. `meta.timestamp` against the newest timeline entry tells you
|
|
67
|
+
* which case you are holding.
|
|
68
|
+
*/
|
|
69
|
+
sessionUsage?: AggregateUsage;
|
|
38
70
|
timeline: DiagnosticEntry[];
|
|
39
71
|
meta: unknown;
|
|
40
72
|
}
|
|
@@ -58,16 +90,41 @@ export interface DebugLog {
|
|
|
58
90
|
*/
|
|
59
91
|
export declare function diagnosticsMessageKey(entry: DiagnosticEntry): string | null;
|
|
60
92
|
/**
|
|
61
|
-
* Reassemble diagnostic entries into the `{ readme, timeline, meta }`
|
|
62
|
-
* shape (GENC-1351 §5.8). `timeline` = the `message`/`turn`/`event` entries sorted
|
|
93
|
+
* Reassemble diagnostic entries into the `{ readme, sessionUsage, timeline, meta }`
|
|
94
|
+
* debug-log shape (GENC-1351 §5.8). `timeline` = the `message`/`turn`/`event` entries sorted
|
|
63
95
|
* by ISO `timestamp` (kind-rank tie-break). `meta` = the **newest** `meta-snapshot`'s
|
|
64
96
|
* block only — a "state at export" photo, matching the single-page log. The older
|
|
65
97
|
* snapshots are intentionally dropped: the per-turn evolution they would show is
|
|
66
98
|
* already in the timeline (`turn.agentSnapshot` + `context.updated` events), so
|
|
67
99
|
* keeping them would just repeat the bulky, near-static `agentSummary`. `readme` =
|
|
68
|
-
* the passed (current) constant.
|
|
69
|
-
*
|
|
70
|
-
*
|
|
100
|
+
* the passed (current) constant. `sessionUsage` = that same newest snapshot's
|
|
101
|
+
* `meta.context.sessionUsage`, lifted to the top level. Pure — reused for both the
|
|
102
|
+
* live current-page log and the reassembled lifetime log stitched from the persisted
|
|
103
|
+
* stream, so both come out shape-identical.
|
|
104
|
+
*
|
|
105
|
+
* NOTE for a caller stitching a STORED stream (a headless consumer harvesting its own
|
|
106
|
+
* lifetime log, say): the newest stored `meta-snapshot` is frozen at the last
|
|
107
|
+
* config-signature change, so `meta` and `sessionUsage` are as old as that unless you
|
|
108
|
+
* pass a fresh snapshot of your own — {@link withFreshMetaSnapshot} does exactly that,
|
|
109
|
+
* and the element's download path goes through it.
|
|
71
110
|
*/
|
|
72
111
|
export declare function assembleDebugLog(entries: DiagnosticEntry[], readme: readonly string[]): DebugLog;
|
|
112
|
+
/**
|
|
113
|
+
* A stored diagnostics stream with its `meta-snapshot`s replaced by `fresh` — what to
|
|
114
|
+
* pass {@link assembleDebugLog} when reassembling a persisted stream for export.
|
|
115
|
+
*
|
|
116
|
+
* The stored snapshots are dropped rather than out-timestamped. `assembleDebugLog` keeps
|
|
117
|
+
* whichever claims the later time, and a stream written on another machine can carry a
|
|
118
|
+
* skewed clock; `fresh` is built from live state and so is the more current by
|
|
119
|
+
* construction. Only one snapshot ever survives reassembly, so the count is unchanged —
|
|
120
|
+
* though a snapshot taken while the driver is unwired (mid-popout) can leave
|
|
121
|
+
* `activeFoldStack`/`activeDebugSnapshot` unset where a stored one had them. The figures
|
|
122
|
+
* this exists for — the context block and `sessionUsage` — read the session store and
|
|
123
|
+
* element props, which survive an unwire.
|
|
124
|
+
*
|
|
125
|
+
* Pure, and separate from the element for that reason: the swap it performs is the fix
|
|
126
|
+
* for a silent bug (a lifetime log reporting near-zero spend against a transcript full of
|
|
127
|
+
* priced messages), so it needs to be testable without mounting anything.
|
|
128
|
+
*/
|
|
129
|
+
export declare function withFreshMetaSnapshot(stored: readonly DiagnosticEntry[], fresh: DiagnosticEntry): DiagnosticEntry[];
|
|
73
130
|
//# sourceMappingURL=diagnostics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/diagnostics.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAC;IACrD,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,mFAAmF;AACnF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAI3E;
|
|
1
|
+
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/diagnostics.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE;;;;;;;;;GASG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAC;IACrD,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,mFAAmF;AACnF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAI3E;AA+BD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,QAAQ,CA+ChG;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,SAAS,eAAe,EAAE,EAClC,KAAK,EAAE,eAAe,GACrB,eAAe,EAAE,CAEnB"}
|
|
@@ -2,6 +2,6 @@ export { PERSISTED_SESSION_VERSION } from './session-snapshot';
|
|
|
2
2
|
export type { PersistedSession } from './session-snapshot';
|
|
3
3
|
export { WebStorageSessionProvider } from './session-persistence-provider';
|
|
4
4
|
export type { SessionPersistenceProvider, SessionPersistenceConfig, SessionPreferences, } from './session-persistence-provider';
|
|
5
|
-
export { assembleDebugLog } from './diagnostics';
|
|
5
|
+
export { assembleDebugLog, withFreshMetaSnapshot } from './diagnostics';
|
|
6
6
|
export type { DiagnosticEntry, DebugLog } from './diagnostics';
|
|
7
7
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,YAAY,EACV,0BAA0B,EAC1B,wBAAwB,EACxB,kBAAkB,GACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,YAAY,EACV,0BAA0B,EAC1B,wBAAwB,EACxB,kBAAkB,GACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -162,9 +162,10 @@ export declare class SessionPersister {
|
|
|
162
162
|
flushDiagnostics(): Promise<void>;
|
|
163
163
|
/**
|
|
164
164
|
* The diagnostic entries not yet appended this load — the forward-capture delta
|
|
165
|
-
* (§5.8). Skips messages present at restore or already appended; emits each
|
|
166
|
-
* (by
|
|
167
|
-
*
|
|
165
|
+
* (§5.8). Skips messages present at restore or already appended; emits each meta-event
|
|
166
|
+
* (by index) once and each turn (by turn-index) once unpriced plus once more when its
|
|
167
|
+
* per-call `usage` lands; appends a `meta-snapshot` only when the block changed.
|
|
168
|
+
* Advances the shared per-session cursors as it goes.
|
|
168
169
|
*
|
|
169
170
|
* TODO(GENC-1461, longer-term): this whole delta step exists only because the element *pulls* the
|
|
170
171
|
* ENTIRE debug log on every throttled flush (`getDiagnosticEntries()` returns the full timeline),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-persister.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/session-persister.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sCAAsC,CAAC;AAWrE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAyB,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAE5E,OAAO,KAAK,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAsCnG;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,cAAc,EAAE,MAAM,wBAAwB,CAAC;IAC/C,+EAA+E;IAC/E,aAAa,EAAE,MAAM,MAAM,EAAE,CAAC;IAC9B,oEAAoE;IACpE,SAAS,EAAE,MAAM,QAAQ,GAAG,SAAS,CAAC;IACtC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,kBAAkB,GAAG,SAAS,CAAC;IAC/C;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,eAAe,EAAE,CAAC;CAC/C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB;IA0Bf,OAAO,CAAC,IAAI;IAzBxB,OAAO,CAAC,UAAU,CAAC,CAAgC;IACnD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,kBAAkB,CAAK;IAC/B;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAA+B;IAC7C;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB,CAAmC;IAC7D;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB,CAAS;gBAEpB,IAAI,EAAE,oBAAoB;IAE9C;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,oBAAoB,GAAG,IAAI;IAI5C,0FAA0F;IAC1F,IAAI,iBAAiB,IAAI,kBAAkB,GAAG,IAAI,CAEjD;IAED,uFAAuF;IACvF,IAAI,sBAAsB,IAAI,OAAO,CAEpC;IAED,OAAO,KAAK,QAAQ,GAEnB;IAED,OAAO,CAAC,OAAO;IAIf;;;;;OAKG;IACH,YAAY,IAAI,IAAI;IAapB;;;;;OAKG;IACH,iBAAiB,IAAI,IAAI;IAOzB;;;;OAIG;YACW,cAAc;IAsJ5B;;;;;;OAMG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAoK9B;;;;;;;OAOG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBvC;;;;OAIG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAatC;;;;;OAKG;IACH,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAchD;;;;;;OAMG;IACH,mBAAmB,IAAI,IAAI;IAiB3B;;;;;;OAMG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvC
|
|
1
|
+
{"version":3,"file":"session-persister.d.ts","sourceRoot":"","sources":["../../../../src/state/persistence/session-persister.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sCAAsC,CAAC;AAWrE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAyB,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAE5E,OAAO,KAAK,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAsCnG;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,cAAc,EAAE,MAAM,wBAAwB,CAAC;IAC/C,+EAA+E;IAC/E,aAAa,EAAE,MAAM,MAAM,EAAE,CAAC;IAC9B,oEAAoE;IACpE,SAAS,EAAE,MAAM,QAAQ,GAAG,SAAS,CAAC;IACtC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,kBAAkB,GAAG,SAAS,CAAC;IAC/C;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,eAAe,EAAE,CAAC;CAC/C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB;IA0Bf,OAAO,CAAC,IAAI;IAzBxB,OAAO,CAAC,UAAU,CAAC,CAAgC;IACnD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,iBAAiB,CAAC,CAAgC;IAC1D,OAAO,CAAC,kBAAkB,CAAK;IAC/B;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAA+B;IAC7C;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB,CAAmC;IAC7D;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB,CAAS;gBAEpB,IAAI,EAAE,oBAAoB;IAE9C;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,oBAAoB,GAAG,IAAI;IAI5C,0FAA0F;IAC1F,IAAI,iBAAiB,IAAI,kBAAkB,GAAG,IAAI,CAEjD;IAED,uFAAuF;IACvF,IAAI,sBAAsB,IAAI,OAAO,CAEpC;IAED,OAAO,KAAK,QAAQ,GAEnB;IAED,OAAO,CAAC,OAAO;IAIf;;;;;OAKG;IACH,YAAY,IAAI,IAAI;IAapB;;;;;OAKG;IACH,iBAAiB,IAAI,IAAI;IAOzB;;;;OAIG;YACW,cAAc;IAsJ5B;;;;;;OAMG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAoK9B;;;;;;;OAOG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBvC;;;;OAIG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAatC;;;;;OAKG;IACH,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAchD;;;;;;OAMG;IACH,mBAAmB,IAAI,IAAI;IAiB3B;;;;;;OAMG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAYvC;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,uBAAuB;IA+C/B,sEAAsE;IACtE,OAAO,IAAI,IAAI;CAMhB"}
|
|
@@ -29,6 +29,26 @@ export declare function totalTokens(usage: AggregateUsage): number;
|
|
|
29
29
|
* @beta
|
|
30
30
|
*/
|
|
31
31
|
export declare function addUsage(a: AggregateUsage, b: AggregateUsage): AggregateUsage;
|
|
32
|
+
/**
|
|
33
|
+
* The four buckets plus USD for ONE message — the per-message fields projected into
|
|
34
|
+
* the disjoint, addable shape, or `undefined` when the message reports no usage at
|
|
35
|
+
* all (a user turn, a display-only reasoning/narration split, a provider that
|
|
36
|
+
* reports none).
|
|
37
|
+
*
|
|
38
|
+
* Shallow by design: it does NOT recurse into `toolCalls[].subAgentTrace` and does
|
|
39
|
+
* NOT count `compaction.rolledUpUsage`, so it answers "what did this one request
|
|
40
|
+
* cost" rather than "what did this message and everything under it cost". Use
|
|
41
|
+
* {@link sumUsage} for the latter — passing a whole transcript through this one
|
|
42
|
+
* message at a time would silently drop both.
|
|
43
|
+
*
|
|
44
|
+
* `costUsd` folds in `externalCostUsd` (non-LLM spend a widget reported), matching
|
|
45
|
+
* `sumUsage`. It is `0` when the provider priced nothing, which is NOT a claim that
|
|
46
|
+
* the call was free — see {@link AggregateUsage}; `UsageRow.costUsd` is the
|
|
47
|
+
* shape that keeps unpriced distinguishable, and is the right tool for a ledger.
|
|
48
|
+
*
|
|
49
|
+
* @beta
|
|
50
|
+
*/
|
|
51
|
+
export declare function messageUsage(m: ChatMessage): AggregateUsage | undefined;
|
|
32
52
|
/**
|
|
33
53
|
* Sum cost and per-bucket token usage across a message list.
|
|
34
54
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sum-usage.d.ts","sourceRoot":"","sources":["../../../src/utils/sum-usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE9E;;;;;GAKG;AACH,wBAAgB,UAAU,IAAI,cAAc,CAQ3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM,CAIzD;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,GAAG,cAAc,CAQ7E;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,GAAG,cAAc,CAIzE"}
|
|
1
|
+
{"version":3,"file":"sum-usage.d.ts","sourceRoot":"","sources":["../../../src/utils/sum-usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE9E;;;;;GAKG;AACH,wBAAgB,UAAU,IAAI,cAAc,CAQ3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM,CAIzD;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,GAAG,cAAc,CAQ7E;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,WAAW,GAAG,cAAc,GAAG,SAAS,CAwBvE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,GAAG,cAAc,CAIzE"}
|
|
@@ -62,7 +62,13 @@ vendorOfModel, } from '@genesislcap/foundation-ai';
|
|
|
62
62
|
// `addUsage`/`emptyUsage` are the composition primitives for a conversation that CONTINUES across
|
|
63
63
|
// rounds — see the guidance on `sumUsage` itself. `totalTokens` exists because the four buckets of
|
|
64
64
|
// an `AggregateUsage` are disjoint and safe to add, which the per-message fields are NOT.
|
|
65
|
-
|
|
65
|
+
//
|
|
66
|
+
// `messageUsage` is the single-message projection of the same arithmetic — what ONE request
|
|
67
|
+
// cost, in the same four-bucket shape. It is what prices a turn snapshot (see
|
|
68
|
+
// `TurnSnapshot.usage`), and the reason a consumer needs it separately is that the response
|
|
69
|
+
// to a *failed* attempt never reaches history, so `sumUsage` over the transcript cannot see
|
|
70
|
+
// that spend at all.
|
|
71
|
+
export { addUsage, emptyUsage, messageUsage, sumUsage, totalTokens } from './utils/sum-usage';
|
|
66
72
|
// Per-call projection, for the cases an aggregate cannot serve: usage rows for
|
|
67
73
|
// per-project attribution, a cost dashboard, or auditing which turns came back
|
|
68
74
|
// unpriced. `usageRows` is guaranteed to reconcile with `sumUsage` — including spend a
|
|
@@ -80,4 +86,8 @@ export { usageRows } from './utils/usage-rows';
|
|
|
80
86
|
// All three modules are DOM-clean, so the node entry stays Node-loadable.
|
|
81
87
|
export { clearSession, getMetaEvents } from './state/debug-event-log';
|
|
82
88
|
export { buildTimelineEntries } from './state/persistence/build-timeline-entries';
|
|
83
|
-
|
|
89
|
+
// `withFreshMetaSnapshot` matters to a headless consumer that stitches a STORED stream: the
|
|
90
|
+
// newest stored `meta-snapshot` is frozen at the last config-signature change, so its context
|
|
91
|
+
// and cost figures are as old as that. Pass a fresh snapshot of your own through this before
|
|
92
|
+
// `assembleDebugLog`, or state in your own output that the totals are historical.
|
|
93
|
+
export { assembleDebugLog, withFreshMetaSnapshot } from './state/persistence/diagnostics';
|
|
@@ -7,7 +7,7 @@ import { createInteractionContext, } from '../../state/interaction-context';
|
|
|
7
7
|
import { applyCondensation } from '../../utils/condense-history';
|
|
8
8
|
import { applyHistoryCap, buildCompactionSummaryPrompt, findCompactionCut, normalizeForProvider, } from '../../utils/history-transform';
|
|
9
9
|
import { logger } from '../../utils/logger';
|
|
10
|
-
import { sumUsage } from '../../utils/sum-usage';
|
|
10
|
+
import { messageUsage, sumUsage } from '../../utils/sum-usage';
|
|
11
11
|
import { TOOL_FOLD_SYMBOL } from '../../utils/tool-fold';
|
|
12
12
|
/**
|
|
13
13
|
* Lift the reportable facts off a {@link BudgetExhaustedError}, or `undefined`
|
|
@@ -683,7 +683,14 @@ export class ChatDriver extends EventTarget {
|
|
|
683
683
|
if (resolvedName !== this.lastDispatchedProviderName) {
|
|
684
684
|
this.lastDispatchedProviderName = resolvedName;
|
|
685
685
|
recordMetaEvent(this.sessionKey, 'provider.selected', {
|
|
686
|
+
// `provider` is the registry SLOT (a tier name like 'high'), kept under that key
|
|
687
|
+
// for compatibility; `model` and `vendor` are what it resolved to. Recording all
|
|
688
|
+
// three is the difference between "the agent switched to its high tier" and
|
|
689
|
+
// knowing which model that actually was — a tier can be repointed mid-session,
|
|
690
|
+
// and a slot name alone cannot distinguish anthropic from gemini.
|
|
686
691
|
provider: resolvedName,
|
|
692
|
+
model: status.model,
|
|
693
|
+
vendor: status.provider,
|
|
687
694
|
agent: this.activeAgentName,
|
|
688
695
|
});
|
|
689
696
|
this.dispatchEvent(new CustomEvent('provider-changed', { detail: { name: resolvedName } }));
|
|
@@ -841,6 +848,10 @@ export class ChatDriver extends EventTarget {
|
|
|
841
848
|
* Push one snapshot to the ring buffer. Called inside `runToolLoop` just
|
|
842
849
|
* before each LLM call — that's the latest point where the prompt, tool
|
|
843
850
|
* surface, and agent state line up with what the model is about to see.
|
|
851
|
+
*
|
|
852
|
+
* Returns the pushed object so the caller can back-fill what only the response
|
|
853
|
+
* knows (`usage`). Mutating it after the fact is safe whether or not the ring
|
|
854
|
+
* buffer has since evicted it — an evicted snapshot is simply no longer exported.
|
|
844
855
|
*/
|
|
845
856
|
recordTurnSnapshot(resolvedSystemPrompt, temperature, toolChoice, tailContext) {
|
|
846
857
|
let agentSnapshot;
|
|
@@ -857,7 +868,7 @@ export class ChatDriver extends EventTarget {
|
|
|
857
868
|
}
|
|
858
869
|
const turnIndex = String(this.globalTurnIndex);
|
|
859
870
|
this.globalTurnIndex += 1;
|
|
860
|
-
|
|
871
|
+
const snapshot = {
|
|
861
872
|
turnIndex,
|
|
862
873
|
timestamp: new Date().toISOString(),
|
|
863
874
|
agentName: this.activeAgentName,
|
|
@@ -868,10 +879,12 @@ export class ChatDriver extends EventTarget {
|
|
|
868
879
|
temperature,
|
|
869
880
|
toolChoice,
|
|
870
881
|
agentSnapshot,
|
|
871
|
-
}
|
|
882
|
+
};
|
|
883
|
+
this.turnSnapshots.push(snapshot);
|
|
872
884
|
if (this.turnSnapshots.length > this.maxTurnSnapshots) {
|
|
873
885
|
this.turnSnapshots.shift();
|
|
874
886
|
}
|
|
887
|
+
return snapshot;
|
|
875
888
|
}
|
|
876
889
|
/**
|
|
877
890
|
* Optional transform applied to conversation history immediately before each LLM request.
|
|
@@ -2134,7 +2147,7 @@ export class ChatDriver extends EventTarget {
|
|
|
2134
2147
|
// on a free-text answer; top-level agents stay 'auto'. (Transports no-op a
|
|
2135
2148
|
// force when no tools are advertised.)
|
|
2136
2149
|
const effectiveToolChoice = resolvedToolChoice !== null && resolvedToolChoice !== void 0 ? resolvedToolChoice : (this.isSubAgent ? 'required' : undefined);
|
|
2137
|
-
this.recordTurnSnapshot(systemPrompt, resolvedTemperature, effectiveToolChoice, tailContext);
|
|
2150
|
+
const turnSnapshot = this.recordTurnSnapshot(systemPrompt, resolvedTemperature, effectiveToolChoice, tailContext);
|
|
2138
2151
|
// Capture the pending user input, then clear the slots BEFORE the chat
|
|
2139
2152
|
// call. `sendMessage` already appended the user message to `this.history`,
|
|
2140
2153
|
// so on retries (empty / malformed) we must rely on history alone —
|
|
@@ -2180,6 +2193,20 @@ export class ChatDriver extends EventTarget {
|
|
|
2180
2193
|
// here and cached for the agent's lifetime.
|
|
2181
2194
|
// oxlint-disable-next-line no-await-in-loop
|
|
2182
2195
|
const activeProvider = yield this.resolveProviderForTurn(promptCtx);
|
|
2196
|
+
// Attribute the turn to the tier/model it resolved. Stamped HERE, not inside
|
|
2197
|
+
// `recordTurnSnapshot`: the snapshot is taken before this line runs, so reading
|
|
2198
|
+
// `lastResolved*` there yields the PREVIOUS call's model — wrong on precisely the
|
|
2199
|
+
// turn where an agent's per-state `provider` selector switches tier, which is the
|
|
2200
|
+
// turn a reader is looking for. `model` is refined to the serving model once the
|
|
2201
|
+
// response lands (see below); until then — and on a call that throws — it is the
|
|
2202
|
+
// model we ASKED for, which is the only thing knowable at that point.
|
|
2203
|
+
if (this.lastResolvedProviderName !== undefined) {
|
|
2204
|
+
turnSnapshot.providerName = this.lastResolvedProviderName;
|
|
2205
|
+
}
|
|
2206
|
+
if (this.lastResolvedProvider !== undefined)
|
|
2207
|
+
turnSnapshot.provider = this.lastResolvedProvider;
|
|
2208
|
+
if (this.lastResolvedModel !== undefined)
|
|
2209
|
+
turnSnapshot.model = this.lastResolvedModel;
|
|
2183
2210
|
let response;
|
|
2184
2211
|
try {
|
|
2185
2212
|
// oxlint-disable-next-line no-await-in-loop
|
|
@@ -2359,6 +2386,19 @@ export class ChatDriver extends EventTarget {
|
|
|
2359
2386
|
if (this.lastResolvedProviderName !== undefined) {
|
|
2360
2387
|
response.providerName = this.lastResolvedProviderName;
|
|
2361
2388
|
}
|
|
2389
|
+
// Back-fill what this call cost onto the snapshot taken just before it, so the
|
|
2390
|
+
// exported debug log prices each model call next to the prompt/tools/state that
|
|
2391
|
+
// produced it (GENC-1480 follow-up). Stamped BEFORE the empty-response branch
|
|
2392
|
+
// below deliberately: a blank or refused response is billed and then thrown away,
|
|
2393
|
+
// so the snapshot is the only place that spend is ever recorded.
|
|
2394
|
+
turnSnapshot.usage = messageUsage(response);
|
|
2395
|
+
// Take the SERVING model over the requested one, now that it is known. `response.model`
|
|
2396
|
+
// was just filled from `lastResolvedModel` if the transport left it unset, so this is
|
|
2397
|
+
// the same rule the message gets — which is the point: a turn and the message it
|
|
2398
|
+
// produced must never disagree about which model ran, including when a server-side
|
|
2399
|
+
// fallback chain answered on a different model than the one we asked for.
|
|
2400
|
+
if (response.model !== undefined)
|
|
2401
|
+
turnSnapshot.model = response.model;
|
|
2362
2402
|
const isThinkingStep = response.content && ((_f = response.toolCalls) === null || _f === void 0 ? void 0 : _f.length);
|
|
2363
2403
|
const isEmptyResponse = !((_g = response.content) === null || _g === void 0 ? void 0 : _g.trim()) && !((_h = response.toolCalls) === null || _h === void 0 ? void 0 : _h.length);
|
|
2364
2404
|
// A pre-output refusal (safety-classifier decline, e.g. Fable 5 `stop_reason: 'refusal'`)
|
|
@@ -2405,8 +2445,23 @@ export class ChatDriver extends EventTarget {
|
|
|
2405
2445
|
// `sumCosts`/`sumTokens` don't double-count and `contextTokens` reads it. Reasoning/narration are
|
|
2406
2446
|
// display-only (usage undefined) and are skipped when building the provider request. `model` /
|
|
2407
2447
|
// `provider` / `providerName` stay on every split message so each is still attributed.
|
|
2448
|
+
//
|
|
2449
|
+
// EVERY usage field has to be cleared here, not just the three the invariant was
|
|
2450
|
+
// originally written against — keep this list in step with the usage fields on
|
|
2451
|
+
// `ChatMessage`. The cache buckets arrived later (GENC-1475) and were left riding
|
|
2452
|
+
// along on the copies, so a response carrying reasoning AND narration counted its
|
|
2453
|
+
// cache read/write volume three times in `sumUsage`/`usageRows` — invisible in the
|
|
2454
|
+
// cost total (which comes from `cost`) but wrong in every bucket display and in the
|
|
2455
|
+
// exported log.
|
|
2408
2456
|
const { reasoning } = response, rest = __rest(response, ["reasoning"]);
|
|
2409
|
-
const displayOnly = {
|
|
2457
|
+
const displayOnly = {
|
|
2458
|
+
cost: undefined,
|
|
2459
|
+
externalCostUsd: undefined,
|
|
2460
|
+
inputTokens: undefined,
|
|
2461
|
+
outputTokens: undefined,
|
|
2462
|
+
cacheReadTokens: undefined,
|
|
2463
|
+
cacheWriteTokens: undefined,
|
|
2464
|
+
};
|
|
2410
2465
|
if (reasoning) {
|
|
2411
2466
|
this.appendToHistory(Object.assign(Object.assign(Object.assign({}, rest), displayOnly), { content: reasoning, toolCalls: undefined, category: 'reasoning' }));
|
|
2412
2467
|
}
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { __awaiter } from "tslib";
|
|
2
|
+
import { assert, createLogicSuite } from '@genesislcap/foundation-testing';
|
|
3
|
+
import { clearMetaEventRegistry, getMetaEvents } from '../../state/debug-event-log';
|
|
4
|
+
import { messageUsage, sumUsage } from '../../utils/sum-usage';
|
|
5
|
+
// Side-effect import — MUST come before `./chat-driver` so the driver subclasses
|
|
6
|
+
// jsdom's EventTarget rather than Node's native one. Mirrors chat-driver.test.ts.
|
|
7
|
+
import './align-event-globals';
|
|
8
|
+
import { ChatDriver } from './chat-driver';
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// Per-call usage on the turn snapshots — the four token buckets plus USD priced
|
|
11
|
+
// onto each `kind: 'turn'` entry of the exported debug log.
|
|
12
|
+
//
|
|
13
|
+
// The snapshot is captured BEFORE the model call (that is where the prompt, tool
|
|
14
|
+
// surface and agent state line up with what the model saw), so usage has to be
|
|
15
|
+
// back-filled when the response lands. Two things follow, and both are asserted
|
|
16
|
+
// here: a completed call's turn agrees with the message it produced, and a call
|
|
17
|
+
// that produced NO message (a blank response, retried) is still priced — that
|
|
18
|
+
// spend appears nowhere in the transcript.
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
const makeRegistry = (provider) => ({
|
|
21
|
+
get: () => provider,
|
|
22
|
+
default: () => provider,
|
|
23
|
+
defaultName: () => 'test',
|
|
24
|
+
names: () => ['test'],
|
|
25
|
+
getStatus: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
|
|
26
|
+
listStatuses: () => __awaiter(void 0, void 0, void 0, function* () { return []; }),
|
|
27
|
+
});
|
|
28
|
+
/** Answers from a queue, then ends the turn with a plain unpriced reply. */
|
|
29
|
+
const scriptedProvider = (responses) => {
|
|
30
|
+
const queue = [...responses];
|
|
31
|
+
return {
|
|
32
|
+
chat: () => __awaiter(void 0, void 0, void 0, function* () { var _a; return (_a = queue.shift()) !== null && _a !== void 0 ? _a : { role: 'assistant', content: 'done' }; }),
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
const agent = (overrides) => (Object.assign({ description: 'test agent' }, overrides));
|
|
36
|
+
const makeDriver = (config, provider) => {
|
|
37
|
+
const driver = new ChatDriver(makeRegistry(provider), {
|
|
38
|
+
maxToolIterations: 50,
|
|
39
|
+
maxFoldOperations: 5,
|
|
40
|
+
sessionKey: '',
|
|
41
|
+
});
|
|
42
|
+
driver.applyAgent(config);
|
|
43
|
+
return driver;
|
|
44
|
+
};
|
|
45
|
+
const def = (name) => ({
|
|
46
|
+
name,
|
|
47
|
+
description: `${name} tool`,
|
|
48
|
+
parameters: { type: 'object', properties: {} },
|
|
49
|
+
});
|
|
50
|
+
/** One priced response: a 1000-token prompt that was mostly a cache hit. */
|
|
51
|
+
const priced = (over = {}) => (Object.assign({ role: 'assistant', content: 'answer', cost: 0.1, inputTokens: 1000, cacheReadTokens: 900, cacheWriteTokens: 50, outputTokens: 20 }, over));
|
|
52
|
+
const suite = createLogicSuite('ChatDriver per-turn usage');
|
|
53
|
+
suite('prices each turn snapshot with the four buckets and USD of its own call', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
54
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([priced()]));
|
|
55
|
+
yield driver.sendMessage('go');
|
|
56
|
+
const [turn] = driver.getTurnSnapshots();
|
|
57
|
+
assert.equal(turn.usage, {
|
|
58
|
+
costUsd: 0.1,
|
|
59
|
+
// The prompt split into disjoint buckets: 1000 total, 900 read + 50 written,
|
|
60
|
+
// so 50 uncached. A reader of the log gets this without re-deriving it.
|
|
61
|
+
uncachedInputTokens: 50,
|
|
62
|
+
cacheReadTokens: 900,
|
|
63
|
+
cacheWriteTokens: 50,
|
|
64
|
+
outputTokens: 20,
|
|
65
|
+
}, 'the turn carries its own call’s usage');
|
|
66
|
+
// The turn and the message it produced are the SAME charge, so they must agree
|
|
67
|
+
// exactly — a log whose two views of one call disagree is worse than one view.
|
|
68
|
+
assert.equal(turn.usage, sumUsage(driver.getHistory()), 'turn agrees with the transcript total');
|
|
69
|
+
}));
|
|
70
|
+
suite('prices every model call in a tool loop separately', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
71
|
+
const driver = makeDriver(agent({
|
|
72
|
+
name: 'a',
|
|
73
|
+
toolDefinitions: [def('work')],
|
|
74
|
+
toolHandlers: { work: () => __awaiter(void 0, void 0, void 0, function* () { return 'worked'; }) },
|
|
75
|
+
}), scriptedProvider([
|
|
76
|
+
priced({ content: '', cost: 0.02, toolCalls: [{ id: 'w1', name: 'work', args: {} }] }),
|
|
77
|
+
priced({ cost: 0.03 }),
|
|
78
|
+
]));
|
|
79
|
+
yield driver.sendMessage('go');
|
|
80
|
+
const snapshots = driver.getTurnSnapshots();
|
|
81
|
+
assert.is(snapshots.length, 2, 'one snapshot per model call, not per user turn');
|
|
82
|
+
assert.equal(snapshots.map((s) => { var _a; return (_a = s.usage) === null || _a === void 0 ? void 0 : _a.costUsd; }), [0.02, 0.03]);
|
|
83
|
+
}));
|
|
84
|
+
suite('prices a blank response that produced no message', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
85
|
+
var _a;
|
|
86
|
+
// A blank turn is billed and then discarded before the retry, so the transcript can
|
|
87
|
+
// never account for it. The snapshot is the only record — which is the whole reason
|
|
88
|
+
// usage is stamped before the empty-response branch runs.
|
|
89
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([
|
|
90
|
+
priced({ content: '', cost: 0.07, outputTokens: 0 }),
|
|
91
|
+
priced({ cost: 0.03 }),
|
|
92
|
+
]));
|
|
93
|
+
yield driver.sendMessage('go');
|
|
94
|
+
const snapshots = driver.getTurnSnapshots();
|
|
95
|
+
assert.is(snapshots.length, 2, 'the retried attempt has its own snapshot');
|
|
96
|
+
assert.is((_a = snapshots[0].usage) === null || _a === void 0 ? void 0 : _a.costUsd, 0.07, 'the discarded attempt is still priced');
|
|
97
|
+
assert.is(sumUsage(driver.getHistory()).costUsd, 0.03, 'and is genuinely absent from the transcript — that is the gap the snapshot fills');
|
|
98
|
+
}));
|
|
99
|
+
suite('leaves usage undefined when the provider reports none', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
100
|
+
// Absent, not zeroed: "this provider reports no usage" (e.g. Chrome's built-in model)
|
|
101
|
+
// must not read as a free call.
|
|
102
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([]));
|
|
103
|
+
yield driver.sendMessage('go');
|
|
104
|
+
assert.is(driver.getTurnSnapshots()[0].usage, undefined);
|
|
105
|
+
}));
|
|
106
|
+
suite.run();
|
|
107
|
+
// ---------------------------------------------------------------------------
|
|
108
|
+
// The reasoning/narration/answer split must not multiply the cache buckets.
|
|
109
|
+
//
|
|
110
|
+
// One model response becomes up to three messages; only the last carries usage.
|
|
111
|
+
// The clearing list was written when usage meant cost + input + output, so when the
|
|
112
|
+
// cache buckets arrived they kept riding along on the display-only copies — and every
|
|
113
|
+
// bucket aggregate (session totals, usage rows, the exported log) counted a split
|
|
114
|
+
// turn's cache volume two or three times. The cost total hid it, since that comes
|
|
115
|
+
// from `cost`, which WAS cleared.
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
const split = createLogicSuite('ChatDriver response split usage');
|
|
118
|
+
split('counts a reasoning + answer split once, not twice', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
119
|
+
const response = priced({ reasoning: 'thinking…' });
|
|
120
|
+
const driver = makeDriver(agent({ name: 'a' }), scriptedProvider([response]));
|
|
121
|
+
yield driver.sendMessage('go');
|
|
122
|
+
const history = driver.getHistory();
|
|
123
|
+
assert.is(history.filter((m) => m.role === 'assistant').length, 2, 'the response did split into a reasoning message and an answer');
|
|
124
|
+
assert.equal(sumUsage(history), messageUsage(response), 'the split totals exactly one call’s usage');
|
|
125
|
+
const reasoning = history.find((m) => m.category === 'reasoning');
|
|
126
|
+
assert.is(reasoning.cacheReadTokens, undefined, 'the display-only copy carries no cache volume');
|
|
127
|
+
assert.is(reasoning.cacheWriteTokens, undefined);
|
|
128
|
+
}));
|
|
129
|
+
split('counts a reasoning + narration + answer split once', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
130
|
+
// The three-way case: content alongside a tool call is interstitial narration, so this
|
|
131
|
+
// response produces reasoning + narration + the tool-call message. Worst case for the
|
|
132
|
+
// old behaviour — cache volume counted three times.
|
|
133
|
+
const first = priced({
|
|
134
|
+
content: 'let me look that up',
|
|
135
|
+
reasoning: 'thinking…',
|
|
136
|
+
cost: 0.02,
|
|
137
|
+
toolCalls: [{ id: 'w1', name: 'work', args: {} }],
|
|
138
|
+
});
|
|
139
|
+
const driver = makeDriver(agent({
|
|
140
|
+
name: 'a',
|
|
141
|
+
toolDefinitions: [def('work')],
|
|
142
|
+
toolHandlers: { work: () => __awaiter(void 0, void 0, void 0, function* () { return 'worked'; }) },
|
|
143
|
+
}), scriptedProvider([first, priced({ cost: 0.03 })]));
|
|
144
|
+
yield driver.sendMessage('go');
|
|
145
|
+
const total = sumUsage(driver.getHistory());
|
|
146
|
+
assert.is(total.costUsd, 0.05, 'two calls, each counted once');
|
|
147
|
+
assert.is(total.cacheReadTokens, 1800, '900 per call — not 2700 with the narration copies');
|
|
148
|
+
assert.is(total.cacheWriteTokens, 100);
|
|
149
|
+
assert.is(total.uncachedInputTokens, 100);
|
|
150
|
+
}));
|
|
151
|
+
split.run();
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
// Per-turn model attribution across a tier switch.
|
|
154
|
+
//
|
|
155
|
+
// The shape this exists for is an agent whose `provider` selector varies by state —
|
|
156
|
+
// e.g. the showcase trade-operations agent: `({state}) => isPlanning(state) ? High : Low`,
|
|
157
|
+
// planning on sonnet and executing on haiku. Each call has to name the model that ran
|
|
158
|
+
// it, on the turn itself: joining a turn to the message after it works for a normal
|
|
159
|
+
// step but not for a call that produced no message, and `provider.selected` fires only
|
|
160
|
+
// when the slot CHANGES, so the nearest event can be many turns back.
|
|
161
|
+
//
|
|
162
|
+
// The ordering trap: the snapshot is recorded BEFORE the provider is resolved for that
|
|
163
|
+
// call, so stamping `lastResolvedModel` at creation time yields the previous call's
|
|
164
|
+
// model — wrong on exactly the turn the tier changes.
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
const tiers = createLogicSuite('ChatDriver per-turn model attribution');
|
|
167
|
+
/** A tiered registry: named slots, each reporting its own model via `getStatus`. */
|
|
168
|
+
const tieredRegistry = (slots, defaultName) => ({
|
|
169
|
+
get: (name) => slots[name],
|
|
170
|
+
default: () => slots[defaultName],
|
|
171
|
+
defaultName: () => defaultName,
|
|
172
|
+
names: () => Object.keys(slots),
|
|
173
|
+
getStatus: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
|
|
174
|
+
listStatuses: () => __awaiter(void 0, void 0, void 0, function* () { return []; }),
|
|
175
|
+
});
|
|
176
|
+
const tierProvider = (model, responses) => {
|
|
177
|
+
const queue = [...responses];
|
|
178
|
+
return {
|
|
179
|
+
getStatus: () => __awaiter(void 0, void 0, void 0, function* () { return ({ model, provider: 'anthropic' }); }),
|
|
180
|
+
chat: () => __awaiter(void 0, void 0, void 0, function* () { var _a; return (_a = queue.shift()) !== null && _a !== void 0 ? _a : { role: 'assistant', content: 'done' }; }),
|
|
181
|
+
};
|
|
182
|
+
};
|
|
183
|
+
tiers('attributes each step to the tier that ran it when the agent switches mid-flow', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
184
|
+
// Responses deliberately carry NO `model` of their own — the common case for a
|
|
185
|
+
// transport that leaves attribution to the driver, and the one where a stale
|
|
186
|
+
// `lastResolved*` read would go unnoticed.
|
|
187
|
+
const slots = {
|
|
188
|
+
high: tierProvider('claude-sonnet-4-6', [
|
|
189
|
+
priced({
|
|
190
|
+
content: '',
|
|
191
|
+
cost: 0.05,
|
|
192
|
+
toolCalls: [{ id: 'p1', name: 'finish_planning', args: {} }],
|
|
193
|
+
}),
|
|
194
|
+
]),
|
|
195
|
+
low: tierProvider('claude-haiku-4-5-20251001', [priced({ content: 'Booked.', cost: 0.001 })]),
|
|
196
|
+
};
|
|
197
|
+
// The "state" the selector reads, advanced by the tool — as a flow agent's machine does.
|
|
198
|
+
let planning = true;
|
|
199
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'low'), {
|
|
200
|
+
maxToolIterations: 20,
|
|
201
|
+
maxFoldOperations: 5,
|
|
202
|
+
sessionKey: 'tiers',
|
|
203
|
+
});
|
|
204
|
+
driver.applyAgent(agent({
|
|
205
|
+
name: 'Trade Operations',
|
|
206
|
+
provider: () => (planning ? 'high' : 'low'),
|
|
207
|
+
toolDefinitions: [def('finish_planning')],
|
|
208
|
+
toolHandlers: {
|
|
209
|
+
finish_planning: () => __awaiter(void 0, void 0, void 0, function* () {
|
|
210
|
+
planning = false;
|
|
211
|
+
return 'planned';
|
|
212
|
+
}),
|
|
213
|
+
},
|
|
214
|
+
}));
|
|
215
|
+
yield driver.sendMessage('book me a trade');
|
|
216
|
+
const snapshots = driver.getTurnSnapshots();
|
|
217
|
+
assert.equal(snapshots.map((s) => { var _a; return [s.providerName, s.model, (_a = s.usage) === null || _a === void 0 ? void 0 : _a.costUsd]; }), [
|
|
218
|
+
['high', 'claude-sonnet-4-6', 0.05],
|
|
219
|
+
['low', 'claude-haiku-4-5-20251001', 0.001],
|
|
220
|
+
], 'the planning call is attributed to the high tier and the execution call to the low one');
|
|
221
|
+
assert.equal(snapshots.map((s) => s.provider), ['anthropic', 'anthropic'], 'the vendor behind each slot is recorded too');
|
|
222
|
+
// A turn and the message it produced must never disagree about which model ran.
|
|
223
|
+
const assistantModels = driver
|
|
224
|
+
.getHistory()
|
|
225
|
+
.filter((m) => m.role === 'assistant' && m.cost != null)
|
|
226
|
+
.map((m) => m.model);
|
|
227
|
+
assert.equal(assistantModels, snapshots.map((s) => s.model));
|
|
228
|
+
}));
|
|
229
|
+
tiers('records the SERVING model when a fallback answers on another model', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
230
|
+
// A server-side fallback chain answers on a different model than the one requested.
|
|
231
|
+
// The transport stamps the real one; the turn must follow it rather than relabel the
|
|
232
|
+
// call as the tier's configured model — that would misattribute the spend.
|
|
233
|
+
const slots = {
|
|
234
|
+
high: tierProvider('claude-fable-5', [
|
|
235
|
+
priced({ content: 'answered by the fallback', model: 'claude-opus-4-8' }),
|
|
236
|
+
]),
|
|
237
|
+
};
|
|
238
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
|
|
239
|
+
maxToolIterations: 20,
|
|
240
|
+
maxFoldOperations: 5,
|
|
241
|
+
sessionKey: 'fallback',
|
|
242
|
+
});
|
|
243
|
+
driver.applyAgent(agent({ name: 'a', provider: 'high' }));
|
|
244
|
+
yield driver.sendMessage('go');
|
|
245
|
+
const [turn] = driver.getTurnSnapshots();
|
|
246
|
+
assert.is(turn.model, 'claude-opus-4-8', 'the model that answered, not the one asked for');
|
|
247
|
+
assert.is(turn.providerName, 'high', 'the slot asked for is still recorded');
|
|
248
|
+
}));
|
|
249
|
+
tiers('names the model and vendor on the provider.selected event, not just the slot', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
250
|
+
var _a, _b, _c;
|
|
251
|
+
clearMetaEventRegistry();
|
|
252
|
+
const slots = {
|
|
253
|
+
high: tierProvider('claude-sonnet-4-6', [priced({ content: 'hi' })]),
|
|
254
|
+
};
|
|
255
|
+
const driver = new ChatDriver(tieredRegistry(slots, 'high'), {
|
|
256
|
+
maxToolIterations: 20,
|
|
257
|
+
maxFoldOperations: 5,
|
|
258
|
+
sessionKey: 'selected',
|
|
259
|
+
});
|
|
260
|
+
driver.applyAgent(agent({ name: 'a', provider: 'high' }));
|
|
261
|
+
yield driver.sendMessage('go');
|
|
262
|
+
const selected = getMetaEvents('selected').find((e) => e.type === 'provider.selected');
|
|
263
|
+
assert.ok(selected, 'a provider.selected event is recorded');
|
|
264
|
+
assert.is((_a = selected.detail) === null || _a === void 0 ? void 0 : _a.provider, 'high', 'the registry slot');
|
|
265
|
+
assert.is((_b = selected.detail) === null || _b === void 0 ? void 0 : _b.model, 'claude-sonnet-4-6', 'and the model behind it');
|
|
266
|
+
assert.is((_c = selected.detail) === null || _c === void 0 ? void 0 : _c.vendor, 'anthropic');
|
|
267
|
+
}));
|
|
268
|
+
tiers.run();
|