@outputai/cli 0.4.0 → 0.4.1-dev.56c13a8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/generated/api.d.ts +120 -0
- package/dist/api/generated/api.js +18 -0
- package/dist/assets/docker/docker-compose-dev.yml +1 -1
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/cost_calculator.js +62 -26
- package/dist/services/cost_calculator.spec.js +211 -9
- package/dist/types/cost.d.ts +16 -3
- package/package.json +4 -4
|
@@ -146,6 +146,58 @@ export interface TraceLogLocalResponse {
|
|
|
146
146
|
/** Absolute path to local trace file */
|
|
147
147
|
localPath: string;
|
|
148
148
|
}
|
|
149
|
+
export interface TraceAttributesCostComponent {
|
|
150
|
+
/** Canonical cost event name (e.g. `cost:llm:request`, `cost:http:request`, `other`). */
|
|
151
|
+
name: string;
|
|
152
|
+
/** Summed USD cost for this component bucket. */
|
|
153
|
+
value: number;
|
|
154
|
+
}
|
|
155
|
+
export type TraceAttributesResponseAttributesCost = {
|
|
156
|
+
/** USD; equal to the sum of `components[].value` */
|
|
157
|
+
total: number;
|
|
158
|
+
/** Cost contributions bucketed by canonical event name (llm / http / other). */
|
|
159
|
+
components: TraceAttributesCostComponent[];
|
|
160
|
+
};
|
|
161
|
+
export type TraceAttributesResponseAttributesTokenUsage = {
|
|
162
|
+
inputTokens: number;
|
|
163
|
+
outputTokens: number;
|
|
164
|
+
cachedInputTokens: number;
|
|
165
|
+
totalTokens: number;
|
|
166
|
+
};
|
|
167
|
+
export type TraceAttributesResponseAttributes = {
|
|
168
|
+
cost: TraceAttributesResponseAttributesCost;
|
|
169
|
+
tokenUsage: TraceAttributesResponseAttributesTokenUsage;
|
|
170
|
+
};
|
|
171
|
+
/**
|
|
172
|
+
* Aggregated cost, token usage, and runtime computed by walking the trace tree of a completed workflow run.
|
|
173
|
+
Component breakdowns under `attributes.cost.components` are grouped by the canonical event name that
|
|
174
|
+
emitted them (inferred from node kind: llm/http/other). Values sum to `attributes.cost.total`.
|
|
175
|
+
|
|
176
|
+
*/
|
|
177
|
+
export interface TraceAttributesResponse {
|
|
178
|
+
/** The workflow execution id */
|
|
179
|
+
workflowId: string;
|
|
180
|
+
/** The specific run id this aggregation belongs to */
|
|
181
|
+
runId: string;
|
|
182
|
+
/**
|
|
183
|
+
* ms epoch of the root trace node's start
|
|
184
|
+
* @nullable
|
|
185
|
+
*/
|
|
186
|
+
startTime: number | null;
|
|
187
|
+
/**
|
|
188
|
+
* ms epoch of the root trace node's end
|
|
189
|
+
* @nullable
|
|
190
|
+
*/
|
|
191
|
+
finishTime: number | null;
|
|
192
|
+
/**
|
|
193
|
+
* ms between finishTime and startTime, null if either timestamp is missing
|
|
194
|
+
* @nullable
|
|
195
|
+
*/
|
|
196
|
+
runtime: number | null;
|
|
197
|
+
attributes: TraceAttributesResponseAttributes;
|
|
198
|
+
/** S3 URL of the underlying trace file (same value `/trace-log` returns under `data` for remote traces). */
|
|
199
|
+
traceUrl: string;
|
|
200
|
+
}
|
|
149
201
|
/**
|
|
150
202
|
* Current run status
|
|
151
203
|
*/
|
|
@@ -950,6 +1002,74 @@ export type getWorkflowIdRunsRidTraceLogResponseError = (getWorkflowIdRunsRidTra
|
|
|
950
1002
|
export type getWorkflowIdRunsRidTraceLogResponse = (getWorkflowIdRunsRidTraceLogResponseSuccess | getWorkflowIdRunsRidTraceLogResponseError);
|
|
951
1003
|
export declare const getGetWorkflowIdRunsRidTraceLogUrl: (id: string, rid: string) => string;
|
|
952
1004
|
export declare const getWorkflowIdRunsRidTraceLog: (id: string, rid: string, options?: ApiRequestOptions) => Promise<getWorkflowIdRunsRidTraceLogResponse>;
|
|
1005
|
+
/**
|
|
1006
|
+
* Returns runtime, cost rolled up by event-name bucket, token-usage totals, and the trace S3 URL
|
|
1007
|
+
for the latest run of the given workflow. Completion-only — returns 424 while the workflow is
|
|
1008
|
+
still running, mirroring `/result` and `/trace-log`. To pin a specific run, use
|
|
1009
|
+
`/workflow/{id}/runs/{rid}/trace-attributes`.
|
|
1010
|
+
|
|
1011
|
+
* @summary Get aggregated trace attributes for a completed workflow (latest run)
|
|
1012
|
+
*/
|
|
1013
|
+
export type getWorkflowIdTraceAttributesResponse200 = {
|
|
1014
|
+
data: TraceAttributesResponse;
|
|
1015
|
+
status: 200;
|
|
1016
|
+
};
|
|
1017
|
+
export type getWorkflowIdTraceAttributesResponse404 = {
|
|
1018
|
+
data: NotFoundResponse;
|
|
1019
|
+
status: 404;
|
|
1020
|
+
};
|
|
1021
|
+
export type getWorkflowIdTraceAttributesResponse424 = {
|
|
1022
|
+
data: FailedDependencyResponse;
|
|
1023
|
+
status: 424;
|
|
1024
|
+
};
|
|
1025
|
+
export type getWorkflowIdTraceAttributesResponse500 = {
|
|
1026
|
+
data: InternalServerErrorResponse;
|
|
1027
|
+
status: 500;
|
|
1028
|
+
};
|
|
1029
|
+
export type getWorkflowIdTraceAttributesResponseSuccess = (getWorkflowIdTraceAttributesResponse200) & {
|
|
1030
|
+
headers: Headers;
|
|
1031
|
+
};
|
|
1032
|
+
export type getWorkflowIdTraceAttributesResponseError = (getWorkflowIdTraceAttributesResponse404 | getWorkflowIdTraceAttributesResponse424 | getWorkflowIdTraceAttributesResponse500) & {
|
|
1033
|
+
headers: Headers;
|
|
1034
|
+
};
|
|
1035
|
+
export type getWorkflowIdTraceAttributesResponse = (getWorkflowIdTraceAttributesResponseSuccess | getWorkflowIdTraceAttributesResponseError);
|
|
1036
|
+
export declare const getGetWorkflowIdTraceAttributesUrl: (id: string) => string;
|
|
1037
|
+
export declare const getWorkflowIdTraceAttributes: (id: string, options?: ApiRequestOptions) => Promise<getWorkflowIdTraceAttributesResponse>;
|
|
1038
|
+
/**
|
|
1039
|
+
* Returns runtime, cost rolled up by event-name bucket, token-usage totals, and the trace S3 URL
|
|
1040
|
+
for the pinned workflow run. Completion-only — returns 424 while the run is still in progress.
|
|
1041
|
+
|
|
1042
|
+
* @summary Get aggregated trace attributes for a specific completed workflow run
|
|
1043
|
+
*/
|
|
1044
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse200 = {
|
|
1045
|
+
data: TraceAttributesResponse;
|
|
1046
|
+
status: 200;
|
|
1047
|
+
};
|
|
1048
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse400 = {
|
|
1049
|
+
data: BadRequestResponse;
|
|
1050
|
+
status: 400;
|
|
1051
|
+
};
|
|
1052
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse404 = {
|
|
1053
|
+
data: NotFoundResponse;
|
|
1054
|
+
status: 404;
|
|
1055
|
+
};
|
|
1056
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse424 = {
|
|
1057
|
+
data: FailedDependencyResponse;
|
|
1058
|
+
status: 424;
|
|
1059
|
+
};
|
|
1060
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse500 = {
|
|
1061
|
+
data: InternalServerErrorResponse;
|
|
1062
|
+
status: 500;
|
|
1063
|
+
};
|
|
1064
|
+
export type getWorkflowIdRunsRidTraceAttributesResponseSuccess = (getWorkflowIdRunsRidTraceAttributesResponse200) & {
|
|
1065
|
+
headers: Headers;
|
|
1066
|
+
};
|
|
1067
|
+
export type getWorkflowIdRunsRidTraceAttributesResponseError = (getWorkflowIdRunsRidTraceAttributesResponse400 | getWorkflowIdRunsRidTraceAttributesResponse404 | getWorkflowIdRunsRidTraceAttributesResponse424 | getWorkflowIdRunsRidTraceAttributesResponse500) & {
|
|
1068
|
+
headers: Headers;
|
|
1069
|
+
};
|
|
1070
|
+
export type getWorkflowIdRunsRidTraceAttributesResponse = (getWorkflowIdRunsRidTraceAttributesResponseSuccess | getWorkflowIdRunsRidTraceAttributesResponseError);
|
|
1071
|
+
export declare const getGetWorkflowIdRunsRidTraceAttributesUrl: (id: string, rid: string) => string;
|
|
1072
|
+
export declare const getWorkflowIdRunsRidTraceAttributes: (id: string, rid: string, options?: ApiRequestOptions) => Promise<getWorkflowIdRunsRidTraceAttributesResponse>;
|
|
953
1073
|
/**
|
|
954
1074
|
* Returns decoded Temporal history events with optional payload inclusion. First page includes workflow metadata; subsequent pages return events only.
|
|
955
1075
|
* @summary Get paginated workflow execution history
|
|
@@ -194,6 +194,24 @@ export const getWorkflowIdRunsRidTraceLog = async (id, rid, options) => {
|
|
|
194
194
|
method: 'GET'
|
|
195
195
|
});
|
|
196
196
|
};
|
|
197
|
+
export const getGetWorkflowIdTraceAttributesUrl = (id) => {
|
|
198
|
+
return `/workflow/${id}/trace-attributes`;
|
|
199
|
+
};
|
|
200
|
+
export const getWorkflowIdTraceAttributes = async (id, options) => {
|
|
201
|
+
return customFetchInstance(getGetWorkflowIdTraceAttributesUrl(id), {
|
|
202
|
+
...options,
|
|
203
|
+
method: 'GET'
|
|
204
|
+
});
|
|
205
|
+
};
|
|
206
|
+
export const getGetWorkflowIdRunsRidTraceAttributesUrl = (id, rid) => {
|
|
207
|
+
return `/workflow/${id}/runs/${rid}/trace-attributes`;
|
|
208
|
+
};
|
|
209
|
+
export const getWorkflowIdRunsRidTraceAttributes = async (id, rid, options) => {
|
|
210
|
+
return customFetchInstance(getGetWorkflowIdRunsRidTraceAttributesUrl(id, rid), {
|
|
211
|
+
...options,
|
|
212
|
+
method: 'GET'
|
|
213
|
+
});
|
|
214
|
+
};
|
|
197
215
|
export const getGetWorkflowIdHistoryUrl = (id, params) => {
|
|
198
216
|
const normalizedParams = new URLSearchParams();
|
|
199
217
|
Object.entries(params || {}).forEach(([key, value]) => {
|
|
@@ -5,6 +5,9 @@ const ARRAY_ACCESS_PATTERN = /^(\w+)\[(\d+)\]$/;
|
|
|
5
5
|
function tokenCost(tokens, pricePerMillion) {
|
|
6
6
|
return (tokens / 1_000_000) * pricePerMillion;
|
|
7
7
|
}
|
|
8
|
+
function isFiniteNumber(value) {
|
|
9
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
10
|
+
}
|
|
8
11
|
export function extractValue(obj, path) {
|
|
9
12
|
if (!path || !obj) {
|
|
10
13
|
return obj;
|
|
@@ -69,20 +72,25 @@ function findCalls(node, match, extract, parentStepName = null, seenIds = new Se
|
|
|
69
72
|
}
|
|
70
73
|
return calls;
|
|
71
74
|
}
|
|
75
|
+
function readAttributeCost(node) {
|
|
76
|
+
const total = node.attributes?.cost?.total;
|
|
77
|
+
return isFiniteNumber(total) ? total : undefined;
|
|
78
|
+
}
|
|
72
79
|
export function findLLMCalls(node, parentStepName = null, seenIds = new Set()) {
|
|
73
|
-
return findCalls(node, n => n.kind === 'llm' && !!n.
|
|
80
|
+
return findCalls(node, n => n.kind === 'llm' && !!n.attributes?.token_usage, (n, stepName) => {
|
|
74
81
|
const loadedPrompt = n.input?.loadedPrompt;
|
|
75
|
-
const outputRecord = n.output;
|
|
76
|
-
const inputRecord = n.input;
|
|
82
|
+
const outputRecord = (n.output ?? {});
|
|
83
|
+
const inputRecord = (n.input ?? {});
|
|
77
84
|
const model = loadedPrompt?.config?.model ||
|
|
78
|
-
outputRecord
|
|
79
|
-
inputRecord
|
|
85
|
+
outputRecord.model ||
|
|
86
|
+
inputRecord.model ||
|
|
80
87
|
'unknown';
|
|
81
88
|
return {
|
|
82
89
|
stepName: stepName || n.name || 'unknown',
|
|
83
90
|
llmName: n.name || 'llm',
|
|
84
91
|
model,
|
|
85
|
-
usage: n.
|
|
92
|
+
usage: n.attributes.token_usage,
|
|
93
|
+
attributeCost: readAttributeCost(n)
|
|
86
94
|
};
|
|
87
95
|
}, parentStepName, seenIds);
|
|
88
96
|
}
|
|
@@ -93,7 +101,8 @@ export function findHTTPCalls(node, parentStepName = null, seenIds = new Set())
|
|
|
93
101
|
method: n.input?.method || 'GET',
|
|
94
102
|
input: n.input || {},
|
|
95
103
|
output: n.output || {},
|
|
96
|
-
status: n.output?.status
|
|
104
|
+
status: n.output?.status,
|
|
105
|
+
attributeCost: readAttributeCost(n)
|
|
97
106
|
}), parentStepName, seenIds);
|
|
98
107
|
}
|
|
99
108
|
export function calculateLLMCallCost(usage, modelPricing) {
|
|
@@ -277,11 +286,14 @@ function aggregateLLMCosts(llmCalls, config) {
|
|
|
277
286
|
const totals = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, cost: 0 };
|
|
278
287
|
for (const call of llmCalls) {
|
|
279
288
|
const { pricing, matchedKey } = findModelPricing(call.model, config.models ?? {});
|
|
280
|
-
const { cost, warning } = calculateLLMCallCost(call.usage, pricing);
|
|
289
|
+
const { cost: yamlCost, warning } = calculateLLMCallCost(call.usage, pricing);
|
|
290
|
+
// Prefer the cost emitted by the LLM provider on the trace node; fall back to
|
|
291
|
+
// yaml pricing so unknown-model warnings still surface for breakdown display.
|
|
292
|
+
const cost = call.attributeCost ?? yamlCost;
|
|
281
293
|
const prefixWarning = (pricing && matchedKey !== call.model) ?
|
|
282
294
|
`priced as ${matchedKey}` :
|
|
283
295
|
undefined;
|
|
284
|
-
if (!pricing) {
|
|
296
|
+
if (!pricing && call.attributeCost === undefined) {
|
|
285
297
|
unknownModels.add(call.model);
|
|
286
298
|
}
|
|
287
299
|
results.push({
|
|
@@ -319,27 +331,51 @@ export function calculateCost(trace, config, traceFile = '') {
|
|
|
319
331
|
continue;
|
|
320
332
|
}
|
|
321
333
|
const serviceInfo = identifyService(call, config.services);
|
|
322
|
-
if (
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
const
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
334
|
+
if (serviceInfo) {
|
|
335
|
+
// attributes.cost on the HTTP node is authoritative when present —
|
|
336
|
+
// addRequestCost() writes the real billed amount there. Fall back to
|
|
337
|
+
// the yaml service classifier for legacy callers that don't emit it.
|
|
338
|
+
const result = call.attributeCost !== undefined ?
|
|
339
|
+
{ step: call.stepName, cost: call.attributeCost, usage: '1 request' } :
|
|
340
|
+
(() => {
|
|
341
|
+
if (serviceInfo.config.type === 'response_cost') {
|
|
342
|
+
const hasCostData = extractValue(call, serviceInfo.config.cost_path);
|
|
343
|
+
const isBillableMethod = serviceInfo.config.billable_method &&
|
|
344
|
+
call.method === serviceInfo.config.billable_method;
|
|
345
|
+
if (!hasCostData && !isBillableMethod) {
|
|
346
|
+
return null;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return calculateServiceCost(call, serviceInfo);
|
|
350
|
+
})();
|
|
351
|
+
if (!result) {
|
|
330
352
|
continue;
|
|
331
353
|
}
|
|
354
|
+
if (!serviceResults[serviceInfo.serviceName]) {
|
|
355
|
+
serviceResults[serviceInfo.serviceName] = {
|
|
356
|
+
serviceName: serviceInfo.serviceName,
|
|
357
|
+
calls: [],
|
|
358
|
+
totalCost: 0
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
serviceResults[serviceInfo.serviceName].calls.push(result);
|
|
362
|
+
serviceResults[serviceInfo.serviceName].totalCost += result.cost;
|
|
363
|
+
continue;
|
|
332
364
|
}
|
|
333
|
-
|
|
334
|
-
if (
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
calls: [],
|
|
338
|
-
|
|
339
|
-
|
|
365
|
+
// Unclassified HTTP node — still surface its cost if addRequestCost was called.
|
|
366
|
+
if (call.attributeCost !== undefined && call.attributeCost > 0) {
|
|
367
|
+
const bucket = 'http';
|
|
368
|
+
if (!serviceResults[bucket]) {
|
|
369
|
+
serviceResults[bucket] = { serviceName: bucket, calls: [], totalCost: 0 };
|
|
370
|
+
}
|
|
371
|
+
serviceResults[bucket].calls.push({
|
|
372
|
+
step: call.stepName,
|
|
373
|
+
cost: call.attributeCost,
|
|
374
|
+
usage: '1 request',
|
|
375
|
+
endpoint: call.url
|
|
376
|
+
});
|
|
377
|
+
serviceResults[bucket].totalCost += call.attributeCost;
|
|
340
378
|
}
|
|
341
|
-
serviceResults[serviceInfo.serviceName].calls.push(result);
|
|
342
|
-
serviceResults[serviceInfo.serviceName].totalCost += result.cost;
|
|
343
379
|
}
|
|
344
380
|
const { results: llmResults, totalInputTokens, totalOutputTokens, totalCachedTokens, totalReasoningTokens, llmTotalCost, unknownModels } = aggregateLLMCosts(llmCalls, config);
|
|
345
381
|
const serviceTotalCost = Object.values(serviceResults).reduce((sum, s) => sum + s.totalCost, 0);
|
|
@@ -27,7 +27,9 @@ const llmTrace = {
|
|
|
27
27
|
kind: 'llm',
|
|
28
28
|
name: 'generate_summary',
|
|
29
29
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
30
|
-
|
|
30
|
+
attributes: {
|
|
31
|
+
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
32
|
+
}
|
|
31
33
|
}
|
|
32
34
|
]
|
|
33
35
|
},
|
|
@@ -41,7 +43,9 @@ const llmTrace = {
|
|
|
41
43
|
kind: 'llm',
|
|
42
44
|
name: 'analyze_data',
|
|
43
45
|
input: { loadedPrompt: { config: { model: 'claude-haiku-4-5' } } },
|
|
44
|
-
|
|
46
|
+
attributes: {
|
|
47
|
+
token_usage: { inputTokens: 2000, outputTokens: 1000, cachedInputTokens: 500 }
|
|
48
|
+
}
|
|
45
49
|
}
|
|
46
50
|
]
|
|
47
51
|
}
|
|
@@ -107,7 +111,9 @@ const duplicateTrace = {
|
|
|
107
111
|
kind: 'llm',
|
|
108
112
|
name: 'generate',
|
|
109
113
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
110
|
-
|
|
114
|
+
attributes: {
|
|
115
|
+
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
116
|
+
}
|
|
111
117
|
}
|
|
112
118
|
]
|
|
113
119
|
},
|
|
@@ -121,7 +127,9 @@ const duplicateTrace = {
|
|
|
121
127
|
kind: 'llm',
|
|
122
128
|
name: 'generate',
|
|
123
129
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
124
|
-
|
|
130
|
+
attributes: {
|
|
131
|
+
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
132
|
+
}
|
|
125
133
|
}
|
|
126
134
|
]
|
|
127
135
|
}
|
|
@@ -181,12 +189,43 @@ describe('findLLMCalls', () => {
|
|
|
181
189
|
expect(calls[0].model).toBe('claude-sonnet-4-5');
|
|
182
190
|
expect(calls[1].model).toBe('claude-haiku-4-5');
|
|
183
191
|
});
|
|
184
|
-
it('extracts token usage', () => {
|
|
192
|
+
it('extracts token usage from attributes.token_usage', () => {
|
|
185
193
|
const calls = findLLMCalls(llmTrace);
|
|
186
194
|
expect(calls[0].usage.inputTokens).toBe(1000);
|
|
187
195
|
expect(calls[0].usage.outputTokens).toBe(500);
|
|
188
196
|
expect(calls[1].usage.cachedInputTokens).toBe(500);
|
|
189
197
|
});
|
|
198
|
+
it('ignores llm nodes that only have legacy output.usage (no attributes.token_usage)', () => {
|
|
199
|
+
const legacyOnlyTrace = {
|
|
200
|
+
kind: 'workflow',
|
|
201
|
+
name: 'wf',
|
|
202
|
+
children: [{
|
|
203
|
+
id: 'llm-legacy',
|
|
204
|
+
kind: 'llm',
|
|
205
|
+
name: 'gen',
|
|
206
|
+
output: { usage: { inputTokens: 999, outputTokens: 999 } }
|
|
207
|
+
}]
|
|
208
|
+
};
|
|
209
|
+
expect(findLLMCalls(legacyOnlyTrace)).toHaveLength(0);
|
|
210
|
+
});
|
|
211
|
+
it('picks up attributeCost from attributes.cost.total', () => {
|
|
212
|
+
const trace = {
|
|
213
|
+
kind: 'workflow',
|
|
214
|
+
name: 'wf',
|
|
215
|
+
children: [{
|
|
216
|
+
id: 'llm-1',
|
|
217
|
+
kind: 'llm',
|
|
218
|
+
name: 'gen',
|
|
219
|
+
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
220
|
+
attributes: {
|
|
221
|
+
token_usage: { inputTokens: 100, outputTokens: 50 },
|
|
222
|
+
cost: { total: 0.0042 }
|
|
223
|
+
}
|
|
224
|
+
}]
|
|
225
|
+
};
|
|
226
|
+
const calls = findLLMCalls(trace);
|
|
227
|
+
expect(calls[0].attributeCost).toBeCloseTo(0.0042, 10);
|
|
228
|
+
});
|
|
190
229
|
it('deduplicates by ID', () => {
|
|
191
230
|
const calls = findLLMCalls(duplicateTrace);
|
|
192
231
|
expect(calls).toHaveLength(1);
|
|
@@ -207,6 +246,22 @@ describe('findHTTPCalls', () => {
|
|
|
207
246
|
expect(calls[0].stepName).toBe('fetch_content');
|
|
208
247
|
expect(calls[1].stepName).toBe('search');
|
|
209
248
|
});
|
|
249
|
+
it('reads attributeCost from attributes.cost.total', () => {
|
|
250
|
+
const trace = {
|
|
251
|
+
kind: 'workflow',
|
|
252
|
+
name: 'wf',
|
|
253
|
+
children: [{
|
|
254
|
+
id: 'http-1',
|
|
255
|
+
kind: 'http',
|
|
256
|
+
name: 'scrape',
|
|
257
|
+
input: { url: 'https://api.gx-scraper.test/scrape', method: 'POST' },
|
|
258
|
+
output: { status: 200 },
|
|
259
|
+
attributes: { cost: { total: 0.5 } }
|
|
260
|
+
}]
|
|
261
|
+
};
|
|
262
|
+
const calls = findHTTPCalls(trace);
|
|
263
|
+
expect(calls[0].attributeCost).toBe(0.5);
|
|
264
|
+
});
|
|
210
265
|
});
|
|
211
266
|
describe('calculateLLMCallCost', () => {
|
|
212
267
|
it('calculates cost for known model', () => {
|
|
@@ -421,7 +476,7 @@ describe('calculateCost', () => {
|
|
|
421
476
|
kind: 'llm',
|
|
422
477
|
name: 'gen',
|
|
423
478
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5-20250514' } } },
|
|
424
|
-
|
|
479
|
+
attributes: { token_usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
425
480
|
}]
|
|
426
481
|
}]
|
|
427
482
|
};
|
|
@@ -430,7 +485,7 @@ describe('calculateCost', () => {
|
|
|
430
485
|
expect(report.unknownModels).toHaveLength(0);
|
|
431
486
|
expect(report.llmCalls[0].warning).toBe('priced as claude-sonnet-4-5');
|
|
432
487
|
});
|
|
433
|
-
it('reports unknown model when no prefix match exists', () => {
|
|
488
|
+
it('reports unknown model when no prefix match exists and no attribute cost is present', () => {
|
|
434
489
|
const trace = {
|
|
435
490
|
kind: 'workflow',
|
|
436
491
|
name: 'test',
|
|
@@ -443,7 +498,7 @@ describe('calculateCost', () => {
|
|
|
443
498
|
kind: 'llm',
|
|
444
499
|
name: 'gen',
|
|
445
500
|
input: { loadedPrompt: { config: { model: 'totally-unknown-model' } } },
|
|
446
|
-
|
|
501
|
+
attributes: { token_usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
447
502
|
}]
|
|
448
503
|
}]
|
|
449
504
|
};
|
|
@@ -465,13 +520,160 @@ describe('calculateCost', () => {
|
|
|
465
520
|
kind: 'llm',
|
|
466
521
|
name: 'gen',
|
|
467
522
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
468
|
-
|
|
523
|
+
attributes: { token_usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
469
524
|
}]
|
|
470
525
|
}]
|
|
471
526
|
};
|
|
472
527
|
const report = calculateCost(trace, testConfig, 'test.json');
|
|
473
528
|
expect(report.llmCalls[0].warning).toBeUndefined();
|
|
474
529
|
});
|
|
530
|
+
it('prefers attributes.cost.total over yaml-computed LLM cost when present', () => {
|
|
531
|
+
const trace = {
|
|
532
|
+
kind: 'workflow',
|
|
533
|
+
name: 'test',
|
|
534
|
+
children: [{
|
|
535
|
+
id: 'step-1',
|
|
536
|
+
kind: 'step',
|
|
537
|
+
name: 'test#gen',
|
|
538
|
+
children: [{
|
|
539
|
+
id: 'llm-1',
|
|
540
|
+
kind: 'llm',
|
|
541
|
+
name: 'gen',
|
|
542
|
+
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
543
|
+
attributes: {
|
|
544
|
+
token_usage: { inputTokens: 1000, outputTokens: 500 },
|
|
545
|
+
// Authoritative: this is what the provider reported, not what yaml infers.
|
|
546
|
+
cost: { total: 0.99 }
|
|
547
|
+
}
|
|
548
|
+
}]
|
|
549
|
+
}]
|
|
550
|
+
};
|
|
551
|
+
const report = calculateCost(trace, testConfig, 'test.json');
|
|
552
|
+
expect(report.llmCalls[0].cost).toBe(0.99);
|
|
553
|
+
expect(report.llmTotalCost).toBe(0.99);
|
|
554
|
+
});
|
|
555
|
+
it('still surfaces an attribute LLM cost for unknown models without warning', () => {
|
|
556
|
+
const trace = {
|
|
557
|
+
kind: 'workflow',
|
|
558
|
+
name: 'test',
|
|
559
|
+
children: [{
|
|
560
|
+
id: 'step-1',
|
|
561
|
+
kind: 'step',
|
|
562
|
+
name: 'test#gen',
|
|
563
|
+
children: [{
|
|
564
|
+
id: 'llm-1',
|
|
565
|
+
kind: 'llm',
|
|
566
|
+
name: 'gen',
|
|
567
|
+
input: { loadedPrompt: { config: { model: 'brand-new-model' } } },
|
|
568
|
+
attributes: {
|
|
569
|
+
token_usage: { inputTokens: 100, outputTokens: 20 },
|
|
570
|
+
cost: { total: 0.0123 }
|
|
571
|
+
}
|
|
572
|
+
}]
|
|
573
|
+
}]
|
|
574
|
+
};
|
|
575
|
+
const report = calculateCost(trace, testConfig, 'test.json');
|
|
576
|
+
expect(report.llmTotalCost).toBeCloseTo(0.0123, 10);
|
|
577
|
+
expect(report.unknownModels).toEqual([]);
|
|
578
|
+
});
|
|
579
|
+
it('includes HTTP request cost from attributes.cost.total for unclassified hosts', () => {
|
|
580
|
+
// gx-scraper-style: not declared in yaml services, but addRequestCost ran
|
|
581
|
+
// and put $0.50 on the HTTP node's attributes.cost.total.
|
|
582
|
+
const trace = {
|
|
583
|
+
kind: 'workflow',
|
|
584
|
+
name: 'scraper_workflow',
|
|
585
|
+
startedAt: 1700000000000,
|
|
586
|
+
endedAt: 1700000100000,
|
|
587
|
+
children: [{
|
|
588
|
+
id: 'step-1',
|
|
589
|
+
kind: 'step',
|
|
590
|
+
name: 'scraper_workflow#scrape',
|
|
591
|
+
children: [{
|
|
592
|
+
id: 'http-1',
|
|
593
|
+
kind: 'http',
|
|
594
|
+
name: 'scrape_request',
|
|
595
|
+
input: { url: 'https://api.gx-scraper.test/v1/scrape', method: 'POST' },
|
|
596
|
+
output: { status: 200, body: {} },
|
|
597
|
+
attributes: { cost: { total: 0.5 } }
|
|
598
|
+
}]
|
|
599
|
+
}]
|
|
600
|
+
};
|
|
601
|
+
const report = calculateCost(trace, testConfig, 'test.json');
|
|
602
|
+
expect(report.serviceTotalCost).toBe(0.5);
|
|
603
|
+
expect(report.totalCost).toBe(0.5);
|
|
604
|
+
expect(report.services).toHaveLength(1);
|
|
605
|
+
expect(report.services[0].calls[0].cost).toBe(0.5);
|
|
606
|
+
});
|
|
607
|
+
it('prefers attributes.cost.total over yaml service classifier for classified HTTP nodes', () => {
|
|
608
|
+
// Exa with both a yaml `response_cost` rule AND an attribute cost — the
|
|
609
|
+
// attribute is authoritative.
|
|
610
|
+
const trace = {
|
|
611
|
+
kind: 'workflow',
|
|
612
|
+
name: 'test_workflow',
|
|
613
|
+
children: [{
|
|
614
|
+
id: 'step-exa',
|
|
615
|
+
kind: 'step',
|
|
616
|
+
name: 'test_workflow#search',
|
|
617
|
+
children: [{
|
|
618
|
+
id: 'http-exa',
|
|
619
|
+
kind: 'http',
|
|
620
|
+
name: 'exa_request',
|
|
621
|
+
input: { url: 'https://api.exa.ai/research', method: 'POST' },
|
|
622
|
+
output: {
|
|
623
|
+
status: 200,
|
|
624
|
+
body: { model: 'exa-research', costDollars: { total: 0.15, numSearches: 1, numPages: 5 } }
|
|
625
|
+
},
|
|
626
|
+
attributes: { cost: { total: 0.22 } }
|
|
627
|
+
}]
|
|
628
|
+
}]
|
|
629
|
+
};
|
|
630
|
+
const report = calculateCost(trace, testConfig, 'test.json');
|
|
631
|
+
expect(report.services).toHaveLength(1);
|
|
632
|
+
expect(report.services[0].serviceName).toBe('exa');
|
|
633
|
+
expect(report.services[0].totalCost).toBe(0.22);
|
|
634
|
+
});
|
|
635
|
+
it('combines LLM and HTTP attribute costs into a single trace total', () => {
|
|
636
|
+
const trace = {
|
|
637
|
+
kind: 'workflow',
|
|
638
|
+
name: 'mixed_workflow',
|
|
639
|
+
startedAt: 1700000000000,
|
|
640
|
+
endedAt: 1700000100000,
|
|
641
|
+
children: [
|
|
642
|
+
{
|
|
643
|
+
id: 'step-llm',
|
|
644
|
+
kind: 'step',
|
|
645
|
+
name: 'mixed_workflow#draft',
|
|
646
|
+
children: [{
|
|
647
|
+
id: 'llm-1',
|
|
648
|
+
kind: 'llm',
|
|
649
|
+
name: 'draft',
|
|
650
|
+
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
651
|
+
attributes: {
|
|
652
|
+
token_usage: { inputTokens: 100, outputTokens: 50 },
|
|
653
|
+
cost: { total: 0.01 }
|
|
654
|
+
}
|
|
655
|
+
}]
|
|
656
|
+
},
|
|
657
|
+
{
|
|
658
|
+
id: 'step-http',
|
|
659
|
+
kind: 'step',
|
|
660
|
+
name: 'mixed_workflow#scrape',
|
|
661
|
+
children: [{
|
|
662
|
+
id: 'http-1',
|
|
663
|
+
kind: 'http',
|
|
664
|
+
name: 'scrape',
|
|
665
|
+
input: { url: 'https://api.gx-scraper.test/v1/scrape', method: 'POST' },
|
|
666
|
+
output: { status: 200, body: {} },
|
|
667
|
+
attributes: { cost: { total: 0.5 } }
|
|
668
|
+
}]
|
|
669
|
+
}
|
|
670
|
+
]
|
|
671
|
+
};
|
|
672
|
+
const report = calculateCost(trace, testConfig, 'test.json');
|
|
673
|
+
expect(report.llmTotalCost).toBeCloseTo(0.01, 10);
|
|
674
|
+
expect(report.serviceTotalCost).toBeCloseTo(0.5, 10);
|
|
675
|
+
expect(report.totalCost).toBeCloseTo(0.51, 10);
|
|
676
|
+
});
|
|
475
677
|
});
|
|
476
678
|
describe('loadPricingConfig', () => {
|
|
477
679
|
beforeEach(() => {
|
package/dist/types/cost.d.ts
CHANGED
|
@@ -8,6 +8,18 @@ export interface TokenUsage {
|
|
|
8
8
|
outputTokens?: number;
|
|
9
9
|
cachedInputTokens?: number;
|
|
10
10
|
reasoningTokens?: number;
|
|
11
|
+
totalTokens?: number;
|
|
12
|
+
}
|
|
13
|
+
export interface TraceCostAttribute {
|
|
14
|
+
total: number;
|
|
15
|
+
components?: Array<{
|
|
16
|
+
name: string;
|
|
17
|
+
value: number;
|
|
18
|
+
}>;
|
|
19
|
+
}
|
|
20
|
+
export interface TraceAttributes {
|
|
21
|
+
cost?: TraceCostAttribute;
|
|
22
|
+
token_usage?: TokenUsage;
|
|
11
23
|
}
|
|
12
24
|
export interface TraceNode {
|
|
13
25
|
id?: string;
|
|
@@ -17,15 +29,15 @@ export interface TraceNode {
|
|
|
17
29
|
endedAt?: number;
|
|
18
30
|
children?: TraceNode[];
|
|
19
31
|
input?: Record<string, unknown>;
|
|
20
|
-
output?: Record<string, unknown
|
|
21
|
-
|
|
22
|
-
};
|
|
32
|
+
output?: Record<string, unknown>;
|
|
33
|
+
attributes?: TraceAttributes;
|
|
23
34
|
}
|
|
24
35
|
export interface LLMCall {
|
|
25
36
|
stepName: string;
|
|
26
37
|
llmName: string;
|
|
27
38
|
model: string;
|
|
28
39
|
usage: TokenUsage;
|
|
40
|
+
attributeCost?: number;
|
|
29
41
|
}
|
|
30
42
|
export interface HTTPCall {
|
|
31
43
|
stepName: string;
|
|
@@ -34,6 +46,7 @@ export interface HTTPCall {
|
|
|
34
46
|
input: Record<string, unknown>;
|
|
35
47
|
output: Record<string, unknown>;
|
|
36
48
|
status?: number;
|
|
49
|
+
attributeCost?: number;
|
|
37
50
|
}
|
|
38
51
|
export interface ModelPricing {
|
|
39
52
|
provider: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@outputai/cli",
|
|
3
|
-
"version": "0.4.0",
|
|
3
|
+
"version": "0.4.1-dev.56c13a8.0",
|
|
4
4
|
"description": "CLI for Output.ai workflow generation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
"semver": "7.7.4",
|
|
37
37
|
"undici": "8.1.0",
|
|
38
38
|
"yaml": "^2.8.3",
|
|
39
|
-
"@outputai/
|
|
40
|
-
"@outputai/
|
|
41
|
-
"@outputai/
|
|
39
|
+
"@outputai/credentials": "0.4.1-dev.56c13a8.0",
|
|
40
|
+
"@outputai/evals": "0.4.1-dev.56c13a8.0",
|
|
41
|
+
"@outputai/llm": "0.4.1-dev.56c13a8.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@types/cli-progress": "3.11.6",
|