@outputai/cli 0.4.1-dev.92bc2fb.0 → 0.4.1-dev.c0b98d8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/generated/api.d.ts +0 -120
- package/dist/api/generated/api.js +0 -18
- package/dist/assets/docker/docker-compose-dev.yml +1 -1
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/cost_calculator.js +26 -62
- package/dist/services/cost_calculator.spec.js +9 -211
- package/dist/types/cost.d.ts +3 -16
- package/package.json +4 -4
|
@@ -146,58 +146,6 @@ export interface TraceLogLocalResponse {
|
|
|
146
146
|
/** Absolute path to local trace file */
|
|
147
147
|
localPath: string;
|
|
148
148
|
}
|
|
149
|
-
export interface TraceAttributesCostComponent {
|
|
150
|
-
/** Canonical cost event name (e.g. `cost:llm:request`, `cost:http:request`, `other`). */
|
|
151
|
-
name: string;
|
|
152
|
-
/** Summed USD cost for this component bucket. */
|
|
153
|
-
value: number;
|
|
154
|
-
}
|
|
155
|
-
export type TraceAttributesResponseAttributesCost = {
|
|
156
|
-
/** USD; equal to the sum of `components[].value` */
|
|
157
|
-
total: number;
|
|
158
|
-
/** Cost contributions bucketed by canonical event name (llm / http / other). */
|
|
159
|
-
components: TraceAttributesCostComponent[];
|
|
160
|
-
};
|
|
161
|
-
export type TraceAttributesResponseAttributesTokenUsage = {
|
|
162
|
-
inputTokens: number;
|
|
163
|
-
outputTokens: number;
|
|
164
|
-
cachedInputTokens: number;
|
|
165
|
-
totalTokens: number;
|
|
166
|
-
};
|
|
167
|
-
export type TraceAttributesResponseAttributes = {
|
|
168
|
-
cost: TraceAttributesResponseAttributesCost;
|
|
169
|
-
tokenUsage: TraceAttributesResponseAttributesTokenUsage;
|
|
170
|
-
};
|
|
171
|
-
/**
|
|
172
|
-
* Aggregated cost, token usage, and runtime computed by walking the trace tree of a completed workflow run.
|
|
173
|
-
Component breakdowns under `attributes.cost.components` are grouped by the canonical event name that
|
|
174
|
-
emitted them (inferred from node kind: llm/http/other). Values sum to `attributes.cost.total`.
|
|
175
|
-
|
|
176
|
-
*/
|
|
177
|
-
export interface TraceAttributesResponse {
|
|
178
|
-
/** The workflow execution id */
|
|
179
|
-
workflowId: string;
|
|
180
|
-
/** The specific run id this aggregation belongs to */
|
|
181
|
-
runId: string;
|
|
182
|
-
/**
|
|
183
|
-
* ms epoch of the root trace node's start
|
|
184
|
-
* @nullable
|
|
185
|
-
*/
|
|
186
|
-
startTime: number | null;
|
|
187
|
-
/**
|
|
188
|
-
* ms epoch of the root trace node's end
|
|
189
|
-
* @nullable
|
|
190
|
-
*/
|
|
191
|
-
finishTime: number | null;
|
|
192
|
-
/**
|
|
193
|
-
* ms between finishTime and startTime, null if either timestamp is missing
|
|
194
|
-
* @nullable
|
|
195
|
-
*/
|
|
196
|
-
runtime: number | null;
|
|
197
|
-
attributes: TraceAttributesResponseAttributes;
|
|
198
|
-
/** S3 URL of the underlying trace file (same value `/trace-log` returns under `data` for remote traces). */
|
|
199
|
-
traceUrl: string;
|
|
200
|
-
}
|
|
201
149
|
/**
|
|
202
150
|
* Current run status
|
|
203
151
|
*/
|
|
@@ -1002,74 +950,6 @@ export type getWorkflowIdRunsRidTraceLogResponseError = (getWorkflowIdRunsRidTra
|
|
|
1002
950
|
export type getWorkflowIdRunsRidTraceLogResponse = (getWorkflowIdRunsRidTraceLogResponseSuccess | getWorkflowIdRunsRidTraceLogResponseError);
|
|
1003
951
|
export declare const getGetWorkflowIdRunsRidTraceLogUrl: (id: string, rid: string) => string;
|
|
1004
952
|
export declare const getWorkflowIdRunsRidTraceLog: (id: string, rid: string, options?: ApiRequestOptions) => Promise<getWorkflowIdRunsRidTraceLogResponse>;
|
|
1005
|
-
/**
|
|
1006
|
-
* Returns runtime, cost rolled up by event-name bucket, token-usage totals, and the trace S3 URL
|
|
1007
|
-
for the latest run of the given workflow. Completion-only — returns 424 while the workflow is
|
|
1008
|
-
still running, mirroring `/result` and `/trace-log`. To pin a specific run, use
|
|
1009
|
-
`/workflow/{id}/runs/{rid}/trace-attributes`.
|
|
1010
|
-
|
|
1011
|
-
* @summary Get aggregated trace attributes for a completed workflow (latest run)
|
|
1012
|
-
*/
|
|
1013
|
-
export type getWorkflowIdTraceAttributesResponse200 = {
|
|
1014
|
-
data: TraceAttributesResponse;
|
|
1015
|
-
status: 200;
|
|
1016
|
-
};
|
|
1017
|
-
export type getWorkflowIdTraceAttributesResponse404 = {
|
|
1018
|
-
data: NotFoundResponse;
|
|
1019
|
-
status: 404;
|
|
1020
|
-
};
|
|
1021
|
-
export type getWorkflowIdTraceAttributesResponse424 = {
|
|
1022
|
-
data: FailedDependencyResponse;
|
|
1023
|
-
status: 424;
|
|
1024
|
-
};
|
|
1025
|
-
export type getWorkflowIdTraceAttributesResponse500 = {
|
|
1026
|
-
data: InternalServerErrorResponse;
|
|
1027
|
-
status: 500;
|
|
1028
|
-
};
|
|
1029
|
-
export type getWorkflowIdTraceAttributesResponseSuccess = (getWorkflowIdTraceAttributesResponse200) & {
|
|
1030
|
-
headers: Headers;
|
|
1031
|
-
};
|
|
1032
|
-
export type getWorkflowIdTraceAttributesResponseError = (getWorkflowIdTraceAttributesResponse404 | getWorkflowIdTraceAttributesResponse424 | getWorkflowIdTraceAttributesResponse500) & {
|
|
1033
|
-
headers: Headers;
|
|
1034
|
-
};
|
|
1035
|
-
export type getWorkflowIdTraceAttributesResponse = (getWorkflowIdTraceAttributesResponseSuccess | getWorkflowIdTraceAttributesResponseError);
|
|
1036
|
-
export declare const getGetWorkflowIdTraceAttributesUrl: (id: string) => string;
|
|
1037
|
-
export declare const getWorkflowIdTraceAttributes: (id: string, options?: ApiRequestOptions) => Promise<getWorkflowIdTraceAttributesResponse>;
|
|
1038
|
-
/**
|
|
1039
|
-
* Returns runtime, cost rolled up by event-name bucket, token-usage totals, and the trace S3 URL
|
|
1040
|
-
for the pinned workflow run. Completion-only — returns 424 while the run is still in progress.
|
|
1041
|
-
|
|
1042
|
-
* @summary Get aggregated trace attributes for a specific completed workflow run
|
|
1043
|
-
*/
|
|
1044
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse200 = {
|
|
1045
|
-
data: TraceAttributesResponse;
|
|
1046
|
-
status: 200;
|
|
1047
|
-
};
|
|
1048
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse400 = {
|
|
1049
|
-
data: BadRequestResponse;
|
|
1050
|
-
status: 400;
|
|
1051
|
-
};
|
|
1052
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse404 = {
|
|
1053
|
-
data: NotFoundResponse;
|
|
1054
|
-
status: 404;
|
|
1055
|
-
};
|
|
1056
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse424 = {
|
|
1057
|
-
data: FailedDependencyResponse;
|
|
1058
|
-
status: 424;
|
|
1059
|
-
};
|
|
1060
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse500 = {
|
|
1061
|
-
data: InternalServerErrorResponse;
|
|
1062
|
-
status: 500;
|
|
1063
|
-
};
|
|
1064
|
-
export type getWorkflowIdRunsRidTraceAttributesResponseSuccess = (getWorkflowIdRunsRidTraceAttributesResponse200) & {
|
|
1065
|
-
headers: Headers;
|
|
1066
|
-
};
|
|
1067
|
-
export type getWorkflowIdRunsRidTraceAttributesResponseError = (getWorkflowIdRunsRidTraceAttributesResponse400 | getWorkflowIdRunsRidTraceAttributesResponse404 | getWorkflowIdRunsRidTraceAttributesResponse424 | getWorkflowIdRunsRidTraceAttributesResponse500) & {
|
|
1068
|
-
headers: Headers;
|
|
1069
|
-
};
|
|
1070
|
-
export type getWorkflowIdRunsRidTraceAttributesResponse = (getWorkflowIdRunsRidTraceAttributesResponseSuccess | getWorkflowIdRunsRidTraceAttributesResponseError);
|
|
1071
|
-
export declare const getGetWorkflowIdRunsRidTraceAttributesUrl: (id: string, rid: string) => string;
|
|
1072
|
-
export declare const getWorkflowIdRunsRidTraceAttributes: (id: string, rid: string, options?: ApiRequestOptions) => Promise<getWorkflowIdRunsRidTraceAttributesResponse>;
|
|
1073
953
|
/**
|
|
1074
954
|
* Returns decoded Temporal history events with optional payload inclusion. First page includes workflow metadata; subsequent pages return events only.
|
|
1075
955
|
* @summary Get paginated workflow execution history
|
|
@@ -194,24 +194,6 @@ export const getWorkflowIdRunsRidTraceLog = async (id, rid, options) => {
|
|
|
194
194
|
method: 'GET'
|
|
195
195
|
});
|
|
196
196
|
};
|
|
197
|
-
export const getGetWorkflowIdTraceAttributesUrl = (id) => {
|
|
198
|
-
return `/workflow/${id}/trace-attributes`;
|
|
199
|
-
};
|
|
200
|
-
export const getWorkflowIdTraceAttributes = async (id, options) => {
|
|
201
|
-
return customFetchInstance(getGetWorkflowIdTraceAttributesUrl(id), {
|
|
202
|
-
...options,
|
|
203
|
-
method: 'GET'
|
|
204
|
-
});
|
|
205
|
-
};
|
|
206
|
-
export const getGetWorkflowIdRunsRidTraceAttributesUrl = (id, rid) => {
|
|
207
|
-
return `/workflow/${id}/runs/${rid}/trace-attributes`;
|
|
208
|
-
};
|
|
209
|
-
export const getWorkflowIdRunsRidTraceAttributes = async (id, rid, options) => {
|
|
210
|
-
return customFetchInstance(getGetWorkflowIdRunsRidTraceAttributesUrl(id, rid), {
|
|
211
|
-
...options,
|
|
212
|
-
method: 'GET'
|
|
213
|
-
});
|
|
214
|
-
};
|
|
215
197
|
export const getGetWorkflowIdHistoryUrl = (id, params) => {
|
|
216
198
|
const normalizedParams = new URLSearchParams();
|
|
217
199
|
Object.entries(params || {}).forEach(([key, value]) => {
|
|
@@ -5,9 +5,6 @@ const ARRAY_ACCESS_PATTERN = /^(\w+)\[(\d+)\]$/;
|
|
|
5
5
|
function tokenCost(tokens, pricePerMillion) {
|
|
6
6
|
return (tokens / 1_000_000) * pricePerMillion;
|
|
7
7
|
}
|
|
8
|
-
function isFiniteNumber(value) {
|
|
9
|
-
return typeof value === 'number' && Number.isFinite(value);
|
|
10
|
-
}
|
|
11
8
|
export function extractValue(obj, path) {
|
|
12
9
|
if (!path || !obj) {
|
|
13
10
|
return obj;
|
|
@@ -72,25 +69,20 @@ function findCalls(node, match, extract, parentStepName = null, seenIds = new Se
|
|
|
72
69
|
}
|
|
73
70
|
return calls;
|
|
74
71
|
}
|
|
75
|
-
function readAttributeCost(node) {
|
|
76
|
-
const total = node.attributes?.cost?.total;
|
|
77
|
-
return isFiniteNumber(total) ? total : undefined;
|
|
78
|
-
}
|
|
79
72
|
export function findLLMCalls(node, parentStepName = null, seenIds = new Set()) {
|
|
80
|
-
return findCalls(node, n => n.kind === 'llm' && !!n.
|
|
73
|
+
return findCalls(node, n => n.kind === 'llm' && !!n.output?.usage, (n, stepName) => {
|
|
81
74
|
const loadedPrompt = n.input?.loadedPrompt;
|
|
82
|
-
const outputRecord =
|
|
83
|
-
const inputRecord =
|
|
75
|
+
const outputRecord = n.output;
|
|
76
|
+
const inputRecord = n.input;
|
|
84
77
|
const model = loadedPrompt?.config?.model ||
|
|
85
|
-
outputRecord
|
|
86
|
-
inputRecord
|
|
78
|
+
outputRecord?.model ||
|
|
79
|
+
inputRecord?.model ||
|
|
87
80
|
'unknown';
|
|
88
81
|
return {
|
|
89
82
|
stepName: stepName || n.name || 'unknown',
|
|
90
83
|
llmName: n.name || 'llm',
|
|
91
84
|
model,
|
|
92
|
-
usage: n.
|
|
93
|
-
attributeCost: readAttributeCost(n)
|
|
85
|
+
usage: n.output.usage
|
|
94
86
|
};
|
|
95
87
|
}, parentStepName, seenIds);
|
|
96
88
|
}
|
|
@@ -101,8 +93,7 @@ export function findHTTPCalls(node, parentStepName = null, seenIds = new Set())
|
|
|
101
93
|
method: n.input?.method || 'GET',
|
|
102
94
|
input: n.input || {},
|
|
103
95
|
output: n.output || {},
|
|
104
|
-
status: n.output?.status
|
|
105
|
-
attributeCost: readAttributeCost(n)
|
|
96
|
+
status: n.output?.status
|
|
106
97
|
}), parentStepName, seenIds);
|
|
107
98
|
}
|
|
108
99
|
export function calculateLLMCallCost(usage, modelPricing) {
|
|
@@ -286,14 +277,11 @@ function aggregateLLMCosts(llmCalls, config) {
|
|
|
286
277
|
const totals = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, cost: 0 };
|
|
287
278
|
for (const call of llmCalls) {
|
|
288
279
|
const { pricing, matchedKey } = findModelPricing(call.model, config.models ?? {});
|
|
289
|
-
const { cost
|
|
290
|
-
// Prefer the cost emitted by the LLM provider on the trace node; fall back to
|
|
291
|
-
// yaml pricing so unknown-model warnings still surface for breakdown display.
|
|
292
|
-
const cost = call.attributeCost ?? yamlCost;
|
|
280
|
+
const { cost, warning } = calculateLLMCallCost(call.usage, pricing);
|
|
293
281
|
const prefixWarning = (pricing && matchedKey !== call.model) ?
|
|
294
282
|
`priced as ${matchedKey}` :
|
|
295
283
|
undefined;
|
|
296
|
-
if (!pricing
|
|
284
|
+
if (!pricing) {
|
|
297
285
|
unknownModels.add(call.model);
|
|
298
286
|
}
|
|
299
287
|
results.push({
|
|
@@ -331,51 +319,27 @@ export function calculateCost(trace, config, traceFile = '') {
|
|
|
331
319
|
continue;
|
|
332
320
|
}
|
|
333
321
|
const serviceInfo = identifyService(call, config.services);
|
|
334
|
-
if (serviceInfo) {
|
|
335
|
-
// attributes.cost on the HTTP node is authoritative when present —
|
|
336
|
-
// addRequestCost() writes the real billed amount there. Fall back to
|
|
337
|
-
// the yaml service classifier for legacy callers that don't emit it.
|
|
338
|
-
const result = call.attributeCost !== undefined ?
|
|
339
|
-
{ step: call.stepName, cost: call.attributeCost, usage: '1 request' } :
|
|
340
|
-
(() => {
|
|
341
|
-
if (serviceInfo.config.type === 'response_cost') {
|
|
342
|
-
const hasCostData = extractValue(call, serviceInfo.config.cost_path);
|
|
343
|
-
const isBillableMethod = serviceInfo.config.billable_method &&
|
|
344
|
-
call.method === serviceInfo.config.billable_method;
|
|
345
|
-
if (!hasCostData && !isBillableMethod) {
|
|
346
|
-
return null;
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
return calculateServiceCost(call, serviceInfo);
|
|
350
|
-
})();
|
|
351
|
-
if (!result) {
|
|
352
|
-
continue;
|
|
353
|
-
}
|
|
354
|
-
if (!serviceResults[serviceInfo.serviceName]) {
|
|
355
|
-
serviceResults[serviceInfo.serviceName] = {
|
|
356
|
-
serviceName: serviceInfo.serviceName,
|
|
357
|
-
calls: [],
|
|
358
|
-
totalCost: 0
|
|
359
|
-
};
|
|
360
|
-
}
|
|
361
|
-
serviceResults[serviceInfo.serviceName].calls.push(result);
|
|
362
|
-
serviceResults[serviceInfo.serviceName].totalCost += result.cost;
|
|
322
|
+
if (!serviceInfo) {
|
|
363
323
|
continue;
|
|
364
324
|
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
const
|
|
368
|
-
|
|
369
|
-
|
|
325
|
+
if (serviceInfo.config.type === 'response_cost') {
|
|
326
|
+
const hasCostData = extractValue(call, serviceInfo.config.cost_path);
|
|
327
|
+
const isBillableMethod = serviceInfo.config.billable_method &&
|
|
328
|
+
call.method === serviceInfo.config.billable_method;
|
|
329
|
+
if (!hasCostData && !isBillableMethod) {
|
|
330
|
+
continue;
|
|
370
331
|
}
|
|
371
|
-
serviceResults[bucket].calls.push({
|
|
372
|
-
step: call.stepName,
|
|
373
|
-
cost: call.attributeCost,
|
|
374
|
-
usage: '1 request',
|
|
375
|
-
endpoint: call.url
|
|
376
|
-
});
|
|
377
|
-
serviceResults[bucket].totalCost += call.attributeCost;
|
|
378
332
|
}
|
|
333
|
+
const result = calculateServiceCost(call, serviceInfo);
|
|
334
|
+
if (!serviceResults[serviceInfo.serviceName]) {
|
|
335
|
+
serviceResults[serviceInfo.serviceName] = {
|
|
336
|
+
serviceName: serviceInfo.serviceName,
|
|
337
|
+
calls: [],
|
|
338
|
+
totalCost: 0
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
serviceResults[serviceInfo.serviceName].calls.push(result);
|
|
342
|
+
serviceResults[serviceInfo.serviceName].totalCost += result.cost;
|
|
379
343
|
}
|
|
380
344
|
const { results: llmResults, totalInputTokens, totalOutputTokens, totalCachedTokens, totalReasoningTokens, llmTotalCost, unknownModels } = aggregateLLMCosts(llmCalls, config);
|
|
381
345
|
const serviceTotalCost = Object.values(serviceResults).reduce((sum, s) => sum + s.totalCost, 0);
|
|
@@ -27,9 +27,7 @@ const llmTrace = {
|
|
|
27
27
|
kind: 'llm',
|
|
28
28
|
name: 'generate_summary',
|
|
29
29
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
30
|
-
|
|
31
|
-
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
32
|
-
}
|
|
30
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
33
31
|
}
|
|
34
32
|
]
|
|
35
33
|
},
|
|
@@ -43,9 +41,7 @@ const llmTrace = {
|
|
|
43
41
|
kind: 'llm',
|
|
44
42
|
name: 'analyze_data',
|
|
45
43
|
input: { loadedPrompt: { config: { model: 'claude-haiku-4-5' } } },
|
|
46
|
-
|
|
47
|
-
token_usage: { inputTokens: 2000, outputTokens: 1000, cachedInputTokens: 500 }
|
|
48
|
-
}
|
|
44
|
+
output: { usage: { inputTokens: 2000, outputTokens: 1000, cachedInputTokens: 500 } }
|
|
49
45
|
}
|
|
50
46
|
]
|
|
51
47
|
}
|
|
@@ -111,9 +107,7 @@ const duplicateTrace = {
|
|
|
111
107
|
kind: 'llm',
|
|
112
108
|
name: 'generate',
|
|
113
109
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
114
|
-
|
|
115
|
-
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
116
|
-
}
|
|
110
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
117
111
|
}
|
|
118
112
|
]
|
|
119
113
|
},
|
|
@@ -127,9 +121,7 @@ const duplicateTrace = {
|
|
|
127
121
|
kind: 'llm',
|
|
128
122
|
name: 'generate',
|
|
129
123
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
130
|
-
|
|
131
|
-
token_usage: { inputTokens: 1000, outputTokens: 500 }
|
|
132
|
-
}
|
|
124
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
133
125
|
}
|
|
134
126
|
]
|
|
135
127
|
}
|
|
@@ -189,43 +181,12 @@ describe('findLLMCalls', () => {
|
|
|
189
181
|
expect(calls[0].model).toBe('claude-sonnet-4-5');
|
|
190
182
|
expect(calls[1].model).toBe('claude-haiku-4-5');
|
|
191
183
|
});
|
|
192
|
-
it('extracts token usage
|
|
184
|
+
it('extracts token usage', () => {
|
|
193
185
|
const calls = findLLMCalls(llmTrace);
|
|
194
186
|
expect(calls[0].usage.inputTokens).toBe(1000);
|
|
195
187
|
expect(calls[0].usage.outputTokens).toBe(500);
|
|
196
188
|
expect(calls[1].usage.cachedInputTokens).toBe(500);
|
|
197
189
|
});
|
|
198
|
-
it('ignores llm nodes that only have legacy output.usage (no attributes.token_usage)', () => {
|
|
199
|
-
const legacyOnlyTrace = {
|
|
200
|
-
kind: 'workflow',
|
|
201
|
-
name: 'wf',
|
|
202
|
-
children: [{
|
|
203
|
-
id: 'llm-legacy',
|
|
204
|
-
kind: 'llm',
|
|
205
|
-
name: 'gen',
|
|
206
|
-
output: { usage: { inputTokens: 999, outputTokens: 999 } }
|
|
207
|
-
}]
|
|
208
|
-
};
|
|
209
|
-
expect(findLLMCalls(legacyOnlyTrace)).toHaveLength(0);
|
|
210
|
-
});
|
|
211
|
-
it('picks up attributeCost from attributes.cost.total', () => {
|
|
212
|
-
const trace = {
|
|
213
|
-
kind: 'workflow',
|
|
214
|
-
name: 'wf',
|
|
215
|
-
children: [{
|
|
216
|
-
id: 'llm-1',
|
|
217
|
-
kind: 'llm',
|
|
218
|
-
name: 'gen',
|
|
219
|
-
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
220
|
-
attributes: {
|
|
221
|
-
token_usage: { inputTokens: 100, outputTokens: 50 },
|
|
222
|
-
cost: { total: 0.0042 }
|
|
223
|
-
}
|
|
224
|
-
}]
|
|
225
|
-
};
|
|
226
|
-
const calls = findLLMCalls(trace);
|
|
227
|
-
expect(calls[0].attributeCost).toBeCloseTo(0.0042, 10);
|
|
228
|
-
});
|
|
229
190
|
it('deduplicates by ID', () => {
|
|
230
191
|
const calls = findLLMCalls(duplicateTrace);
|
|
231
192
|
expect(calls).toHaveLength(1);
|
|
@@ -246,22 +207,6 @@ describe('findHTTPCalls', () => {
|
|
|
246
207
|
expect(calls[0].stepName).toBe('fetch_content');
|
|
247
208
|
expect(calls[1].stepName).toBe('search');
|
|
248
209
|
});
|
|
249
|
-
it('reads attributeCost from attributes.cost.total', () => {
|
|
250
|
-
const trace = {
|
|
251
|
-
kind: 'workflow',
|
|
252
|
-
name: 'wf',
|
|
253
|
-
children: [{
|
|
254
|
-
id: 'http-1',
|
|
255
|
-
kind: 'http',
|
|
256
|
-
name: 'scrape',
|
|
257
|
-
input: { url: 'https://api.gx-scraper.test/scrape', method: 'POST' },
|
|
258
|
-
output: { status: 200 },
|
|
259
|
-
attributes: { cost: { total: 0.5 } }
|
|
260
|
-
}]
|
|
261
|
-
};
|
|
262
|
-
const calls = findHTTPCalls(trace);
|
|
263
|
-
expect(calls[0].attributeCost).toBe(0.5);
|
|
264
|
-
});
|
|
265
210
|
});
|
|
266
211
|
describe('calculateLLMCallCost', () => {
|
|
267
212
|
it('calculates cost for known model', () => {
|
|
@@ -476,7 +421,7 @@ describe('calculateCost', () => {
|
|
|
476
421
|
kind: 'llm',
|
|
477
422
|
name: 'gen',
|
|
478
423
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5-20250514' } } },
|
|
479
|
-
|
|
424
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
480
425
|
}]
|
|
481
426
|
}]
|
|
482
427
|
};
|
|
@@ -485,7 +430,7 @@ describe('calculateCost', () => {
|
|
|
485
430
|
expect(report.unknownModels).toHaveLength(0);
|
|
486
431
|
expect(report.llmCalls[0].warning).toBe('priced as claude-sonnet-4-5');
|
|
487
432
|
});
|
|
488
|
-
it('reports unknown model when no prefix match exists
|
|
433
|
+
it('reports unknown model when no prefix match exists', () => {
|
|
489
434
|
const trace = {
|
|
490
435
|
kind: 'workflow',
|
|
491
436
|
name: 'test',
|
|
@@ -498,7 +443,7 @@ describe('calculateCost', () => {
|
|
|
498
443
|
kind: 'llm',
|
|
499
444
|
name: 'gen',
|
|
500
445
|
input: { loadedPrompt: { config: { model: 'totally-unknown-model' } } },
|
|
501
|
-
|
|
446
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
502
447
|
}]
|
|
503
448
|
}]
|
|
504
449
|
};
|
|
@@ -520,160 +465,13 @@ describe('calculateCost', () => {
|
|
|
520
465
|
kind: 'llm',
|
|
521
466
|
name: 'gen',
|
|
522
467
|
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
523
|
-
|
|
468
|
+
output: { usage: { inputTokens: 1000, outputTokens: 500 } }
|
|
524
469
|
}]
|
|
525
470
|
}]
|
|
526
471
|
};
|
|
527
472
|
const report = calculateCost(trace, testConfig, 'test.json');
|
|
528
473
|
expect(report.llmCalls[0].warning).toBeUndefined();
|
|
529
474
|
});
|
|
530
|
-
it('prefers attributes.cost.total over yaml-computed LLM cost when present', () => {
|
|
531
|
-
const trace = {
|
|
532
|
-
kind: 'workflow',
|
|
533
|
-
name: 'test',
|
|
534
|
-
children: [{
|
|
535
|
-
id: 'step-1',
|
|
536
|
-
kind: 'step',
|
|
537
|
-
name: 'test#gen',
|
|
538
|
-
children: [{
|
|
539
|
-
id: 'llm-1',
|
|
540
|
-
kind: 'llm',
|
|
541
|
-
name: 'gen',
|
|
542
|
-
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
543
|
-
attributes: {
|
|
544
|
-
token_usage: { inputTokens: 1000, outputTokens: 500 },
|
|
545
|
-
// Authoritative: this is what the provider reported, not what yaml infers.
|
|
546
|
-
cost: { total: 0.99 }
|
|
547
|
-
}
|
|
548
|
-
}]
|
|
549
|
-
}]
|
|
550
|
-
};
|
|
551
|
-
const report = calculateCost(trace, testConfig, 'test.json');
|
|
552
|
-
expect(report.llmCalls[0].cost).toBe(0.99);
|
|
553
|
-
expect(report.llmTotalCost).toBe(0.99);
|
|
554
|
-
});
|
|
555
|
-
it('still surfaces an attribute LLM cost for unknown models without warning', () => {
|
|
556
|
-
const trace = {
|
|
557
|
-
kind: 'workflow',
|
|
558
|
-
name: 'test',
|
|
559
|
-
children: [{
|
|
560
|
-
id: 'step-1',
|
|
561
|
-
kind: 'step',
|
|
562
|
-
name: 'test#gen',
|
|
563
|
-
children: [{
|
|
564
|
-
id: 'llm-1',
|
|
565
|
-
kind: 'llm',
|
|
566
|
-
name: 'gen',
|
|
567
|
-
input: { loadedPrompt: { config: { model: 'brand-new-model' } } },
|
|
568
|
-
attributes: {
|
|
569
|
-
token_usage: { inputTokens: 100, outputTokens: 20 },
|
|
570
|
-
cost: { total: 0.0123 }
|
|
571
|
-
}
|
|
572
|
-
}]
|
|
573
|
-
}]
|
|
574
|
-
};
|
|
575
|
-
const report = calculateCost(trace, testConfig, 'test.json');
|
|
576
|
-
expect(report.llmTotalCost).toBeCloseTo(0.0123, 10);
|
|
577
|
-
expect(report.unknownModels).toEqual([]);
|
|
578
|
-
});
|
|
579
|
-
it('includes HTTP request cost from attributes.cost.total for unclassified hosts', () => {
|
|
580
|
-
// gx-scraper-style: not declared in yaml services, but addRequestCost ran
|
|
581
|
-
// and put $0.50 on the HTTP node's attributes.cost.total.
|
|
582
|
-
const trace = {
|
|
583
|
-
kind: 'workflow',
|
|
584
|
-
name: 'scraper_workflow',
|
|
585
|
-
startedAt: 1700000000000,
|
|
586
|
-
endedAt: 1700000100000,
|
|
587
|
-
children: [{
|
|
588
|
-
id: 'step-1',
|
|
589
|
-
kind: 'step',
|
|
590
|
-
name: 'scraper_workflow#scrape',
|
|
591
|
-
children: [{
|
|
592
|
-
id: 'http-1',
|
|
593
|
-
kind: 'http',
|
|
594
|
-
name: 'scrape_request',
|
|
595
|
-
input: { url: 'https://api.gx-scraper.test/v1/scrape', method: 'POST' },
|
|
596
|
-
output: { status: 200, body: {} },
|
|
597
|
-
attributes: { cost: { total: 0.5 } }
|
|
598
|
-
}]
|
|
599
|
-
}]
|
|
600
|
-
};
|
|
601
|
-
const report = calculateCost(trace, testConfig, 'test.json');
|
|
602
|
-
expect(report.serviceTotalCost).toBe(0.5);
|
|
603
|
-
expect(report.totalCost).toBe(0.5);
|
|
604
|
-
expect(report.services).toHaveLength(1);
|
|
605
|
-
expect(report.services[0].calls[0].cost).toBe(0.5);
|
|
606
|
-
});
|
|
607
|
-
it('prefers attributes.cost.total over yaml service classifier for classified HTTP nodes', () => {
|
|
608
|
-
// Exa with both a yaml `response_cost` rule AND an attribute cost — the
|
|
609
|
-
// attribute is authoritative.
|
|
610
|
-
const trace = {
|
|
611
|
-
kind: 'workflow',
|
|
612
|
-
name: 'test_workflow',
|
|
613
|
-
children: [{
|
|
614
|
-
id: 'step-exa',
|
|
615
|
-
kind: 'step',
|
|
616
|
-
name: 'test_workflow#search',
|
|
617
|
-
children: [{
|
|
618
|
-
id: 'http-exa',
|
|
619
|
-
kind: 'http',
|
|
620
|
-
name: 'exa_request',
|
|
621
|
-
input: { url: 'https://api.exa.ai/research', method: 'POST' },
|
|
622
|
-
output: {
|
|
623
|
-
status: 200,
|
|
624
|
-
body: { model: 'exa-research', costDollars: { total: 0.15, numSearches: 1, numPages: 5 } }
|
|
625
|
-
},
|
|
626
|
-
attributes: { cost: { total: 0.22 } }
|
|
627
|
-
}]
|
|
628
|
-
}]
|
|
629
|
-
};
|
|
630
|
-
const report = calculateCost(trace, testConfig, 'test.json');
|
|
631
|
-
expect(report.services).toHaveLength(1);
|
|
632
|
-
expect(report.services[0].serviceName).toBe('exa');
|
|
633
|
-
expect(report.services[0].totalCost).toBe(0.22);
|
|
634
|
-
});
|
|
635
|
-
it('combines LLM and HTTP attribute costs into a single trace total', () => {
|
|
636
|
-
const trace = {
|
|
637
|
-
kind: 'workflow',
|
|
638
|
-
name: 'mixed_workflow',
|
|
639
|
-
startedAt: 1700000000000,
|
|
640
|
-
endedAt: 1700000100000,
|
|
641
|
-
children: [
|
|
642
|
-
{
|
|
643
|
-
id: 'step-llm',
|
|
644
|
-
kind: 'step',
|
|
645
|
-
name: 'mixed_workflow#draft',
|
|
646
|
-
children: [{
|
|
647
|
-
id: 'llm-1',
|
|
648
|
-
kind: 'llm',
|
|
649
|
-
name: 'draft',
|
|
650
|
-
input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
|
|
651
|
-
attributes: {
|
|
652
|
-
token_usage: { inputTokens: 100, outputTokens: 50 },
|
|
653
|
-
cost: { total: 0.01 }
|
|
654
|
-
}
|
|
655
|
-
}]
|
|
656
|
-
},
|
|
657
|
-
{
|
|
658
|
-
id: 'step-http',
|
|
659
|
-
kind: 'step',
|
|
660
|
-
name: 'mixed_workflow#scrape',
|
|
661
|
-
children: [{
|
|
662
|
-
id: 'http-1',
|
|
663
|
-
kind: 'http',
|
|
664
|
-
name: 'scrape',
|
|
665
|
-
input: { url: 'https://api.gx-scraper.test/v1/scrape', method: 'POST' },
|
|
666
|
-
output: { status: 200, body: {} },
|
|
667
|
-
attributes: { cost: { total: 0.5 } }
|
|
668
|
-
}]
|
|
669
|
-
}
|
|
670
|
-
]
|
|
671
|
-
};
|
|
672
|
-
const report = calculateCost(trace, testConfig, 'test.json');
|
|
673
|
-
expect(report.llmTotalCost).toBeCloseTo(0.01, 10);
|
|
674
|
-
expect(report.serviceTotalCost).toBeCloseTo(0.5, 10);
|
|
675
|
-
expect(report.totalCost).toBeCloseTo(0.51, 10);
|
|
676
|
-
});
|
|
677
475
|
});
|
|
678
476
|
describe('loadPricingConfig', () => {
|
|
679
477
|
beforeEach(() => {
|
package/dist/types/cost.d.ts
CHANGED
|
@@ -8,18 +8,6 @@ export interface TokenUsage {
|
|
|
8
8
|
outputTokens?: number;
|
|
9
9
|
cachedInputTokens?: number;
|
|
10
10
|
reasoningTokens?: number;
|
|
11
|
-
totalTokens?: number;
|
|
12
|
-
}
|
|
13
|
-
export interface TraceCostAttribute {
|
|
14
|
-
total: number;
|
|
15
|
-
components?: Array<{
|
|
16
|
-
name: string;
|
|
17
|
-
value: number;
|
|
18
|
-
}>;
|
|
19
|
-
}
|
|
20
|
-
export interface TraceAttributes {
|
|
21
|
-
cost?: TraceCostAttribute;
|
|
22
|
-
token_usage?: TokenUsage;
|
|
23
11
|
}
|
|
24
12
|
export interface TraceNode {
|
|
25
13
|
id?: string;
|
|
@@ -29,15 +17,15 @@ export interface TraceNode {
|
|
|
29
17
|
endedAt?: number;
|
|
30
18
|
children?: TraceNode[];
|
|
31
19
|
input?: Record<string, unknown>;
|
|
32
|
-
output?: Record<string, unknown
|
|
33
|
-
|
|
20
|
+
output?: Record<string, unknown> & {
|
|
21
|
+
usage?: TokenUsage;
|
|
22
|
+
};
|
|
34
23
|
}
|
|
35
24
|
export interface LLMCall {
|
|
36
25
|
stepName: string;
|
|
37
26
|
llmName: string;
|
|
38
27
|
model: string;
|
|
39
28
|
usage: TokenUsage;
|
|
40
|
-
attributeCost?: number;
|
|
41
29
|
}
|
|
42
30
|
export interface HTTPCall {
|
|
43
31
|
stepName: string;
|
|
@@ -46,7 +34,6 @@ export interface HTTPCall {
|
|
|
46
34
|
input: Record<string, unknown>;
|
|
47
35
|
output: Record<string, unknown>;
|
|
48
36
|
status?: number;
|
|
49
|
-
attributeCost?: number;
|
|
50
37
|
}
|
|
51
38
|
export interface ModelPricing {
|
|
52
39
|
provider: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@outputai/cli",
|
|
3
|
-
"version": "0.4.1-dev.
|
|
3
|
+
"version": "0.4.1-dev.c0b98d8.0",
|
|
4
4
|
"description": "CLI for Output.ai workflow generation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
"semver": "7.7.4",
|
|
37
37
|
"undici": "8.1.0",
|
|
38
38
|
"yaml": "^2.8.3",
|
|
39
|
-
"@outputai/credentials": "0.4.1-dev.
|
|
40
|
-
"@outputai/evals": "0.4.1-dev.
|
|
41
|
-
"@outputai/llm": "0.4.1-dev.
|
|
39
|
+
"@outputai/credentials": "0.4.1-dev.c0b98d8.0",
|
|
40
|
+
"@outputai/evals": "0.4.1-dev.c0b98d8.0",
|
|
41
|
+
"@outputai/llm": "0.4.1-dev.c0b98d8.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@types/cli-progress": "3.11.6",
|