@outputai/llm 0.1.13-next.f537949.0 → 0.2.1-dev.82acd68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/agent.js +31 -27
- package/src/agent.spec.js +47 -13
- package/src/ai_sdk.js +12 -8
- package/src/ai_sdk.spec.js +83 -216
- package/src/cost/index.js +9 -40
- package/src/cost/index.spec.js +42 -20
- package/src/index.d.ts +57 -13
- package/src/index.js +1 -1
- package/src/utils/message.spec.js +29 -0
- package/src/utils/response_wrappers.js +70 -0
- package/src/utils/response_wrappers.spec.js +172 -0
- package/src/{source_extraction.js → utils/source_extraction.js} +14 -0
- package/src/{source_extraction.spec.js → utils/source_extraction.spec.js} +26 -1
- package/src/utils/trace.js +18 -0
- package/src/utils/trace.spec.js +95 -0
- package/src/response_utils.js +0 -21
- package/src/trace_utils.js +0 -30
- /package/src/{message_utils.js → utils/message.js} +0 -0
package/src/ai_sdk.spec.js
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
2
|
|
|
3
|
-
const
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
vi.
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
3
|
+
const traceMocks = vi.hoisted( () => ( {
|
|
4
|
+
startTrace: vi.fn( () => 'trace-id' ),
|
|
5
|
+
endTraceWithError: vi.fn()
|
|
6
|
+
} ) );
|
|
7
|
+
|
|
8
|
+
const wrapMocks = vi.hoisted( () => ( {
|
|
9
|
+
wrapTextResponse: vi.fn(),
|
|
10
|
+
wrapStreamOnFinishResponse: vi.fn()
|
|
11
|
+
} ) );
|
|
12
|
+
|
|
13
|
+
vi.mock( './utils/trace.js', () => ( {
|
|
14
|
+
startTrace: ( ...args ) => traceMocks.startTrace( ...args ),
|
|
15
|
+
endTraceWithError: ( ...args ) => traceMocks.endTraceWithError( ...args )
|
|
16
|
+
} ) );
|
|
17
|
+
|
|
18
|
+
vi.mock( './utils/response_wrappers.js', () => ( {
|
|
19
|
+
wrapTextResponse: ( ...args ) => wrapMocks.wrapTextResponse( ...args ),
|
|
20
|
+
wrapStreamOnFinishResponse: ( ...args ) => wrapMocks.wrapStreamOnFinishResponse( ...args )
|
|
21
|
+
} ) );
|
|
13
22
|
|
|
14
23
|
const loadModelImpl = vi.fn();
|
|
15
24
|
const loadToolsImpl = vi.fn();
|
|
@@ -48,16 +57,6 @@ vi.mock( './skill.js', async importOriginal => {
|
|
|
48
57
|
};
|
|
49
58
|
} );
|
|
50
59
|
|
|
51
|
-
const extractSourcesFromStepsImpl = vi.fn().mockReturnValue( [] );
|
|
52
|
-
vi.mock( './source_extraction.js', () => ( {
|
|
53
|
-
extractSourcesFromSteps: ( ...args ) => extractSourcesFromStepsImpl( ...args )
|
|
54
|
-
} ) );
|
|
55
|
-
|
|
56
|
-
const calculateLLMCallCostImpl = vi.fn();
|
|
57
|
-
vi.mock( './cost/index.js', () => ( {
|
|
58
|
-
calculateLLMCallCost: ( ...args ) => calculateLLMCallCostImpl( ...args )
|
|
59
|
-
} ) );
|
|
60
|
-
|
|
61
60
|
const importSut = async () => import( './ai_sdk.js' );
|
|
62
61
|
|
|
63
62
|
const basePrompt = {
|
|
@@ -70,27 +69,45 @@ const basePrompt = {
|
|
|
70
69
|
messages: [ { role: 'user', content: 'Hi' } ]
|
|
71
70
|
};
|
|
72
71
|
|
|
73
|
-
|
|
72
|
+
/** Mutable payload from `AI.generateText` — identity checks prove `generateText` returns `wrapTextResponse(...)` without substitution. */
|
|
73
|
+
const generateTextAiFixture = {
|
|
74
|
+
response: {
|
|
75
|
+
text: 'TEXT',
|
|
76
|
+
sources: [],
|
|
77
|
+
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
|
78
|
+
totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
|
79
|
+
finishReason: 'stop'
|
|
80
|
+
}
|
|
81
|
+
};
|
|
74
82
|
|
|
75
83
|
beforeEach( () => {
|
|
76
|
-
emitEventSpy.mockReset();
|
|
77
84
|
loadModelImpl.mockReset().mockReturnValue( 'MODEL' );
|
|
78
85
|
loadPromptImpl.mockReset().mockReturnValue( { ...basePrompt, messages: [ ...basePrompt.messages ] } );
|
|
79
|
-
extractSourcesFromStepsImpl.mockReset().mockReturnValue( [] );
|
|
80
|
-
calculateLLMCallCostImpl.mockReset().mockResolvedValue( cost );
|
|
81
86
|
aiFns.tool.mockReset().mockImplementation( def => def );
|
|
82
87
|
aiFns.stepCountIs.mockReset().mockImplementation( n => ( { type: 'stepCount', count: n } ) );
|
|
83
88
|
loadPromptSkillsImpl.mockReset().mockReturnValue( [] );
|
|
84
89
|
loadColocatedSkillsImpl.mockReset().mockReturnValue( [] );
|
|
85
90
|
|
|
91
|
+
traceMocks.startTrace.mockReset().mockReturnValue( 'trace-id' );
|
|
92
|
+
traceMocks.endTraceWithError.mockReset();
|
|
93
|
+
|
|
94
|
+
wrapMocks.wrapTextResponse.mockReset().mockImplementation( async ( { response } ) => response );
|
|
95
|
+
|
|
96
|
+
wrapMocks.wrapStreamOnFinishResponse.mockReset().mockImplementation( ( { onFinish } ) => ( {
|
|
97
|
+
async onFinish( response ) {
|
|
98
|
+
onFinish?.( response );
|
|
99
|
+
}
|
|
100
|
+
} ) );
|
|
101
|
+
|
|
86
102
|
const defaultUsage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
87
|
-
|
|
103
|
+
generateTextAiFixture.response = {
|
|
88
104
|
text: 'TEXT',
|
|
89
105
|
sources: [],
|
|
90
106
|
usage: defaultUsage,
|
|
91
107
|
totalUsage: defaultUsage,
|
|
92
108
|
finishReason: 'stop'
|
|
93
|
-
}
|
|
109
|
+
};
|
|
110
|
+
aiFns.generateText.mockReset().mockResolvedValue( generateTextAiFixture.response );
|
|
94
111
|
|
|
95
112
|
aiFns.streamText.mockReset().mockReturnValue( {
|
|
96
113
|
textStream: 'MOCK_TEXT_STREAM',
|
|
@@ -108,27 +125,23 @@ afterEach( async () => {
|
|
|
108
125
|
} );
|
|
109
126
|
|
|
110
127
|
describe( 'ai_sdk', () => {
|
|
111
|
-
it( 'generateText: validates,
|
|
128
|
+
it( 'generateText: validates, delegates trace/wrap utils, calls AI and returns wrapTextResponse output', async () => {
|
|
112
129
|
const { generateText } = await importSut();
|
|
130
|
+
const defaultUsage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
113
131
|
const result = await generateText( { prompt: 'test_prompt@v1' } );
|
|
114
132
|
|
|
115
133
|
expect( validators.validateGenerateTextArgs ).toHaveBeenCalledWith( { prompt: 'test_prompt@v1' } );
|
|
116
134
|
expect( loadPromptImpl ).toHaveBeenCalledWith( 'test_prompt@v1', undefined, undefined );
|
|
117
|
-
expect(
|
|
118
|
-
expect(
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
);
|
|
122
|
-
expect(
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
} );
|
|
126
|
-
const defaultUsage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
127
|
-
expect( emitEventSpy ).toHaveBeenCalledTimes( 1 );
|
|
128
|
-
expect( emitEventSpy ).toHaveBeenCalledWith( 'llm:call_cost', {
|
|
135
|
+
expect( traceMocks.startTrace ).toHaveBeenCalledTimes( 1 );
|
|
136
|
+
expect( traceMocks.startTrace ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
137
|
+
name: 'generateText',
|
|
138
|
+
prompt: 'test_prompt@v1'
|
|
139
|
+
} ) );
|
|
140
|
+
expect( wrapMocks.wrapTextResponse ).toHaveBeenCalledTimes( 1 );
|
|
141
|
+
expect( wrapMocks.wrapTextResponse ).toHaveBeenCalledWith( {
|
|
142
|
+
traceId: 'trace-id',
|
|
129
143
|
modelId: basePrompt.config.model,
|
|
130
|
-
|
|
131
|
-
usage: defaultUsage
|
|
144
|
+
response: generateTextAiFixture.response
|
|
132
145
|
} );
|
|
133
146
|
|
|
134
147
|
expect( loadModelImpl ).toHaveBeenCalledWith( basePrompt );
|
|
@@ -138,10 +151,8 @@ describe( 'ai_sdk', () => {
|
|
|
138
151
|
temperature: 0.3,
|
|
139
152
|
providerOptions: basePrompt.config.providerOptions
|
|
140
153
|
} );
|
|
141
|
-
expect( result
|
|
142
|
-
expect( result.
|
|
143
|
-
expect( result.usage ).toEqual( { inputTokens: 10, outputTokens: 5, totalTokens: 15 } );
|
|
144
|
-
expect( result.finishReason ).toBe( 'stop' );
|
|
154
|
+
expect( result ).toBe( generateTextAiFixture.response );
|
|
155
|
+
expect( result.totalUsage ).toEqual( defaultUsage );
|
|
145
156
|
} );
|
|
146
157
|
|
|
147
158
|
it( 'generateText: passes provider-specific options to AI SDK', async () => {
|
|
@@ -221,26 +232,16 @@ describe( 'ai_sdk', () => {
|
|
|
221
232
|
expect( result.response ).toEqual( { id: 'req_123', modelId: 'gpt-4o-2024-05-13' } );
|
|
222
233
|
} );
|
|
223
234
|
|
|
224
|
-
it( 'generateText: includes unified result field that matches text', async () => {
|
|
225
|
-
const { generateText } = await importSut();
|
|
226
|
-
const response = await generateText( { prompt: 'test_prompt@v1' } );
|
|
227
|
-
|
|
228
|
-
expect( response.result ).toBe( 'TEXT' );
|
|
229
|
-
expect( response.result ).toBe( response.text );
|
|
230
|
-
} );
|
|
231
|
-
|
|
232
235
|
it( 'generateText: traces error and rethrows when AI SDK fails', async () => {
|
|
233
236
|
const error = new Error( 'API rate limit exceeded' );
|
|
234
237
|
aiFns.generateText.mockRejectedValueOnce( error );
|
|
235
238
|
const { generateText } = await importSut();
|
|
236
239
|
|
|
237
240
|
await expect( generateText( { prompt: 'test_prompt@v1' } ) ).rejects.toThrow( 'API rate limit exceeded' );
|
|
238
|
-
expect(
|
|
239
|
-
expect.objectContaining( { details: error } )
|
|
240
|
-
);
|
|
241
|
+
expect( traceMocks.endTraceWithError ).toHaveBeenCalledWith( { traceId: 'trace-id', error } );
|
|
241
242
|
} );
|
|
242
243
|
|
|
243
|
-
it( 'generateText:
|
|
244
|
+
it( 'generateText: passes the AI response object through to wrapTextResponse and returns it', async () => {
|
|
244
245
|
const responseWithGetter = {
|
|
245
246
|
_internalText: 'TEXT_FROM_GETTER',
|
|
246
247
|
get text() {
|
|
@@ -254,10 +255,15 @@ describe( 'ai_sdk', () => {
|
|
|
254
255
|
aiFns.generateText.mockResolvedValueOnce( responseWithGetter );
|
|
255
256
|
|
|
256
257
|
const { generateText } = await importSut();
|
|
257
|
-
const
|
|
258
|
+
const out = await generateText( { prompt: 'test_prompt@v1' } );
|
|
258
259
|
|
|
259
|
-
expect(
|
|
260
|
-
|
|
260
|
+
expect( wrapMocks.wrapTextResponse ).toHaveBeenCalledWith( {
|
|
261
|
+
traceId: 'trace-id',
|
|
262
|
+
modelId: basePrompt.config.model,
|
|
263
|
+
response: responseWithGetter
|
|
264
|
+
} );
|
|
265
|
+
expect( out ).toBe( responseWithGetter );
|
|
266
|
+
expect( out.text ).toBe( 'TEXT_FROM_GETTER' );
|
|
261
267
|
} );
|
|
262
268
|
|
|
263
269
|
it( 'generateText: passes through AI SDK options like tools and maxRetries', async () => {
|
|
@@ -318,13 +324,11 @@ describe( 'ai_sdk', () => {
|
|
|
318
324
|
);
|
|
319
325
|
} );
|
|
320
326
|
|
|
321
|
-
it( 'generateText:
|
|
327
|
+
it( 'generateText: does not add structured output fields the AI response lacks', async () => {
|
|
322
328
|
const { generateText } = await importSut();
|
|
323
329
|
const result = await generateText( { prompt: 'test_prompt@v1' } );
|
|
324
330
|
|
|
325
331
|
expect( result.object ).toBeUndefined();
|
|
326
|
-
expect( result.text ).toBe( 'TEXT' );
|
|
327
|
-
expect( result.result ).toBe( 'TEXT' );
|
|
328
332
|
} );
|
|
329
333
|
|
|
330
334
|
it( 'generateText: passes through unknown future options for forward compatibility', async () => {
|
|
@@ -344,13 +348,13 @@ describe( 'ai_sdk', () => {
|
|
|
344
348
|
);
|
|
345
349
|
} );
|
|
346
350
|
|
|
347
|
-
it( 'streamText: validates,
|
|
351
|
+
it( 'streamText: validates, delegates trace/wrap utils, calls AI streamText and returns stream result', async () => {
|
|
348
352
|
const { streamText } = await importSut();
|
|
349
353
|
const result = streamText( { prompt: 'test_prompt@v1' } );
|
|
350
354
|
|
|
351
355
|
expect( validators.validateStreamTextArgs ).toHaveBeenCalledWith( { prompt: 'test_prompt@v1' } );
|
|
352
356
|
expect( loadPromptImpl ).toHaveBeenCalledWith( 'test_prompt@v1', undefined );
|
|
353
|
-
expect(
|
|
357
|
+
expect( traceMocks.startTrace ).toHaveBeenCalledTimes( 1 );
|
|
354
358
|
|
|
355
359
|
expect( loadModelImpl ).toHaveBeenCalledWith( basePrompt );
|
|
356
360
|
expect( aiFns.streamText ).toHaveBeenCalledWith(
|
|
@@ -367,12 +371,18 @@ describe( 'ai_sdk', () => {
|
|
|
367
371
|
expect( result.fullStream ).toBe( 'MOCK_FULL_STREAM' );
|
|
368
372
|
} );
|
|
369
373
|
|
|
370
|
-
it( 'streamText:
|
|
374
|
+
it( 'streamText: forwards stream onFinish through wrapStreamOnFinishResponse to the user callback', async () => {
|
|
371
375
|
const { streamText } = await importSut();
|
|
372
376
|
const userOnFinish = vi.fn();
|
|
373
377
|
|
|
374
378
|
streamText( { prompt: 'test_prompt@v1', onFinish: userOnFinish } );
|
|
375
379
|
|
|
380
|
+
expect( wrapMocks.wrapStreamOnFinishResponse ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
381
|
+
traceId: 'trace-id',
|
|
382
|
+
modelId: basePrompt.config.model,
|
|
383
|
+
onFinish: userOnFinish
|
|
384
|
+
} ) );
|
|
385
|
+
|
|
376
386
|
const callArgs = aiFns.streamText.mock.calls[0][0];
|
|
377
387
|
const usage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
378
388
|
const finishEvent = {
|
|
@@ -384,22 +394,7 @@ describe( 'ai_sdk', () => {
|
|
|
384
394
|
};
|
|
385
395
|
await callArgs.onFinish( finishEvent );
|
|
386
396
|
|
|
387
|
-
expect(
|
|
388
|
-
expect( emitEventSpy ).toHaveBeenCalledWith( 'llm:call_cost', {
|
|
389
|
-
modelId: basePrompt.config.model,
|
|
390
|
-
cost,
|
|
391
|
-
usage
|
|
392
|
-
} );
|
|
393
|
-
expect( tracingSpies.addEventEnd ).toHaveBeenCalledWith(
|
|
394
|
-
expect.objectContaining( {
|
|
395
|
-
details: {
|
|
396
|
-
result: 'STREAMED_TEXT',
|
|
397
|
-
usage,
|
|
398
|
-
cost,
|
|
399
|
-
providerMetadata: finishEvent.providerMetadata
|
|
400
|
-
}
|
|
401
|
-
} )
|
|
402
|
-
);
|
|
397
|
+
expect( userOnFinish ).toHaveBeenCalledTimes( 1 );
|
|
403
398
|
expect( userOnFinish ).toHaveBeenCalledWith( finishEvent );
|
|
404
399
|
} );
|
|
405
400
|
|
|
@@ -413,9 +408,7 @@ describe( 'ai_sdk', () => {
|
|
|
413
408
|
const error = new Error( 'Stream failed' );
|
|
414
409
|
callArgs.onError( { error } );
|
|
415
410
|
|
|
416
|
-
expect(
|
|
417
|
-
expect.objectContaining( { details: error } )
|
|
418
|
-
);
|
|
411
|
+
expect( traceMocks.endTraceWithError ).toHaveBeenCalledWith( { traceId: 'trace-id', error } );
|
|
419
412
|
expect( userOnError ).toHaveBeenCalledWith( { error } );
|
|
420
413
|
} );
|
|
421
414
|
|
|
@@ -433,11 +426,6 @@ describe( 'ai_sdk', () => {
|
|
|
433
426
|
finishReason: 'stop'
|
|
434
427
|
};
|
|
435
428
|
await expect( callArgs.onFinish( finishEvent ) ).resolves.toBeUndefined();
|
|
436
|
-
expect( emitEventSpy ).toHaveBeenCalledWith( 'llm:call_cost', {
|
|
437
|
-
modelId: basePrompt.config.model,
|
|
438
|
-
cost,
|
|
439
|
-
usage
|
|
440
|
-
} );
|
|
441
429
|
expect( () => callArgs.onError( { error: new Error( 'fail' ) } ) ).not.toThrow();
|
|
442
430
|
} );
|
|
443
431
|
|
|
@@ -499,15 +487,11 @@ describe( 'ai_sdk', () => {
|
|
|
499
487
|
|
|
500
488
|
streamText( { prompt: 'test_prompt@v1', variables: vars } );
|
|
501
489
|
|
|
502
|
-
expect(
|
|
503
|
-
kind: 'llm',
|
|
490
|
+
expect( traceMocks.startTrace ).toHaveBeenCalledWith( {
|
|
504
491
|
name: 'streamText',
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
variables: vars,
|
|
509
|
-
loadedPrompt: basePrompt
|
|
510
|
-
}
|
|
492
|
+
prompt: 'test_prompt@v1',
|
|
493
|
+
variables: vars,
|
|
494
|
+
loadedPrompt: basePrompt
|
|
511
495
|
} );
|
|
512
496
|
} );
|
|
513
497
|
|
|
@@ -519,9 +503,7 @@ describe( 'ai_sdk', () => {
|
|
|
519
503
|
const { streamText } = await importSut();
|
|
520
504
|
|
|
521
505
|
expect( () => streamText( { prompt: 'test_prompt@v1' } ) ).toThrow( syncError );
|
|
522
|
-
expect(
|
|
523
|
-
expect.objectContaining( { details: syncError } )
|
|
524
|
-
);
|
|
506
|
+
expect( traceMocks.endTraceWithError ).toHaveBeenCalledWith( { traceId: 'trace-id', error: syncError } );
|
|
525
507
|
} );
|
|
526
508
|
|
|
527
509
|
it( 'streamText: passes variables to prompt loader', async () => {
|
|
@@ -533,95 +515,6 @@ describe( 'ai_sdk', () => {
|
|
|
533
515
|
expect( loadPromptImpl ).toHaveBeenCalledWith( 'test_prompt@v1', vars );
|
|
534
516
|
} );
|
|
535
517
|
|
|
536
|
-
it( 'generateText: merges tool-extracted sources into response.sources', async () => {
|
|
537
|
-
const extracted = [
|
|
538
|
-
{ type: 'source', sourceType: 'url', id: 'abc123', url: 'https://tool.com/1', title: 'Tool 1' },
|
|
539
|
-
{ type: 'source', sourceType: 'url', id: 'def456', url: 'https://tool.com/2', title: 'Tool 2' }
|
|
540
|
-
];
|
|
541
|
-
extractSourcesFromStepsImpl.mockReturnValue( extracted );
|
|
542
|
-
|
|
543
|
-
const usageTools = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
544
|
-
aiFns.generateText.mockResolvedValueOnce( {
|
|
545
|
-
text: 'answer',
|
|
546
|
-
sources: [],
|
|
547
|
-
steps: [ { toolResults: [] } ],
|
|
548
|
-
usage: usageTools,
|
|
549
|
-
totalUsage: usageTools,
|
|
550
|
-
finishReason: 'stop'
|
|
551
|
-
} );
|
|
552
|
-
|
|
553
|
-
const { generateText } = await importSut();
|
|
554
|
-
const result = await generateText( { prompt: 'test_prompt@v1' } );
|
|
555
|
-
|
|
556
|
-
expect( result.sources ).toEqual( extracted );
|
|
557
|
-
} );
|
|
558
|
-
|
|
559
|
-
it( 'generateText: deduplicates extracted sources against native sources', async () => {
|
|
560
|
-
const nativeSources = [
|
|
561
|
-
{ type: 'source', sourceType: 'url', id: 'native1', url: 'https://shared.com', title: 'Native' }
|
|
562
|
-
];
|
|
563
|
-
const extracted = [
|
|
564
|
-
{ type: 'source', sourceType: 'url', id: 'ext1', url: 'https://shared.com', title: 'Extracted' },
|
|
565
|
-
{ type: 'source', sourceType: 'url', id: 'ext2', url: 'https://unique.com', title: 'Unique' }
|
|
566
|
-
];
|
|
567
|
-
extractSourcesFromStepsImpl.mockReturnValue( extracted );
|
|
568
|
-
|
|
569
|
-
const usageDedup = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
570
|
-
aiFns.generateText.mockResolvedValueOnce( {
|
|
571
|
-
text: 'answer',
|
|
572
|
-
sources: nativeSources,
|
|
573
|
-
steps: [ { toolResults: [] } ],
|
|
574
|
-
usage: usageDedup,
|
|
575
|
-
totalUsage: usageDedup,
|
|
576
|
-
finishReason: 'stop'
|
|
577
|
-
} );
|
|
578
|
-
|
|
579
|
-
const { generateText } = await importSut();
|
|
580
|
-
const result = await generateText( { prompt: 'test_prompt@v1' } );
|
|
581
|
-
|
|
582
|
-
expect( result.sources ).toHaveLength( 2 );
|
|
583
|
-
expect( result.sources[0].url ).toBe( 'https://shared.com' );
|
|
584
|
-
expect( result.sources[0].title ).toBe( 'Native' );
|
|
585
|
-
expect( result.sources[1].url ).toBe( 'https://unique.com' );
|
|
586
|
-
} );
|
|
587
|
-
|
|
588
|
-
it( 'generateText: returns native sources unchanged when no tool sources extracted', async () => {
|
|
589
|
-
const nativeSources = [
|
|
590
|
-
{ type: 'source', sourceType: 'url', id: 'n1', url: 'https://native.com', title: 'Native' }
|
|
591
|
-
];
|
|
592
|
-
extractSourcesFromStepsImpl.mockReturnValue( [] );
|
|
593
|
-
|
|
594
|
-
const usageNative = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
595
|
-
aiFns.generateText.mockResolvedValueOnce( {
|
|
596
|
-
text: 'answer',
|
|
597
|
-
sources: nativeSources,
|
|
598
|
-
usage: usageNative,
|
|
599
|
-
totalUsage: usageNative,
|
|
600
|
-
finishReason: 'stop'
|
|
601
|
-
} );
|
|
602
|
-
|
|
603
|
-
const { generateText } = await importSut();
|
|
604
|
-
const result = await generateText( { prompt: 'test_prompt@v1' } );
|
|
605
|
-
|
|
606
|
-
expect( result.sources ).toEqual( nativeSources );
|
|
607
|
-
} );
|
|
608
|
-
|
|
609
|
-
it( 'generateText: includes costs from cost module in trace details', async () => {
|
|
610
|
-
const customCost = { total: 0.02, components: { input: { value: 0.01 }, output: { value: 0.01 } } };
|
|
611
|
-
calculateLLMCallCostImpl.mockResolvedValueOnce( customCost );
|
|
612
|
-
|
|
613
|
-
const { generateText } = await importSut();
|
|
614
|
-
await generateText( { prompt: 'test_prompt@v1' } );
|
|
615
|
-
|
|
616
|
-
expect( calculateLLMCallCostImpl ).toHaveBeenCalledWith( {
|
|
617
|
-
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
|
618
|
-
modelId: basePrompt.config.model
|
|
619
|
-
} );
|
|
620
|
-
expect( tracingSpies.addEventEnd ).toHaveBeenCalledWith(
|
|
621
|
-
expect.objectContaining( { details: expect.objectContaining( { cost: customCost } ) } )
|
|
622
|
-
);
|
|
623
|
-
} );
|
|
624
|
-
|
|
625
518
|
it( 'generateText: loads frontmatter skills from prompt config using promptFileDir', async () => {
|
|
626
519
|
const frontmatterSkill = { name: 'fm_skill', description: 'FM', instructions: '# FM' };
|
|
627
520
|
loadPromptImpl.mockReturnValue( {
|
|
@@ -805,30 +698,4 @@ describe( 'ai_sdk', () => {
|
|
|
805
698
|
expect.objectContaining( { stopWhen: customStop } )
|
|
806
699
|
);
|
|
807
700
|
} );
|
|
808
|
-
|
|
809
|
-
it( 'generateText: includes sourcesFromTools in trace details', async () => {
|
|
810
|
-
const extracted = [
|
|
811
|
-
{ type: 'source', sourceType: 'url', id: 'abc', url: 'https://t.com', title: 'T' }
|
|
812
|
-
];
|
|
813
|
-
extractSourcesFromStepsImpl.mockReturnValue( extracted );
|
|
814
|
-
|
|
815
|
-
const usageSources = { inputTokens: 10, outputTokens: 5, totalTokens: 15 };
|
|
816
|
-
aiFns.generateText.mockResolvedValueOnce( {
|
|
817
|
-
text: 'TEXT',
|
|
818
|
-
sources: [],
|
|
819
|
-
steps: [],
|
|
820
|
-
usage: usageSources,
|
|
821
|
-
totalUsage: usageSources,
|
|
822
|
-
finishReason: 'stop'
|
|
823
|
-
} );
|
|
824
|
-
|
|
825
|
-
const { generateText } = await importSut();
|
|
826
|
-
await generateText( { prompt: 'test_prompt@v1' } );
|
|
827
|
-
|
|
828
|
-
expect( tracingSpies.addEventEnd ).toHaveBeenCalledWith(
|
|
829
|
-
expect.objectContaining( {
|
|
830
|
-
details: expect.objectContaining( { sourcesFromTools: extracted } )
|
|
831
|
-
} )
|
|
832
|
-
);
|
|
833
|
-
} );
|
|
834
701
|
} );
|
package/src/cost/index.js
CHANGED
|
@@ -4,41 +4,10 @@ import Decimal from 'decimal.js';
|
|
|
4
4
|
const M = 1_000_000;
|
|
5
5
|
const calcCost = ( tokens, ppm ) => Decimal( tokens ?? 0 ).div( M ).mul( ppm ).toNumber();
|
|
6
6
|
|
|
7
|
-
/**
|
|
8
|
-
* Calculates the input cost based on the input value
|
|
9
|
-
*/
|
|
10
|
-
const calculateInput = ( { tokens, cost } ) =>
|
|
11
|
-
!Number.isFinite( cost.input ) ? { value: null, message: 'Missing input cost' } : { value: calcCost( tokens, cost.input ) };
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Calculates the input cost based on the cache_read
|
|
15
|
-
*/
|
|
16
|
-
const calculateCachedInput = ( { tokens, cost } ) =>
|
|
17
|
-
!Number.isFinite( cost.cache_read ) ? { value: null, message: 'Missing cache input cost' } : { value: calcCost( tokens, cost.cache_read ) };
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Calculates the output cost based on the output value
|
|
21
|
-
*/
|
|
22
|
-
const calculateOutput = ( { tokens, cost } ) =>
|
|
23
|
-
!Number.isFinite( cost.output ) ? { value: null, message: 'Missing output' } : { value: calcCost( tokens, cost.output ) };
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Calculates the reasoning cost based on the reasoning token's
|
|
27
|
-
* If there isn't reasoning costs, this means this providers doesn't differentiate reasoning vs output,
|
|
28
|
-
* so don't calculate it as the price is included in output
|
|
29
|
-
*/
|
|
30
|
-
const calculateReasoning = ( { tokens, cost } ) =>
|
|
31
|
-
Number.isFinite( cost.reasoning ) ? { value: calcCost( tokens, cost.reasoning ) } : undefined;
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Calculates the total cost based on the components
|
|
35
|
-
*/
|
|
36
|
-
const calculateTotal = components => Object.values( components ).reduce( ( v, e ) => v.plus( e?.value ? e.value : 0 ), Decimal( 0 ) ).toNumber();
|
|
37
|
-
|
|
38
7
|
/**
|
|
39
8
|
* Calculates the cost of an llm call based on the model and usage.
|
|
40
9
|
* @param {object} args
|
|
41
|
-
* @param {string} args.modelId - Name of the
|
|
10
|
+
* @param {string} args.modelId - Name of the model, provider prefix is optional
|
|
42
11
|
* @param {object} args.usage - Usage, as returned from AI SDK
|
|
43
12
|
* @returns {object} The cost with total value and components
|
|
44
13
|
*/
|
|
@@ -58,14 +27,14 @@ export const calculateLLMCallCost = async ( { modelId, usage } ) => {
|
|
|
58
27
|
|
|
59
28
|
const nonCachedTokens = inputTokens - ( cachedInputTokens ?? 0 );
|
|
60
29
|
|
|
61
|
-
const components =
|
|
62
|
-
input
|
|
63
|
-
|
|
64
|
-
output
|
|
65
|
-
reasoning
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
return { total:
|
|
30
|
+
const components = [
|
|
31
|
+
Number.isFinite( cost.input ) ? { name: 'input_tokens', value: calcCost( nonCachedTokens, cost.input ) } : false,
|
|
32
|
+
Number.isFinite( cost.cache_read ) ? { name: 'input_cached_tokens', value: calcCost( cachedInputTokens, cost.cache_read ) } : false,
|
|
33
|
+
Number.isFinite( cost.output ) ? { name: 'output_tokens', value: calcCost( outputTokens, cost.output ) } : false,
|
|
34
|
+
/* When there aren't reasoning costs, the providers doesn't differentiate reasoning vs output, so the price is included in the output */
|
|
35
|
+
Number.isFinite( cost.reasoning ) ? { name: 'reasoning_tokens', value: calcCost( reasoningTokens, cost.reasoning ) } : false
|
|
36
|
+
].filter( v => !!v );
|
|
37
|
+
return { total: components.reduce( ( v, e ) => v.plus( e.value ), Decimal( 0 ) ).toNumber(), components };
|
|
69
38
|
} catch ( error ) {
|
|
70
39
|
console.error( 'Error calculating LLM call costs', error );
|
|
71
40
|
return { total: null, message: `Error calculating LLM call costs: ${error.constructor.name} - ${error.message}` };
|
package/src/cost/index.spec.js
CHANGED
|
@@ -47,10 +47,11 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
47
47
|
} );
|
|
48
48
|
|
|
49
49
|
expect( result.total ).toBe( 7 );
|
|
50
|
-
expect( result.components
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
expect( result.components ).toEqual( [
|
|
51
|
+
{ name: 'input_tokens', value: 2 },
|
|
52
|
+
{ name: 'input_cached_tokens', value: 0 },
|
|
53
|
+
{ name: 'output_tokens', value: 5 }
|
|
54
|
+
] );
|
|
54
55
|
} );
|
|
55
56
|
|
|
56
57
|
it( 'splits input into non-cached and cached at respective rates', async () => {
|
|
@@ -62,13 +63,15 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
62
63
|
usage: { inputTokens: 1_000_000, cachedInputTokens: 500_000, outputTokens: 100_000 }
|
|
63
64
|
} );
|
|
64
65
|
|
|
65
|
-
expect( result.components
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
expect( result.components ).toEqual( [
|
|
67
|
+
{ name: 'input_tokens', value: 2 },
|
|
68
|
+
{ name: 'input_cached_tokens', value: 0.5 },
|
|
69
|
+
{ name: 'output_tokens', value: 1 }
|
|
70
|
+
] );
|
|
68
71
|
expect( result.total ).toBeCloseTo( 3.5 );
|
|
69
72
|
} );
|
|
70
73
|
|
|
71
|
-
it( '
|
|
74
|
+
it( 'omits cached component when model has no cache_read (non-cached rate applies to full input minus cached)', async () => {
|
|
72
75
|
mockFetchModelsPricing.mockResolvedValue( new Map( [ [ 'no-cache', { input: 2, output: 10 } ] ] ) );
|
|
73
76
|
|
|
74
77
|
const result = await calculateLLMCallCost( {
|
|
@@ -76,12 +79,14 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
76
79
|
usage: { inputTokens: 1_000_000, cachedInputTokens: 200_000, outputTokens: 0 }
|
|
77
80
|
} );
|
|
78
81
|
|
|
79
|
-
expect( result.components
|
|
80
|
-
|
|
82
|
+
expect( result.components ).toEqual( [
|
|
83
|
+
{ name: 'input_tokens', value: 1.6 },
|
|
84
|
+
{ name: 'output_tokens', value: 0 }
|
|
85
|
+
] );
|
|
81
86
|
expect( result.total ).toBe( 1.6 );
|
|
82
87
|
} );
|
|
83
88
|
|
|
84
|
-
it( '
|
|
89
|
+
it( 'omits input component when pricing has no input rate', async () => {
|
|
85
90
|
mockFetchModelsPricing.mockResolvedValue( new Map( [ [ 'out-only', { output: 10 } ] ] ) );
|
|
86
91
|
|
|
87
92
|
const result = await calculateLLMCallCost( {
|
|
@@ -90,11 +95,12 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
90
95
|
} );
|
|
91
96
|
|
|
92
97
|
expect( result.total ).toBe( 0.0005 );
|
|
93
|
-
expect( result.components
|
|
94
|
-
|
|
98
|
+
expect( result.components ).toEqual( [
|
|
99
|
+
{ name: 'output_tokens', value: 0.0005 }
|
|
100
|
+
] );
|
|
95
101
|
} );
|
|
96
102
|
|
|
97
|
-
it( '
|
|
103
|
+
it( 'omits output component when pricing has no output rate', async () => {
|
|
98
104
|
mockFetchModelsPricing.mockResolvedValue( new Map( [ [ 'in-only', { input: 1 } ] ] ) );
|
|
99
105
|
|
|
100
106
|
const result = await calculateLLMCallCost( {
|
|
@@ -103,8 +109,9 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
103
109
|
} );
|
|
104
110
|
|
|
105
111
|
expect( result.total ).toBe( 0.0001 );
|
|
106
|
-
expect( result.components
|
|
107
|
-
|
|
112
|
+
expect( result.components ).toEqual( [
|
|
113
|
+
{ name: 'input_tokens', value: 0.0001 }
|
|
114
|
+
] );
|
|
108
115
|
} );
|
|
109
116
|
|
|
110
117
|
it( 'uses reasoning cost when present', async () => {
|
|
@@ -119,7 +126,11 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
119
126
|
} );
|
|
120
127
|
|
|
121
128
|
expect( result.total ).toBeCloseTo( 0.0033 );
|
|
122
|
-
expect( result.components
|
|
129
|
+
expect( result.components ).toEqual( [
|
|
130
|
+
{ name: 'input_tokens', value: 0.0001 },
|
|
131
|
+
{ name: 'output_tokens', value: 0.0002 },
|
|
132
|
+
{ name: 'reasoning_tokens', value: 0.003 }
|
|
133
|
+
] );
|
|
123
134
|
} );
|
|
124
135
|
|
|
125
136
|
it( 'omits reasoning component when reasoning cost missing (included in output)', async () => {
|
|
@@ -131,10 +142,13 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
131
142
|
} );
|
|
132
143
|
|
|
133
144
|
expect( result.total ).toBeCloseTo( 0.0003 );
|
|
134
|
-
expect( result.components
|
|
145
|
+
expect( result.components ).toEqual( [
|
|
146
|
+
{ name: 'input_tokens', value: 0.0001 },
|
|
147
|
+
{ name: 'output_tokens', value: 0.0002 }
|
|
148
|
+
] );
|
|
135
149
|
} );
|
|
136
150
|
|
|
137
|
-
it( '
|
|
151
|
+
it( 'includes reasoning component with zero when reasoningTokens is zero', async () => {
|
|
138
152
|
mockFetchModelsPricing.mockResolvedValue( new Map( [ [
|
|
139
153
|
'full',
|
|
140
154
|
{ input: 2, output: 8, reasoning: 60 }
|
|
@@ -145,7 +159,11 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
145
159
|
usage: { inputTokens: 100, outputTokens: 50, reasoningTokens: 0 }
|
|
146
160
|
} );
|
|
147
161
|
|
|
148
|
-
expect( result.components
|
|
162
|
+
expect( result.components ).toEqual( [
|
|
163
|
+
{ name: 'input_tokens', value: 0.0002 },
|
|
164
|
+
{ name: 'output_tokens', value: 0.0004 },
|
|
165
|
+
{ name: 'reasoning_tokens', value: 0 }
|
|
166
|
+
] );
|
|
149
167
|
expect( result.total ).toBeCloseTo( 0.0006 );
|
|
150
168
|
} );
|
|
151
169
|
|
|
@@ -158,5 +176,9 @@ describe( 'calculateLLMCallCost', () => {
|
|
|
158
176
|
} );
|
|
159
177
|
|
|
160
178
|
expect( result.total ).toBe( 0 );
|
|
179
|
+
expect( result.components ).toEqual( [
|
|
180
|
+
{ name: 'input_tokens', value: 0 },
|
|
181
|
+
{ name: 'output_tokens', value: 0 }
|
|
182
|
+
] );
|
|
161
183
|
} );
|
|
162
184
|
} );
|