@outputai/llm 0.11.0 → 0.11.1-next.2223fa5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agent.js +105 -83
- package/src/agent.spec.js +312 -225
- package/src/ai_provider.js +2 -2
- package/src/ai_sdk_options.js +93 -39
- package/src/ai_sdk_options.spec.js +314 -154
- package/src/consts.js +6 -0
- package/src/generate.js +81 -0
- package/src/generate.spec.js +466 -0
- package/src/index.d.ts +222 -242
- package/src/index.js +3 -4
- package/src/prompt/content.js +70 -0
- package/src/prompt/content.spec.js +208 -0
- package/src/prompt/{escape.js → interpolations.js} +11 -18
- package/src/prompt/interpolations.spec.js +109 -0
- package/src/prompt/loader.full.spec.js +293 -0
- package/src/prompt/loader.js +45 -39
- package/src/prompt/loader.spec.js +186 -289
- package/src/prompt/markup/attributes.js +52 -0
- package/src/prompt/markup/attributes.spec.js +132 -0
- package/src/prompt/markup/nodes.js +71 -0
- package/src/prompt/markup/nodes.spec.js +333 -0
- package/src/prompt/markup/tokenizer.js +3 -0
- package/src/prompt/markup/tokenizer.spec.js +149 -0
- package/src/prompt/markup/tokens.js +26 -0
- package/src/prompt/markup/tokens.spec.js +250 -0
- package/src/prompt/validations.js +98 -67
- package/src/prompt/validations.spec.js +204 -47
- package/src/{prompt/load_content.js → utils/file.js} +12 -15
- package/src/utils/file.spec.js +89 -0
- package/src/utils/models.js +15 -0
- package/src/utils/models.spec.js +119 -0
- package/src/utils/skills.js +74 -0
- package/src/utils/skills.spec.js +168 -0
- package/src/utils/sources.js +48 -0
- package/src/utils/sources.spec.js +122 -0
- package/src/utils/stream.js +22 -0
- package/src/utils/stream.spec.js +55 -0
- package/src/utils/tools.js +47 -0
- package/src/utils/tools.spec.js +167 -0
- package/src/utils/wrap.js +148 -0
- package/src/utils/wrap.spec.js +359 -0
- package/src/validations.js +149 -27
- package/src/validations.spec.js +451 -53
- package/src/ai_model.js +0 -60
- package/src/ai_model.spec.js +0 -259
- package/src/ai_sdk.js +0 -92
- package/src/ai_sdk.spec.js +0 -564
- package/src/prompt/block_options.js +0 -58
- package/src/prompt/block_options.spec.js +0 -71
- package/src/prompt/blocks.js +0 -47
- package/src/prompt/blocks.spec.js +0 -63
- package/src/prompt/escape.spec.js +0 -159
- package/src/prompt/load_content.spec.js +0 -83
- package/src/prompt/loader_validation.spec.js +0 -128
- package/src/prompt/parser.js +0 -16
- package/src/prompt/parser.spec.js +0 -186
- package/src/prompt/prepare_text.js +0 -27
- package/src/prompt/prepare_text.spec.js +0 -141
- package/src/prompt/skill.js +0 -128
- package/src/prompt/skill.spec.js +0 -172
- package/src/utils/message.js +0 -3
- package/src/utils/message.spec.js +0 -29
- package/src/utils/response_wrappers.js +0 -100
- package/src/utils/response_wrappers.spec.js +0 -240
- package/src/utils/source_extraction.js +0 -53
- package/src/utils/source_extraction.spec.js +0 -194
- package/src/utils/trace.js +0 -19
- package/src/utils/trace.spec.js +0 -112
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const getProvider = vi.hoisted( () => vi.fn() );
|
|
4
|
+
|
|
5
|
+
vi.mock( '../ai_provider.js', () => ( {
|
|
6
|
+
getProvider
|
|
7
|
+
} ) );
|
|
8
|
+
|
|
9
|
+
vi.mock( 'ai', () => ( {
|
|
10
|
+
tool: def => def
|
|
11
|
+
} ) );
|
|
12
|
+
|
|
13
|
+
import { buildLoadSkillTool, loadPromptTools } from './tools.js';
|
|
14
|
+
|
|
15
|
+
const skills = [
|
|
16
|
+
{ name: 'writer', description: 'Writes copy', instructions: '# Writer\nDo it.' },
|
|
17
|
+
{ name: 'reviewer', description: 'Reviews drafts', instructions: 'Review carefully.' }
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
describe( 'buildLoadSkillTool', () => {
|
|
21
|
+
it( 'returns a tool that loads instructions by skill name', () => {
|
|
22
|
+
const tool = buildLoadSkillTool( skills );
|
|
23
|
+
|
|
24
|
+
expect( tool.description ).toBe( 'Get detailed instructions for a named skill' );
|
|
25
|
+
expect( tool.execute( { name: 'writer' } ) ).toBe( '# Writer\nDo it.' );
|
|
26
|
+
expect( tool.execute( { name: 'reviewer' } ) ).toBe( 'Review carefully.' );
|
|
27
|
+
} );
|
|
28
|
+
|
|
29
|
+
it( 'returns available skill names when the name is unknown', () => {
|
|
30
|
+
const tool = buildLoadSkillTool( skills );
|
|
31
|
+
|
|
32
|
+
expect( tool.execute( { name: 'missing' } ) ).toBe(
|
|
33
|
+
'Skill "missing" not found. Available: writer, reviewer'
|
|
34
|
+
);
|
|
35
|
+
} );
|
|
36
|
+
|
|
37
|
+
it( 'lists no names when the catalog is empty', () => {
|
|
38
|
+
const tool = buildLoadSkillTool( [] );
|
|
39
|
+
|
|
40
|
+
expect( tool.execute( { name: 'writer' } ) ).toBe( 'Skill "writer" not found. Available: ' );
|
|
41
|
+
} );
|
|
42
|
+
} );
|
|
43
|
+
|
|
44
|
+
describe( 'loadPromptTools', () => {
|
|
45
|
+
it( 'returns null and does not load the provider when no tools are configured', () => {
|
|
46
|
+
const result = loadPromptTools( {
|
|
47
|
+
config: {
|
|
48
|
+
provider: 'vertex',
|
|
49
|
+
model: 'gemini-2.0-flash'
|
|
50
|
+
}
|
|
51
|
+
} );
|
|
52
|
+
|
|
53
|
+
expect( result ).toBeNull();
|
|
54
|
+
expect( getProvider ).not.toHaveBeenCalled();
|
|
55
|
+
} );
|
|
56
|
+
|
|
57
|
+
it( 'returns null and does not load the provider when tools config is empty', () => {
|
|
58
|
+
const result = loadPromptTools( {
|
|
59
|
+
config: {
|
|
60
|
+
provider: 'vertex',
|
|
61
|
+
tools: {}
|
|
62
|
+
}
|
|
63
|
+
} );
|
|
64
|
+
|
|
65
|
+
expect( result ).toBeNull();
|
|
66
|
+
expect( getProvider ).not.toHaveBeenCalled();
|
|
67
|
+
} );
|
|
68
|
+
|
|
69
|
+
it( 'throws when the provider has no tools object', () => {
|
|
70
|
+
getProvider.mockReturnValue( vi.fn() );
|
|
71
|
+
|
|
72
|
+
expect( () => loadPromptTools( {
|
|
73
|
+
name: 'foo',
|
|
74
|
+
config: {
|
|
75
|
+
provider: 'azure',
|
|
76
|
+
tools: {
|
|
77
|
+
webSearch: {}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
} ) ).toThrow( 'Provider "azure" used by prompt "foo" does not support provider-specific tools.' );
|
|
81
|
+
} );
|
|
82
|
+
|
|
83
|
+
it( 'loads a single provider tool with config', () => {
|
|
84
|
+
const googleSearch = vi.fn( config => ( { type: 'googleSearch', config } ) );
|
|
85
|
+
const provider = {
|
|
86
|
+
tools: {
|
|
87
|
+
googleSearch
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
getProvider.mockReturnValue( provider );
|
|
91
|
+
|
|
92
|
+
const result = loadPromptTools( {
|
|
93
|
+
config: {
|
|
94
|
+
provider: 'vertex',
|
|
95
|
+
tools: {
|
|
96
|
+
googleSearch: {
|
|
97
|
+
mode: 'MODE_DYNAMIC',
|
|
98
|
+
dynamicThreshold: 0.8
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
} );
|
|
103
|
+
|
|
104
|
+
expect( getProvider ).toHaveBeenCalledWith( 'vertex' );
|
|
105
|
+
expect( googleSearch ).toHaveBeenCalledWith( {
|
|
106
|
+
mode: 'MODE_DYNAMIC',
|
|
107
|
+
dynamicThreshold: 0.8
|
|
108
|
+
} );
|
|
109
|
+
expect( result ).toEqual( {
|
|
110
|
+
googleSearch: {
|
|
111
|
+
type: 'googleSearch',
|
|
112
|
+
config: {
|
|
113
|
+
mode: 'MODE_DYNAMIC',
|
|
114
|
+
dynamicThreshold: 0.8
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
} );
|
|
118
|
+
} );
|
|
119
|
+
|
|
120
|
+
it( 'loads multiple provider tools', () => {
|
|
121
|
+
const googleSearch = vi.fn( config => ( { type: 'googleSearch', config } ) );
|
|
122
|
+
const urlContext = vi.fn( config => ( { type: 'urlContext', config } ) );
|
|
123
|
+
getProvider.mockReturnValue( {
|
|
124
|
+
tools: {
|
|
125
|
+
googleSearch,
|
|
126
|
+
urlContext
|
|
127
|
+
}
|
|
128
|
+
} );
|
|
129
|
+
|
|
130
|
+
const result = loadPromptTools( {
|
|
131
|
+
config: {
|
|
132
|
+
provider: 'vertex',
|
|
133
|
+
tools: {
|
|
134
|
+
googleSearch: {
|
|
135
|
+
mode: 'MODE_DYNAMIC'
|
|
136
|
+
},
|
|
137
|
+
urlContext: {}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
} );
|
|
141
|
+
|
|
142
|
+
expect( Object.keys( result ) ).toEqual( [ 'googleSearch', 'urlContext' ] );
|
|
143
|
+
expect( googleSearch ).toHaveBeenCalledWith( { mode: 'MODE_DYNAMIC' } );
|
|
144
|
+
expect( urlContext ).toHaveBeenCalledWith( {} );
|
|
145
|
+
expect( result.googleSearch.type ).toBe( 'googleSearch' );
|
|
146
|
+
expect( result.urlContext.type ).toBe( 'urlContext' );
|
|
147
|
+
} );
|
|
148
|
+
|
|
149
|
+
it( 'throws when a configured tool is not supported by the provider', () => {
|
|
150
|
+
getProvider.mockReturnValue( {
|
|
151
|
+
tools: {
|
|
152
|
+
googleSearch: vi.fn(),
|
|
153
|
+
urlContext: vi.fn()
|
|
154
|
+
}
|
|
155
|
+
} );
|
|
156
|
+
|
|
157
|
+
expect( () => loadPromptTools( {
|
|
158
|
+
name: 'foo',
|
|
159
|
+
config: {
|
|
160
|
+
provider: 'vertex',
|
|
161
|
+
tools: {
|
|
162
|
+
unknownTool: {}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} ) ).toThrow( 'Provider "vertex" used by prompt "foo" does not support these tools: unknownTool.' );
|
|
166
|
+
} );
|
|
167
|
+
} );
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { extractSources } from './sources.js';
|
|
2
|
+
import { calculateLLMCallCost } from '../cost/index.js';
|
|
3
|
+
import { calculateBase64FileSize } from './image.js';
|
|
4
|
+
import { Tracing, Event } from '@outputai/core/sdk/runtime';
|
|
5
|
+
import { mapAiError } from './error_handler.js';
|
|
6
|
+
import { isPromise } from 'node:util/types';
|
|
7
|
+
import { Logger } from '@outputai/core';
|
|
8
|
+
|
|
9
|
+
/** Creates a proxy of the AI SDK response and attach virtual getters to it based on an object map */
|
|
10
|
+
const createResponseProxy = ( { response, properties } ) => new Proxy( response, {
|
|
11
|
+
get( target, prop, receiver ) {
|
|
12
|
+
return Object.hasOwn( properties, prop ) ? properties[prop] : Reflect.get( target, prop, receiver );
|
|
13
|
+
}
|
|
14
|
+
} );
|
|
15
|
+
|
|
16
|
+
/** Generate a trace id, starts the tracing and return the id */
|
|
17
|
+
const startTrace = ( { name, prompt } ) => {
|
|
18
|
+
const traceId = `${name}-${Date.now()}`;
|
|
19
|
+
Tracing.addEventStart( { kind: 'llm', id: traceId, name, details: { prompt } } );
|
|
20
|
+
return traceId;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/** Map AI Error, add an error trace entry and return the new error */
|
|
24
|
+
const handleError = ( { traceId, error: originalError } ) => {
|
|
25
|
+
const error = mapAiError( originalError );
|
|
26
|
+
Tracing.addEventError( { id: traceId, details: error } );
|
|
27
|
+
return error;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/** Extract usage, calculate the cost, emit an event and return cost/usage */
|
|
31
|
+
const handleCost = async ( { traceId, response, prompt } ) => {
|
|
32
|
+
const { model: modelId, provider: providerId } = prompt.config;
|
|
33
|
+
const usage = response.totalUsage ?? response.usage; // eg: image has .usage only
|
|
34
|
+
const cost = await calculateLLMCallCost( { usage, modelId, providerId } );
|
|
35
|
+
if ( cost ) {
|
|
36
|
+
Tracing.addEventAttribute( { eventId: traceId, attribute: cost } );
|
|
37
|
+
Event.emit( 'cost:llm:request', cost );
|
|
38
|
+
}
|
|
39
|
+
return { cost, usage };
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Runs a completing AI SDK call (`generateText`, `generateImage`, `generateTextWithStreaming`,
|
|
44
|
+
* `Agent.generate`, `Agent.generateWithStreaming`): start the LLM trace, run `fn`, attach cost
|
|
45
|
+
* (and sources on text), end the trace, and return a proxied response.
|
|
46
|
+
*
|
|
47
|
+
* Text responses get `result` (`text`), `cost`, and merged `sources`. Image responses get
|
|
48
|
+
* `result` (`image`) and `cost`. Trace usage is `response.totalUsage` if set, otherwise `response.usage`.
|
|
49
|
+
*
|
|
50
|
+
* @param {object} args
|
|
51
|
+
* @param {string} args.name - Trace event name
|
|
52
|
+
* @param {object} args.prompt - Loaded prompt (`config.provider` / `config.model` used for cost)
|
|
53
|
+
* @param {() => Promise<object>} args.fn - AI SDK call; must return the SDK response
|
|
54
|
+
* @returns {Promise<object>} Proxied SDK response
|
|
55
|
+
*/
|
|
56
|
+
export const wrapGeneration = async ( { name, prompt, fn } ) => {
|
|
57
|
+
const traceId = startTrace( { name, prompt } );
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
const response = await fn();
|
|
61
|
+
const { cost, usage } = await handleCost( { traceId, response, prompt } );
|
|
62
|
+
|
|
63
|
+
if ( Array.isArray( response.images ) ) {
|
|
64
|
+
const { image, images, providerMetadata } = response;
|
|
65
|
+
const mappedImages = images.map( ( { mediaType, base64 } ) => ( { size: calculateBase64FileSize( base64 ), mediaType } ) );
|
|
66
|
+
|
|
67
|
+
Tracing.addEventEnd( { id: traceId, details: { result: mappedImages, usage, cost, providerMetadata } } );
|
|
68
|
+
return createResponseProxy( { response, properties: { cost, result: image } } );
|
|
69
|
+
} else {
|
|
70
|
+
const { text: result, providerMetadata } = response;
|
|
71
|
+
const sources = extractSources( response );
|
|
72
|
+
|
|
73
|
+
Tracing.addEventEnd( { id: traceId, details: { result, usage, cost, providerMetadata, sources } } );
|
|
74
|
+
return createResponseProxy( { response, properties: { cost, sources, result } } );
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
} catch ( error ) {
|
|
78
|
+
throw handleError( { traceId, error } );
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Starts an LLM trace around a live AI SDK stream (`streamText`, `Agent.stream`).
|
|
84
|
+
*
|
|
85
|
+
* `fn` receives `onFinishHook(response, callback)` and `onErrorHook(event, callback)`. Call
|
|
86
|
+
* `onFinishHook` from the SDK `onFinish`: it wraps the response, ends the trace, then invokes
|
|
87
|
+
* `callback` without rethrowing. Call `onErrorHook` from SDK `onError`: it maps the error,
|
|
88
|
+
* records it, and invokes `callback` without rethrowing. Callback failures are logged.
|
|
89
|
+
*
|
|
90
|
+
* A throw or rejected Promise from `fn` (stream creation / Agent setup) is mapped and recorded
|
|
91
|
+
* on the LLM trace. A non-Promise return (the `streamText` stream) is returned immediately.
|
|
92
|
+
*
|
|
93
|
+
* @param {object} args
|
|
94
|
+
* @param {string} args.name - Trace event name
|
|
95
|
+
* @param {object} args.prompt - Loaded prompt (`config.provider` / `config.model` used for cost)
|
|
96
|
+
* @param {(hooks: { onFinishHook: Function, onErrorHook: Function }) => object | Promise<object>} args.fn -
|
|
97
|
+
* Return the SDK stream, or a Promise of that stream (`Agent.stream`); wire the hooks into SDK
|
|
98
|
+
* `onFinish` / `onError`
|
|
99
|
+
* @returns {object | Promise<object>} Value returned by `fn`; a rejected Promise is remapped
|
|
100
|
+
*/
|
|
101
|
+
export const wrapStream = ( { name, prompt, fn } ) => {
|
|
102
|
+
const traceId = startTrace( { name, prompt } );
|
|
103
|
+
|
|
104
|
+
const onFinishHook = async ( response, callback ) => {
|
|
105
|
+
const state = { proxyResponse: null };
|
|
106
|
+
try {
|
|
107
|
+
const { text: result, providerMetadata } = response;
|
|
108
|
+
const { cost, usage } = await handleCost( { traceId, response, prompt } );
|
|
109
|
+
const sources = extractSources( response );
|
|
110
|
+
Tracing.addEventEnd( { id: traceId, details: { result, usage, cost, providerMetadata, sources } } );
|
|
111
|
+
state.proxyResponse = createResponseProxy( { response, properties: { cost, sources, result } } );
|
|
112
|
+
} catch ( error ) {
|
|
113
|
+
throw handleError( { traceId, error } );
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
await callback?.( state.proxyResponse );
|
|
118
|
+
} catch ( callbackError ) {
|
|
119
|
+
// ignore these as this callback is fire and forget
|
|
120
|
+
Logger.error( 'Stream onFinish() callback failed', {
|
|
121
|
+
namespace: 'LLM',
|
|
122
|
+
error: callbackError instanceof Error ? callbackError.message : String( callbackError )
|
|
123
|
+
} );
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const onErrorHook = async ( event, callback ) => {
|
|
128
|
+
const error = handleError( { traceId, error: event.error } );
|
|
129
|
+
try {
|
|
130
|
+
await callback?.( error );
|
|
131
|
+
} catch ( callbackError ) {
|
|
132
|
+
// ignore these as this callback is fire and forget
|
|
133
|
+
Logger.error( 'Stream onError() callback failed', {
|
|
134
|
+
namespace: 'LLM',
|
|
135
|
+
error: callbackError instanceof Error ? callbackError.message : String( callbackError )
|
|
136
|
+
} );
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
const stream = fn( { onFinishHook, onErrorHook } );
|
|
142
|
+
return isPromise( stream ) ? stream.catch( error => {
|
|
143
|
+
throw handleError( { traceId, error } );
|
|
144
|
+
} ) : stream;
|
|
145
|
+
} catch ( error ) {
|
|
146
|
+
throw handleError( { traceId, error } );
|
|
147
|
+
}
|
|
148
|
+
};
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import streamResponseFixture from './__fixtures__/stream_response.json' with { type: 'json' };
|
|
3
|
+
|
|
4
|
+
const mocks = vi.hoisted( () => ( {
|
|
5
|
+
extractSources: vi.fn(),
|
|
6
|
+
calculateLLMCallCost: vi.fn(),
|
|
7
|
+
calculateBase64FileSize: vi.fn(),
|
|
8
|
+
mapAiError: vi.fn()
|
|
9
|
+
} ) );
|
|
10
|
+
|
|
11
|
+
vi.mock( './sources.js', () => ( {
|
|
12
|
+
extractSources: mocks.extractSources
|
|
13
|
+
} ) );
|
|
14
|
+
|
|
15
|
+
vi.mock( '../cost/index.js', () => ( {
|
|
16
|
+
calculateLLMCallCost: mocks.calculateLLMCallCost
|
|
17
|
+
} ) );
|
|
18
|
+
|
|
19
|
+
vi.mock( './image.js', () => ( {
|
|
20
|
+
calculateBase64FileSize: mocks.calculateBase64FileSize
|
|
21
|
+
} ) );
|
|
22
|
+
|
|
23
|
+
vi.mock( './error_handler.js', () => ( {
|
|
24
|
+
mapAiError: mocks.mapAiError
|
|
25
|
+
} ) );
|
|
26
|
+
|
|
27
|
+
vi.mock( '@outputai/core/sdk/runtime', () => ( {
|
|
28
|
+
Tracing: {
|
|
29
|
+
addEventStart: vi.fn(),
|
|
30
|
+
addEventError: vi.fn(),
|
|
31
|
+
addEventAttribute: vi.fn(),
|
|
32
|
+
addEventEnd: vi.fn()
|
|
33
|
+
},
|
|
34
|
+
Event: {
|
|
35
|
+
emit: vi.fn()
|
|
36
|
+
}
|
|
37
|
+
} ) );
|
|
38
|
+
|
|
39
|
+
import { Tracing, Event } from '@outputai/core/sdk/runtime';
|
|
40
|
+
import { wrapGeneration, wrapStream } from './wrap.js';
|
|
41
|
+
|
|
42
|
+
const tracing = vi.mocked( Tracing, true );
|
|
43
|
+
const event = vi.mocked( Event, true );
|
|
44
|
+
|
|
45
|
+
const clone = value => structuredClone( value );
|
|
46
|
+
|
|
47
|
+
const prompt = {
|
|
48
|
+
name: 'writer@v1',
|
|
49
|
+
config: { provider: 'openai', model: 'test-model' }
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const mockCost = { total: 0.001, components: [] };
|
|
53
|
+
const mappedError = new Error( 'mapped' );
|
|
54
|
+
|
|
55
|
+
describe( 'wrapGeneration / wrapStream', () => {
|
|
56
|
+
beforeEach( () => {
|
|
57
|
+
vi.clearAllMocks();
|
|
58
|
+
vi.spyOn( Date, 'now' ).mockReturnValue( 9_000_000_000 );
|
|
59
|
+
mocks.extractSources.mockReturnValue( [] );
|
|
60
|
+
mocks.calculateLLMCallCost.mockResolvedValue( mockCost );
|
|
61
|
+
mocks.calculateBase64FileSize.mockReturnValue( 1234 );
|
|
62
|
+
mocks.mapAiError.mockReturnValue( mappedError );
|
|
63
|
+
} );
|
|
64
|
+
|
|
65
|
+
afterEach( () => {
|
|
66
|
+
vi.restoreAllMocks();
|
|
67
|
+
} );
|
|
68
|
+
|
|
69
|
+
describe( 'wrapGeneration', () => {
|
|
70
|
+
it( 'starts an llm trace, wraps a text response, and ends with cost and sources', async () => {
|
|
71
|
+
const response = clone( streamResponseFixture );
|
|
72
|
+
const mergedSources = [ { url: 'https://merged.test' } ];
|
|
73
|
+
mocks.extractSources.mockReturnValue( mergedSources );
|
|
74
|
+
|
|
75
|
+
const wrapped = await wrapGeneration( {
|
|
76
|
+
name: 'generateText',
|
|
77
|
+
prompt,
|
|
78
|
+
fn: async () => response
|
|
79
|
+
} );
|
|
80
|
+
|
|
81
|
+
expect( tracing.addEventStart ).toHaveBeenCalledWith( {
|
|
82
|
+
kind: 'llm',
|
|
83
|
+
id: 'generateText-9000000000',
|
|
84
|
+
name: 'generateText',
|
|
85
|
+
details: { prompt }
|
|
86
|
+
} );
|
|
87
|
+
expect( mocks.calculateLLMCallCost ).toHaveBeenCalledWith( {
|
|
88
|
+
usage: response.totalUsage,
|
|
89
|
+
modelId: 'test-model',
|
|
90
|
+
providerId: 'openai'
|
|
91
|
+
} );
|
|
92
|
+
expect( tracing.addEventAttribute ).toHaveBeenCalledWith( {
|
|
93
|
+
eventId: 'generateText-9000000000',
|
|
94
|
+
attribute: mockCost
|
|
95
|
+
} );
|
|
96
|
+
expect( event.emit ).toHaveBeenCalledWith( 'cost:llm:request', mockCost );
|
|
97
|
+
expect( mocks.extractSources ).toHaveBeenCalledWith( response );
|
|
98
|
+
expect( tracing.addEventEnd ).toHaveBeenCalledWith( {
|
|
99
|
+
id: 'generateText-9000000000',
|
|
100
|
+
details: {
|
|
101
|
+
result: response.text,
|
|
102
|
+
usage: response.totalUsage,
|
|
103
|
+
cost: mockCost,
|
|
104
|
+
providerMetadata: response.providerMetadata,
|
|
105
|
+
sources: mergedSources
|
|
106
|
+
}
|
|
107
|
+
} );
|
|
108
|
+
expect( wrapped.result ).toBe( response.text );
|
|
109
|
+
expect( wrapped.cost ).toEqual( mockCost );
|
|
110
|
+
expect( wrapped.sources ).toBe( mergedSources );
|
|
111
|
+
expect( wrapped.text ).toBe( response.text );
|
|
112
|
+
} );
|
|
113
|
+
|
|
114
|
+
it( 'prefers totalUsage over usage when both are present', async () => {
|
|
115
|
+
const response = {
|
|
116
|
+
text: 'hi',
|
|
117
|
+
totalUsage: { inputTokens: 2 },
|
|
118
|
+
usage: { inputTokens: 99 },
|
|
119
|
+
steps: [],
|
|
120
|
+
sources: []
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
await wrapGeneration( { name: 'generateText', prompt, fn: async () => response } );
|
|
124
|
+
|
|
125
|
+
expect( mocks.calculateLLMCallCost ).toHaveBeenCalledWith( {
|
|
126
|
+
usage: { inputTokens: 2 },
|
|
127
|
+
modelId: 'test-model',
|
|
128
|
+
providerId: 'openai'
|
|
129
|
+
} );
|
|
130
|
+
} );
|
|
131
|
+
|
|
132
|
+
it( 'wraps an image response using usage and mapped image metadata', async () => {
|
|
133
|
+
const image = { mediaType: 'image/png', base64: 'abc' };
|
|
134
|
+
const response = {
|
|
135
|
+
image,
|
|
136
|
+
images: [ image ],
|
|
137
|
+
usage: { inputTokens: 8, outputTokens: 16 },
|
|
138
|
+
providerMetadata: { openai: {} }
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
const wrapped = await wrapGeneration( {
|
|
142
|
+
name: 'generateImage',
|
|
143
|
+
prompt,
|
|
144
|
+
fn: async () => response
|
|
145
|
+
} );
|
|
146
|
+
|
|
147
|
+
expect( mocks.calculateLLMCallCost ).toHaveBeenCalledWith( {
|
|
148
|
+
usage: response.usage,
|
|
149
|
+
modelId: 'test-model',
|
|
150
|
+
providerId: 'openai'
|
|
151
|
+
} );
|
|
152
|
+
expect( mocks.calculateBase64FileSize ).toHaveBeenCalledWith( 'abc' );
|
|
153
|
+
expect( tracing.addEventEnd ).toHaveBeenCalledWith( {
|
|
154
|
+
id: 'generateImage-9000000000',
|
|
155
|
+
details: {
|
|
156
|
+
result: [ { size: 1234, mediaType: 'image/png' } ],
|
|
157
|
+
usage: response.usage,
|
|
158
|
+
cost: mockCost,
|
|
159
|
+
providerMetadata: response.providerMetadata
|
|
160
|
+
}
|
|
161
|
+
} );
|
|
162
|
+
expect( mocks.extractSources ).not.toHaveBeenCalled();
|
|
163
|
+
expect( wrapped.result ).toBe( image );
|
|
164
|
+
expect( wrapped.cost ).toEqual( mockCost );
|
|
165
|
+
} );
|
|
166
|
+
|
|
167
|
+
it( 'skips cost attribute and event when cost is missing', async () => {
|
|
168
|
+
mocks.calculateLLMCallCost.mockResolvedValue( null );
|
|
169
|
+
const response = clone( streamResponseFixture );
|
|
170
|
+
|
|
171
|
+
const wrapped = await wrapGeneration( {
|
|
172
|
+
name: 'generateText',
|
|
173
|
+
prompt,
|
|
174
|
+
fn: async () => response
|
|
175
|
+
} );
|
|
176
|
+
|
|
177
|
+
expect( tracing.addEventAttribute ).not.toHaveBeenCalled();
|
|
178
|
+
expect( event.emit ).not.toHaveBeenCalled();
|
|
179
|
+
expect( tracing.addEventEnd ).toHaveBeenCalledWith( {
|
|
180
|
+
id: 'generateText-9000000000',
|
|
181
|
+
details: expect.objectContaining( { cost: null, result: response.text } )
|
|
182
|
+
} );
|
|
183
|
+
expect( wrapped.cost ).toBeNull();
|
|
184
|
+
} );
|
|
185
|
+
|
|
186
|
+
it( 'maps fn errors onto the llm trace and rethrows', async () => {
|
|
187
|
+
const original = new Error( 'boom' );
|
|
188
|
+
|
|
189
|
+
await expect( wrapGeneration( {
|
|
190
|
+
name: 'Agent.generate',
|
|
191
|
+
prompt,
|
|
192
|
+
fn: async () => {
|
|
193
|
+
throw original;
|
|
194
|
+
}
|
|
195
|
+
} ) ).rejects.toBe( mappedError );
|
|
196
|
+
|
|
197
|
+
expect( mocks.mapAiError ).toHaveBeenCalledWith( original );
|
|
198
|
+
expect( tracing.addEventError ).toHaveBeenCalledWith( {
|
|
199
|
+
id: 'Agent.generate-9000000000',
|
|
200
|
+
details: mappedError
|
|
201
|
+
} );
|
|
202
|
+
expect( tracing.addEventEnd ).not.toHaveBeenCalled();
|
|
203
|
+
} );
|
|
204
|
+
} );
|
|
205
|
+
|
|
206
|
+
describe( 'wrapStream', () => {
|
|
207
|
+
const hooksFrom = name => {
|
|
208
|
+
const fn = vi.fn( () => ( {} ) );
|
|
209
|
+
wrapStream( { name, prompt, fn } );
|
|
210
|
+
return fn.mock.calls[0][0];
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
it( 'starts an llm trace, returns fn result, and passes hooks', () => {
|
|
214
|
+
const stream = { kind: 'stream' };
|
|
215
|
+
const fn = vi.fn( () => stream );
|
|
216
|
+
|
|
217
|
+
const result = wrapStream( { name: 'streamText', prompt, fn } );
|
|
218
|
+
|
|
219
|
+
expect( result ).toBe( stream );
|
|
220
|
+
expect( tracing.addEventStart ).toHaveBeenCalledWith( {
|
|
221
|
+
kind: 'llm',
|
|
222
|
+
id: 'streamText-9000000000',
|
|
223
|
+
name: 'streamText',
|
|
224
|
+
details: { prompt }
|
|
225
|
+
} );
|
|
226
|
+
expect( fn ).toHaveBeenCalledWith( {
|
|
227
|
+
onFinishHook: expect.any( Function ),
|
|
228
|
+
onErrorHook: expect.any( Function )
|
|
229
|
+
} );
|
|
230
|
+
expect( tracing.addEventEnd ).not.toHaveBeenCalled();
|
|
231
|
+
} );
|
|
232
|
+
|
|
233
|
+
it( 'wraps onFinish, ends the trace, then awaits the callback', async () => {
|
|
234
|
+
const response = clone( streamResponseFixture );
|
|
235
|
+
const mergedSources = [ { url: 'https://s.test' } ];
|
|
236
|
+
mocks.extractSources.mockReturnValue( mergedSources );
|
|
237
|
+
const callback = vi.fn();
|
|
238
|
+
const { onFinishHook } = hooksFrom( 'Agent.stream' );
|
|
239
|
+
|
|
240
|
+
await onFinishHook( response, callback );
|
|
241
|
+
|
|
242
|
+
expect( mocks.extractSources ).toHaveBeenCalledWith( response );
|
|
243
|
+
expect( callback ).toHaveBeenCalledOnce();
|
|
244
|
+
const proxied = callback.mock.calls[0][0];
|
|
245
|
+
expect( proxied.result ).toBe( response.text );
|
|
246
|
+
expect( proxied.cost ).toEqual( mockCost );
|
|
247
|
+
expect( proxied.sources ).toBe( mergedSources );
|
|
248
|
+
expect( tracing.addEventEnd ).toHaveBeenCalledWith( {
|
|
249
|
+
id: 'Agent.stream-9000000000',
|
|
250
|
+
details: {
|
|
251
|
+
result: response.text,
|
|
252
|
+
usage: response.totalUsage,
|
|
253
|
+
cost: mockCost,
|
|
254
|
+
providerMetadata: response.providerMetadata,
|
|
255
|
+
sources: mergedSources
|
|
256
|
+
}
|
|
257
|
+
} );
|
|
258
|
+
expect( tracing.addEventEnd.mock.invocationCallOrder[0] ).toBeLessThan( callback.mock.invocationCallOrder[0] );
|
|
259
|
+
} );
|
|
260
|
+
|
|
261
|
+
it( 'ends the trace and swallows throws from the onFinish callback', async () => {
|
|
262
|
+
const response = clone( streamResponseFixture );
|
|
263
|
+
const { onFinishHook } = hooksFrom( 'Agent.stream' );
|
|
264
|
+
|
|
265
|
+
await expect( onFinishHook( response, async () => {
|
|
266
|
+
throw new Error( 'user onFinish' );
|
|
267
|
+
} ) ).resolves.toBeUndefined();
|
|
268
|
+
|
|
269
|
+
expect( tracing.addEventEnd ).toHaveBeenCalledOnce();
|
|
270
|
+
expect( tracing.addEventError ).not.toHaveBeenCalled();
|
|
271
|
+
expect( mocks.mapAiError ).not.toHaveBeenCalled();
|
|
272
|
+
} );
|
|
273
|
+
|
|
274
|
+
it( 'maps onError events and forwards the mapped error to the callback', () => {
|
|
275
|
+
const original = new Error( 'stream failed' );
|
|
276
|
+
const callback = vi.fn();
|
|
277
|
+
const { onErrorHook } = hooksFrom( 'streamText' );
|
|
278
|
+
|
|
279
|
+
onErrorHook( { error: original }, callback );
|
|
280
|
+
|
|
281
|
+
expect( mocks.mapAiError ).toHaveBeenCalledWith( original );
|
|
282
|
+
expect( tracing.addEventError ).toHaveBeenCalledWith( {
|
|
283
|
+
id: 'streamText-9000000000',
|
|
284
|
+
details: mappedError
|
|
285
|
+
} );
|
|
286
|
+
expect( callback ).toHaveBeenCalledWith( mappedError );
|
|
287
|
+
} );
|
|
288
|
+
|
|
289
|
+
it( 'swallows throws from the onError callback', async () => {
|
|
290
|
+
const { onErrorHook } = hooksFrom( 'streamText' );
|
|
291
|
+
|
|
292
|
+
await expect( onErrorHook( { error: new Error( 'x' ) }, () => {
|
|
293
|
+
throw new Error( 'user onError' );
|
|
294
|
+
} ) ).resolves.toBeUndefined();
|
|
295
|
+
} );
|
|
296
|
+
|
|
297
|
+
it( 'awaits and swallows rejected promises from the onError callback', async () => {
|
|
298
|
+
const state = { rejection: null };
|
|
299
|
+
const callback = vi.fn( () => new Promise( ( _, reject ) => {
|
|
300
|
+
state.rejection = reject;
|
|
301
|
+
} ) );
|
|
302
|
+
const { onErrorHook } = hooksFrom( 'streamText' );
|
|
303
|
+
|
|
304
|
+
const hookPromise = onErrorHook( { error: new Error( 'x' ) }, callback );
|
|
305
|
+
|
|
306
|
+
expect( hookPromise ).toBeInstanceOf( Promise );
|
|
307
|
+
state.rejection( new Error( 'async user onError' ) );
|
|
308
|
+
await expect( hookPromise ).resolves.toBeUndefined();
|
|
309
|
+
} );
|
|
310
|
+
|
|
311
|
+
it( 'maps a throw from fn onto the llm trace and rethrows', () => {
|
|
312
|
+
const original = new Error( 'create stream' );
|
|
313
|
+
|
|
314
|
+
expect( () => wrapStream( {
|
|
315
|
+
name: 'streamText',
|
|
316
|
+
prompt,
|
|
317
|
+
fn: () => {
|
|
318
|
+
throw original;
|
|
319
|
+
}
|
|
320
|
+
} ) ).toThrow( mappedError );
|
|
321
|
+
|
|
322
|
+
expect( mocks.mapAiError ).toHaveBeenCalledWith( original );
|
|
323
|
+
expect( tracing.addEventError ).toHaveBeenCalledWith( {
|
|
324
|
+
id: 'streamText-9000000000',
|
|
325
|
+
details: mappedError
|
|
326
|
+
} );
|
|
327
|
+
} );
|
|
328
|
+
|
|
329
|
+
it( 'maps a rejected promise from fn onto the llm trace and rethrows', async () => {
|
|
330
|
+
const original = new Error( 'prepareCall failed' );
|
|
331
|
+
|
|
332
|
+
await expect( wrapStream( {
|
|
333
|
+
name: 'Agent.stream',
|
|
334
|
+
prompt,
|
|
335
|
+
fn: () => Promise.reject( original )
|
|
336
|
+
} ) ).rejects.toBe( mappedError );
|
|
337
|
+
|
|
338
|
+
expect( mocks.mapAiError ).toHaveBeenCalledWith( original );
|
|
339
|
+
expect( tracing.addEventError ).toHaveBeenCalledWith( {
|
|
340
|
+
id: 'Agent.stream-9000000000',
|
|
341
|
+
details: mappedError
|
|
342
|
+
} );
|
|
343
|
+
expect( tracing.addEventEnd ).not.toHaveBeenCalled();
|
|
344
|
+
} );
|
|
345
|
+
|
|
346
|
+
it( 'returns a fulfilled promise from fn without mapping', async () => {
|
|
347
|
+
const stream = { kind: 'stream' };
|
|
348
|
+
|
|
349
|
+
await expect( wrapStream( {
|
|
350
|
+
name: 'Agent.stream',
|
|
351
|
+
prompt,
|
|
352
|
+
fn: () => Promise.resolve( stream )
|
|
353
|
+
} ) ).resolves.toBe( stream );
|
|
354
|
+
|
|
355
|
+
expect( mocks.mapAiError ).not.toHaveBeenCalled();
|
|
356
|
+
expect( tracing.addEventError ).not.toHaveBeenCalled();
|
|
357
|
+
} );
|
|
358
|
+
} );
|
|
359
|
+
} );
|