@cogitator-ai/test-utils 0.2.20 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conformance/fake-provider.d.ts +59 -0
- package/dist/conformance/fake-provider.d.ts.map +1 -0
- package/dist/conformance/fake-provider.js +134 -0
- package/dist/conformance/index.d.ts +9 -0
- package/dist/conformance/index.d.ts.map +1 -0
- package/dist/conformance/index.js +8 -0
- package/dist/conformance/scenarios.d.ts +95 -0
- package/dist/conformance/scenarios.d.ts.map +1 -0
- package/dist/conformance/scenarios.js +287 -0
- package/dist/conformance/wires/anthropic.d.ts +7 -0
- package/dist/conformance/wires/anthropic.d.ts.map +1 -0
- package/dist/conformance/wires/anthropic.js +139 -0
- package/dist/conformance/wires/bedrock.d.ts +8 -0
- package/dist/conformance/wires/bedrock.d.ts.map +1 -0
- package/dist/conformance/wires/bedrock.js +129 -0
- package/dist/conformance/wires/encoding.d.ts +11 -0
- package/dist/conformance/wires/encoding.d.ts.map +1 -0
- package/dist/conformance/wires/encoding.js +38 -0
- package/dist/conformance/wires/gemini.d.ts +7 -0
- package/dist/conformance/wires/gemini.d.ts.map +1 -0
- package/dist/conformance/wires/gemini.js +121 -0
- package/dist/conformance/wires/ollama.d.ts +8 -0
- package/dist/conformance/wires/ollama.d.ts.map +1 -0
- package/dist/conformance/wires/ollama.js +83 -0
- package/dist/conformance/wires/openai.d.ts +11 -0
- package/dist/conformance/wires/openai.d.ts.map +1 -0
- package/dist/conformance/wires/openai.js +345 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { CONFORMANCE_FIXTURE as F, } from '../scenarios.js';
|
|
2
|
+
import { sse } from './encoding.js';
|
|
3
|
+
const anthropicError = (status, type, message, headers) => ({
|
|
4
|
+
type: 'json',
|
|
5
|
+
status,
|
|
6
|
+
...(headers && { headers }),
|
|
7
|
+
body: { type: 'error', error: { type, message }, request_id: 'req_conformance' },
|
|
8
|
+
});
|
|
9
|
+
/** The error types of the Messages API (platform.claude.com/docs/en/api/errors). */
|
|
10
|
+
const ANTHROPIC_ERRORS = {
|
|
11
|
+
'rate-limit': anthropicError(429, 'rate_limit_error', 'Number of requests has exceeded your rate limit', {
|
|
12
|
+
'retry-after': String(F.retryAfterSeconds),
|
|
13
|
+
}),
|
|
14
|
+
quota: anthropicError(400, 'invalid_request_error', 'Your credit balance is too low to access the Anthropic API.'),
|
|
15
|
+
auth: anthropicError(401, 'authentication_error', 'invalid x-api-key'),
|
|
16
|
+
overloaded: anthropicError(529, 'overloaded_error', 'Overloaded'),
|
|
17
|
+
'server-error': anthropicError(500, 'api_error', 'Internal server error'),
|
|
18
|
+
'context-overflow': anthropicError(400, 'invalid_request_error', 'prompt is too long: 210000 tokens > 200000 maximum'),
|
|
19
|
+
};
|
|
20
|
+
const TURNS = {
|
|
21
|
+
text: { text: F.text, stopReason: 'end_turn' },
|
|
22
|
+
'tool-call': { toolInput: JSON.stringify(F.toolCall.arguments), stopReason: 'tool_use' },
|
|
23
|
+
'tool-call-with-stop': {
|
|
24
|
+
toolInput: JSON.stringify(F.toolCall.arguments),
|
|
25
|
+
stopReason: 'end_turn',
|
|
26
|
+
},
|
|
27
|
+
'truncated-tool-call': { toolInput: F.truncatedArguments, stopReason: 'max_tokens' },
|
|
28
|
+
'content-filter': { stopReason: 'refusal' },
|
|
29
|
+
};
|
|
30
|
+
const MESSAGE = {
|
|
31
|
+
id: 'msg_conformance',
|
|
32
|
+
type: 'message',
|
|
33
|
+
role: 'assistant',
|
|
34
|
+
model: 'conformance',
|
|
35
|
+
stop_sequence: null,
|
|
36
|
+
};
|
|
37
|
+
function message(turn) {
|
|
38
|
+
const content = [];
|
|
39
|
+
if (turn.text)
|
|
40
|
+
content.push({ type: 'text', text: turn.text });
|
|
41
|
+
if (turn.toolInput !== undefined) {
|
|
42
|
+
content.push({
|
|
43
|
+
type: 'tool_use',
|
|
44
|
+
id: F.toolCall.id,
|
|
45
|
+
name: F.toolCall.name,
|
|
46
|
+
input: safeJson(turn.toolInput),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
...MESSAGE,
|
|
51
|
+
content,
|
|
52
|
+
stop_reason: turn.stopReason,
|
|
53
|
+
usage: { input_tokens: F.usage.inputTokens, output_tokens: F.usage.outputTokens },
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function safeJson(text) {
|
|
57
|
+
try {
|
|
58
|
+
return JSON.parse(text);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return {};
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const event = (type, data) => sse({ type, ...data }, type);
|
|
65
|
+
const messageStart = event('message_start', {
|
|
66
|
+
message: {
|
|
67
|
+
...MESSAGE,
|
|
68
|
+
content: [],
|
|
69
|
+
stop_reason: null,
|
|
70
|
+
usage: { input_tokens: F.usage.inputTokens, output_tokens: 1 },
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
const textStart = (text) => [
|
|
74
|
+
event('content_block_start', { index: 0, content_block: { type: 'text', text: '' } }),
|
|
75
|
+
event('content_block_delta', { index: 0, delta: { type: 'text_delta', text } }),
|
|
76
|
+
];
|
|
77
|
+
function stream(turn) {
|
|
78
|
+
const events = [messageStart];
|
|
79
|
+
let index = 0;
|
|
80
|
+
if (turn.text) {
|
|
81
|
+
events.push(event('content_block_start', { index, content_block: { type: 'text', text: '' } }), ...F.textChunks.map((text) => event('content_block_delta', { index, delta: { type: 'text_delta', text } })), event('content_block_stop', { index }));
|
|
82
|
+
index++;
|
|
83
|
+
}
|
|
84
|
+
if (turn.toolInput !== undefined) {
|
|
85
|
+
const half = Math.ceil(turn.toolInput.length / 2);
|
|
86
|
+
events.push(event('content_block_start', {
|
|
87
|
+
index,
|
|
88
|
+
content_block: { type: 'tool_use', id: F.toolCall.id, name: F.toolCall.name, input: {} },
|
|
89
|
+
}), event('content_block_delta', {
|
|
90
|
+
index,
|
|
91
|
+
delta: { type: 'input_json_delta', partial_json: turn.toolInput.slice(0, half) },
|
|
92
|
+
}), event('content_block_delta', {
|
|
93
|
+
index,
|
|
94
|
+
delta: { type: 'input_json_delta', partial_json: turn.toolInput.slice(half) },
|
|
95
|
+
}), event('content_block_stop', { index }));
|
|
96
|
+
}
|
|
97
|
+
events.push(event('message_delta', {
|
|
98
|
+
delta: { stop_reason: turn.stopReason, stop_sequence: null },
|
|
99
|
+
usage: { output_tokens: F.usage.outputTokens },
|
|
100
|
+
}), event('message_stop', {}));
|
|
101
|
+
return events;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The Anthropic Messages API at `/v1/messages`. An error inside a stream that began is an
|
|
105
|
+
* `error` event, as the API sends an overload after the response started.
|
|
106
|
+
*/
|
|
107
|
+
export const anthropicWire = {
|
|
108
|
+
name: 'Anthropic Messages',
|
|
109
|
+
path: () => /\/v1\/messages$/,
|
|
110
|
+
reply(scenario, mode) {
|
|
111
|
+
const error = ANTHROPIC_ERRORS[scenario];
|
|
112
|
+
if (error)
|
|
113
|
+
return error;
|
|
114
|
+
const streamed = (chunks, hang = false) => ({
|
|
115
|
+
type: 'stream',
|
|
116
|
+
contentType: 'text/event-stream',
|
|
117
|
+
chunks,
|
|
118
|
+
hang,
|
|
119
|
+
});
|
|
120
|
+
switch (scenario) {
|
|
121
|
+
case 'stream-error':
|
|
122
|
+
return streamed([
|
|
123
|
+
messageStart,
|
|
124
|
+
...textStart(F.partialText),
|
|
125
|
+
event('error', { error: { type: 'overloaded_error', message: 'Overloaded' } }),
|
|
126
|
+
]);
|
|
127
|
+
case 'abort':
|
|
128
|
+
return mode === 'chat'
|
|
129
|
+
? { type: 'hang' }
|
|
130
|
+
: streamed([messageStart, ...textStart(F.partialText)], true);
|
|
131
|
+
default: {
|
|
132
|
+
const turn = TURNS[scenario];
|
|
133
|
+
if (!turn)
|
|
134
|
+
return undefined;
|
|
135
|
+
return mode === 'chat' ? { type: 'json', body: message(turn) } : streamed(stream(turn));
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type ProviderWire } from '../scenarios.js';
|
|
2
|
+
/**
|
|
3
|
+
* Bedrock Runtime's Converse and ConverseStream (`/model/{id}/converse`, `/converse-stream`),
|
|
4
|
+
* streaming in the AWS event stream encoding. An error inside a stream that began is an
|
|
5
|
+
* exception message, such as `serviceUnavailableException`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const bedrockConverseWire: ProviderWire;
|
|
8
|
+
//# sourceMappingURL=bedrock.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/bedrock.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AA6ItB;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,EAAE,YA+BjC,CAAC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { CONFORMANCE_FIXTURE as F, } from '../scenarios.js';
|
|
2
|
+
import { awsEventStreamMessage } from './encoding.js';
|
|
3
|
+
const bedrockError = (status, exception, message, headers) => ({
|
|
4
|
+
type: 'json',
|
|
5
|
+
status,
|
|
6
|
+
headers: {
|
|
7
|
+
'x-amzn-errortype': `${exception}:http://internal.amazon.com/coral/com.amazon.bedrock/`,
|
|
8
|
+
...headers,
|
|
9
|
+
},
|
|
10
|
+
body: { __type: exception, message },
|
|
11
|
+
});
|
|
12
|
+
/** The exceptions of the Bedrock Runtime API, with the statuses it sends them with. */
|
|
13
|
+
const BEDROCK_ERRORS = {
|
|
14
|
+
'rate-limit': bedrockError(429, 'ThrottlingException', 'Too many requests, please wait.', {
|
|
15
|
+
'retry-after': String(F.retryAfterSeconds),
|
|
16
|
+
}),
|
|
17
|
+
quota: bedrockError(400, 'ServiceQuotaExceededException', 'Your request exceeds the service quota for your account.'),
|
|
18
|
+
auth: bedrockError(403, 'UnrecognizedClientException', 'The security token included in the request is invalid.'),
|
|
19
|
+
overloaded: bedrockError(503, 'ServiceUnavailableException', 'Service unavailable. Try again later.'),
|
|
20
|
+
'server-error': bedrockError(500, 'InternalServerException', 'Internal server error.'),
|
|
21
|
+
'context-overflow': bedrockError(400, 'ValidationException', 'Input is too long for requested model.'),
|
|
22
|
+
};
|
|
23
|
+
const TURNS = {
|
|
24
|
+
text: { text: F.text, stopReason: 'end_turn' },
|
|
25
|
+
'tool-call': { toolInput: JSON.stringify(F.toolCall.arguments), stopReason: 'tool_use' },
|
|
26
|
+
'tool-call-with-stop': {
|
|
27
|
+
toolInput: JSON.stringify(F.toolCall.arguments),
|
|
28
|
+
stopReason: 'end_turn',
|
|
29
|
+
},
|
|
30
|
+
'truncated-tool-call': { toolInput: F.truncatedArguments, stopReason: 'max_tokens' },
|
|
31
|
+
'content-filter': { stopReason: 'content_filtered' },
|
|
32
|
+
};
|
|
33
|
+
const USAGE = {
|
|
34
|
+
inputTokens: F.usage.inputTokens,
|
|
35
|
+
outputTokens: F.usage.outputTokens,
|
|
36
|
+
totalTokens: F.usage.inputTokens + F.usage.outputTokens,
|
|
37
|
+
};
|
|
38
|
+
const event = (type, payload) => awsEventStreamMessage({ ':event-type': type, ':content-type': 'application/json', ':message-type': 'event' }, payload);
|
|
39
|
+
const exception = (type, message) => awsEventStreamMessage({ ':exception-type': type, ':content-type': 'application/json', ':message-type': 'exception' }, { message });
|
|
40
|
+
function converse(turn) {
|
|
41
|
+
const content = [];
|
|
42
|
+
if (turn.text)
|
|
43
|
+
content.push({ text: turn.text });
|
|
44
|
+
if (turn.toolInput !== undefined) {
|
|
45
|
+
content.push({
|
|
46
|
+
toolUse: { toolUseId: F.toolCall.id, name: F.toolCall.name, input: parsed(turn.toolInput) },
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
output: { message: { role: 'assistant', content } },
|
|
51
|
+
stopReason: turn.stopReason,
|
|
52
|
+
usage: USAGE,
|
|
53
|
+
metrics: { latencyMs: 1 },
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** The JSON of a tool input, or an empty input for JSON cut off at the token limit. */
|
|
57
|
+
function parsed(text) {
|
|
58
|
+
try {
|
|
59
|
+
return JSON.parse(text);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return {};
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function stream(turn) {
|
|
66
|
+
const events = [event('messageStart', { role: 'assistant' })];
|
|
67
|
+
let index = 0;
|
|
68
|
+
if (turn.text) {
|
|
69
|
+
for (const text of F.textChunks) {
|
|
70
|
+
events.push(event('contentBlockDelta', { contentBlockIndex: index, delta: { text } }));
|
|
71
|
+
}
|
|
72
|
+
events.push(event('contentBlockStop', { contentBlockIndex: index }));
|
|
73
|
+
index++;
|
|
74
|
+
}
|
|
75
|
+
if (turn.toolInput !== undefined) {
|
|
76
|
+
const half = Math.ceil(turn.toolInput.length / 2);
|
|
77
|
+
events.push(event('contentBlockStart', {
|
|
78
|
+
contentBlockIndex: index,
|
|
79
|
+
start: { toolUse: { toolUseId: F.toolCall.id, name: F.toolCall.name } },
|
|
80
|
+
}), event('contentBlockDelta', {
|
|
81
|
+
contentBlockIndex: index,
|
|
82
|
+
delta: { toolUse: { input: turn.toolInput.slice(0, half) } },
|
|
83
|
+
}), event('contentBlockDelta', {
|
|
84
|
+
contentBlockIndex: index,
|
|
85
|
+
delta: { toolUse: { input: turn.toolInput.slice(half) } },
|
|
86
|
+
}), event('contentBlockStop', { contentBlockIndex: index }));
|
|
87
|
+
}
|
|
88
|
+
events.push(event('messageStop', { stopReason: turn.stopReason }), event('metadata', { usage: USAGE, metrics: { latencyMs: 1 } }));
|
|
89
|
+
return events;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Bedrock Runtime's Converse and ConverseStream (`/model/{id}/converse`, `/converse-stream`),
|
|
93
|
+
* streaming in the AWS event stream encoding. An error inside a stream that began is an
|
|
94
|
+
* exception message, such as `serviceUnavailableException`.
|
|
95
|
+
*/
|
|
96
|
+
export const bedrockConverseWire = {
|
|
97
|
+
name: 'Bedrock Converse',
|
|
98
|
+
path: (mode) => (mode === 'chat' ? /\/model\/[^/]+\/converse$/ : /\/converse-stream$/),
|
|
99
|
+
reply(scenario, mode) {
|
|
100
|
+
const error = BEDROCK_ERRORS[scenario];
|
|
101
|
+
if (error)
|
|
102
|
+
return error;
|
|
103
|
+
const streamed = (chunks, hang = false) => ({
|
|
104
|
+
type: 'stream',
|
|
105
|
+
contentType: 'application/vnd.amazon.eventstream',
|
|
106
|
+
chunks,
|
|
107
|
+
hang,
|
|
108
|
+
});
|
|
109
|
+
const started = [
|
|
110
|
+
event('messageStart', { role: 'assistant' }),
|
|
111
|
+
event('contentBlockDelta', { contentBlockIndex: 0, delta: { text: F.partialText } }),
|
|
112
|
+
];
|
|
113
|
+
switch (scenario) {
|
|
114
|
+
case 'stream-error':
|
|
115
|
+
return streamed([
|
|
116
|
+
...started,
|
|
117
|
+
exception('serviceUnavailableException', 'Service unavailable. Try again later.'),
|
|
118
|
+
]);
|
|
119
|
+
case 'abort':
|
|
120
|
+
return mode === 'chat' ? { type: 'hang' } : streamed(started, true);
|
|
121
|
+
default: {
|
|
122
|
+
const turn = TURNS[scenario];
|
|
123
|
+
if (!turn)
|
|
124
|
+
return undefined;
|
|
125
|
+
return mode === 'chat' ? { type: 'json', body: converse(turn) } : streamed(stream(turn));
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** One server-sent event: `data:` lines, with an `event:` line when `event` is given. */
|
|
2
|
+
export declare function sse(data: unknown, event?: string): string;
|
|
3
|
+
/** One line of newline-delimited JSON. */
|
|
4
|
+
export declare function ndjson(data: unknown): string;
|
|
5
|
+
/**
|
|
6
|
+
* One message of the AWS event stream encoding (`application/vnd.amazon.eventstream`), as Bedrock
|
|
7
|
+
* streams `ConverseStream` events: a prelude with the lengths and its CRC32, string headers, the
|
|
8
|
+
* JSON payload, and the CRC32 of the whole message.
|
|
9
|
+
*/
|
|
10
|
+
export declare function awsEventStreamMessage(headers: Record<string, string>, payload: unknown): Uint8Array;
|
|
11
|
+
//# sourceMappingURL=encoding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encoding.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/encoding.ts"],"names":[],"mappings":"AAEA,yFAAyF;AACzF,wBAAgB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,0CAA0C;AAC1C,wBAAgB,MAAM,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAE5C;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,OAAO,EAAE,OAAO,GACf,UAAU,CAyBZ"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { crc32 } from 'node:zlib';
|
|
2
|
+
/** One server-sent event: `data:` lines, with an `event:` line when `event` is given. */
|
|
3
|
+
export function sse(data, event) {
|
|
4
|
+
const body = typeof data === 'string' ? data : JSON.stringify(data);
|
|
5
|
+
return `${event ? `event: ${event}\n` : ''}data: ${body}\n\n`;
|
|
6
|
+
}
|
|
7
|
+
/** One line of newline-delimited JSON. */
|
|
8
|
+
export function ndjson(data) {
|
|
9
|
+
return `${JSON.stringify(data)}\n`;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* One message of the AWS event stream encoding (`application/vnd.amazon.eventstream`), as Bedrock
|
|
13
|
+
* streams `ConverseStream` events: a prelude with the lengths and its CRC32, string headers, the
|
|
14
|
+
* JSON payload, and the CRC32 of the whole message.
|
|
15
|
+
*/
|
|
16
|
+
export function awsEventStreamMessage(headers, payload) {
|
|
17
|
+
const headerBytes = Buffer.concat(Object.entries(headers).map(([name, value]) => {
|
|
18
|
+
const nameBytes = Buffer.from(name, 'utf8');
|
|
19
|
+
const valueBytes = Buffer.from(value, 'utf8');
|
|
20
|
+
const header = Buffer.alloc(1 + nameBytes.length + 1 + 2 + valueBytes.length);
|
|
21
|
+
let offset = header.writeUInt8(nameBytes.length, 0);
|
|
22
|
+
offset += nameBytes.copy(header, offset);
|
|
23
|
+
offset = header.writeUInt8(7, offset);
|
|
24
|
+
offset = header.writeUInt16BE(valueBytes.length, offset);
|
|
25
|
+
valueBytes.copy(header, offset);
|
|
26
|
+
return header;
|
|
27
|
+
}));
|
|
28
|
+
const payloadBytes = Buffer.from(JSON.stringify(payload), 'utf8');
|
|
29
|
+
const total = 12 + headerBytes.length + payloadBytes.length + 4;
|
|
30
|
+
const message = Buffer.alloc(total);
|
|
31
|
+
message.writeUInt32BE(total, 0);
|
|
32
|
+
message.writeUInt32BE(headerBytes.length, 4);
|
|
33
|
+
message.writeUInt32BE(crc32(message.subarray(0, 8)), 8);
|
|
34
|
+
headerBytes.copy(message, 12);
|
|
35
|
+
payloadBytes.copy(message, 12 + headerBytes.length);
|
|
36
|
+
message.writeUInt32BE(crc32(message.subarray(0, total - 4)), total - 4);
|
|
37
|
+
return new Uint8Array(message);
|
|
38
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type ProviderWire } from '../scenarios.js';
|
|
2
|
+
/**
|
|
3
|
+
* The Gemini API's `generateContent` and `streamGenerateContent?alt=sse`. An error inside a
|
|
4
|
+
* stream that began is a chunk holding `{ error }`, as Google's own SDK expects it.
|
|
5
|
+
*/
|
|
6
|
+
export declare const geminiWire: ProviderWire;
|
|
7
|
+
//# sourceMappingURL=gemini.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/gemini.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AAwGtB;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,YAqCxB,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { CONFORMANCE_FIXTURE as F, } from '../scenarios.js';
|
|
2
|
+
import { sse } from './encoding.js';
|
|
3
|
+
const geminiError = (code, status, message, details) => ({
|
|
4
|
+
type: 'json',
|
|
5
|
+
status: code,
|
|
6
|
+
body: { error: { code, message, status, ...(details && { details }) } },
|
|
7
|
+
});
|
|
8
|
+
const quotaFailure = (quotaId) => ({
|
|
9
|
+
'@type': 'type.googleapis.com/google.rpc.QuotaFailure',
|
|
10
|
+
violations: [
|
|
11
|
+
{
|
|
12
|
+
quotaMetric: 'generativelanguage.googleapis.com/generate_content_free_tier_requests',
|
|
13
|
+
quotaId,
|
|
14
|
+
quotaDimensions: { location: 'global', model: 'gemini-conformance' },
|
|
15
|
+
},
|
|
16
|
+
],
|
|
17
|
+
});
|
|
18
|
+
/** The errors of the Gemini API (ai.google.dev/gemini-api/docs/generate-content/api-errors). */
|
|
19
|
+
const GEMINI_ERRORS = {
|
|
20
|
+
'rate-limit': geminiError(429, 'RESOURCE_EXHAUSTED', 'You exceeded your current quota.', [
|
|
21
|
+
quotaFailure('GenerateRequestsPerMinutePerProjectPerModel-FreeTier'),
|
|
22
|
+
{
|
|
23
|
+
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
|
|
24
|
+
retryDelay: `${F.retryAfterSeconds}s`,
|
|
25
|
+
},
|
|
26
|
+
]),
|
|
27
|
+
quota: geminiError(429, 'RESOURCE_EXHAUSTED', 'You exceeded your current quota.', [
|
|
28
|
+
quotaFailure('GenerateRequestsPerDayPerProjectPerModel-FreeTier'),
|
|
29
|
+
{ '@type': 'type.googleapis.com/google.rpc.RetryInfo', retryDelay: '44s' },
|
|
30
|
+
]),
|
|
31
|
+
auth: geminiError(400, 'INVALID_ARGUMENT', 'API key not valid. Please pass a valid API key.', [
|
|
32
|
+
{
|
|
33
|
+
'@type': 'type.googleapis.com/google.rpc.ErrorInfo',
|
|
34
|
+
reason: 'API_KEY_INVALID',
|
|
35
|
+
domain: 'googleapis.com',
|
|
36
|
+
},
|
|
37
|
+
]),
|
|
38
|
+
overloaded: geminiError(503, 'UNAVAILABLE', 'The model is overloaded. Please try again later.'),
|
|
39
|
+
'server-error': geminiError(500, 'INTERNAL', 'An internal error has occurred.'),
|
|
40
|
+
'context-overflow': geminiError(400, 'INVALID_ARGUMENT', 'The input token count (1200000) exceeds the maximum number of tokens allowed (1048576).'),
|
|
41
|
+
};
|
|
42
|
+
const TURNS = {
|
|
43
|
+
text: { text: F.text, finishReason: 'STOP' },
|
|
44
|
+
'tool-call': { call: true, finishReason: 'STOP' },
|
|
45
|
+
'tool-call-with-stop': { call: true, finishReason: 'STOP' },
|
|
46
|
+
'truncated-tool-call': { call: true, finishReason: 'MAX_TOKENS' },
|
|
47
|
+
'content-filter': { finishReason: 'SAFETY' },
|
|
48
|
+
};
|
|
49
|
+
const USAGE = {
|
|
50
|
+
promptTokenCount: F.usage.inputTokens,
|
|
51
|
+
candidatesTokenCount: F.usage.outputTokens,
|
|
52
|
+
totalTokenCount: F.usage.inputTokens + F.usage.outputTokens,
|
|
53
|
+
};
|
|
54
|
+
const FUNCTION_CALL = {
|
|
55
|
+
functionCall: { id: F.toolCall.id, name: F.toolCall.name, args: F.toolCall.arguments },
|
|
56
|
+
};
|
|
57
|
+
function candidate(parts, finishReason) {
|
|
58
|
+
return {
|
|
59
|
+
...(parts.length > 0 && { content: { role: 'model', parts } }),
|
|
60
|
+
...(finishReason && { finishReason }),
|
|
61
|
+
index: 0,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function partsOf(turn) {
|
|
65
|
+
return [...(turn.text ? [{ text: turn.text }] : []), ...(turn.call ? [FUNCTION_CALL] : [])];
|
|
66
|
+
}
|
|
67
|
+
function stream(turn) {
|
|
68
|
+
const chunks = [];
|
|
69
|
+
if (turn.text) {
|
|
70
|
+
for (const text of F.textChunks)
|
|
71
|
+
chunks.push(sse({ candidates: [candidate([{ text }])] }));
|
|
72
|
+
}
|
|
73
|
+
chunks.push(sse({
|
|
74
|
+
candidates: [candidate(turn.call ? [FUNCTION_CALL] : [], turn.finishReason)],
|
|
75
|
+
usageMetadata: USAGE,
|
|
76
|
+
}));
|
|
77
|
+
return chunks;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The Gemini API's `generateContent` and `streamGenerateContent?alt=sse`. An error inside a
|
|
81
|
+
* stream that began is a chunk holding `{ error }`, as Google's own SDK expects it.
|
|
82
|
+
*/
|
|
83
|
+
export const geminiWire = {
|
|
84
|
+
name: 'Gemini',
|
|
85
|
+
path: (mode) => mode === 'chat' ? /\/models\/[^/]+:generateContent$/ : /:streamGenerateContent\?alt=sse$/,
|
|
86
|
+
reply(scenario, mode) {
|
|
87
|
+
const error = GEMINI_ERRORS[scenario];
|
|
88
|
+
if (error)
|
|
89
|
+
return error;
|
|
90
|
+
const streamed = (chunks, hang = false) => ({
|
|
91
|
+
type: 'stream',
|
|
92
|
+
contentType: 'text/event-stream',
|
|
93
|
+
chunks,
|
|
94
|
+
hang,
|
|
95
|
+
});
|
|
96
|
+
const started = sse({ candidates: [candidate([{ text: F.partialText }])] });
|
|
97
|
+
switch (scenario) {
|
|
98
|
+
case 'stream-error':
|
|
99
|
+
return streamed([
|
|
100
|
+
started,
|
|
101
|
+
sse({ error: { code: 503, message: 'The model is overloaded.', status: 'UNAVAILABLE' } }),
|
|
102
|
+
]);
|
|
103
|
+
case 'abort':
|
|
104
|
+
return mode === 'chat' ? { type: 'hang' } : streamed([started], true);
|
|
105
|
+
default: {
|
|
106
|
+
const turn = TURNS[scenario];
|
|
107
|
+
if (!turn)
|
|
108
|
+
return undefined;
|
|
109
|
+
return mode === 'chat'
|
|
110
|
+
? {
|
|
111
|
+
type: 'json',
|
|
112
|
+
body: {
|
|
113
|
+
candidates: [candidate(partsOf(turn), turn.finishReason)],
|
|
114
|
+
usageMetadata: USAGE,
|
|
115
|
+
},
|
|
116
|
+
}
|
|
117
|
+
: streamed(stream(turn));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type ProviderWire } from '../scenarios.js';
|
|
2
|
+
/**
|
|
3
|
+
* Ollama's native `/api/chat`, streaming newline-delimited JSON. Ollama has no content filter,
|
|
4
|
+
* and it hands out tool calls only once they are whole, so a cut-off call never reaches the
|
|
5
|
+
* client.
|
|
6
|
+
*/
|
|
7
|
+
export declare const ollamaWire: ProviderWire;
|
|
8
|
+
//# sourceMappingURL=ollama.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/ollama.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AAwDtB;;;;GAIG;AACH,eAAO,MAAM,UAAU,EAAE,YAoCxB,CAAC"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { CONFORMANCE_FIXTURE as F, } from '../scenarios.js';
|
|
2
|
+
import { ndjson } from './encoding.js';
|
|
3
|
+
const ollamaError = (status, error, headers) => ({ type: 'json', status, body: { error }, ...(headers && { headers }) });
|
|
4
|
+
/** What Ollama and Ollama Cloud answer when a request fails: a status and `{ error }`. */
|
|
5
|
+
const OLLAMA_ERRORS = {
|
|
6
|
+
'rate-limit': ollamaError(429, 'too many requests', {
|
|
7
|
+
'retry-after': String(F.retryAfterSeconds),
|
|
8
|
+
}),
|
|
9
|
+
quota: ollamaError(429, "you've reached your weekly usage limit, upgrade for higher limits"),
|
|
10
|
+
auth: ollamaError(401, 'unauthorized'),
|
|
11
|
+
overloaded: ollamaError(503, 'server busy, please try again. maximum pending requests exceeded'),
|
|
12
|
+
'server-error': ollamaError(500, 'llama runner process has terminated: exit status 2'),
|
|
13
|
+
'context-overflow': ollamaError(400, 'input length exceeds the context length'),
|
|
14
|
+
};
|
|
15
|
+
const TURNS = {
|
|
16
|
+
text: { text: F.text, doneReason: 'stop' },
|
|
17
|
+
'tool-call': { call: true, doneReason: 'stop' },
|
|
18
|
+
'tool-call-with-stop': { call: true, doneReason: 'stop' },
|
|
19
|
+
'truncated-tool-call': { doneReason: 'length' },
|
|
20
|
+
};
|
|
21
|
+
const TOOL_CALL = { function: { name: F.toolCall.name, arguments: F.toolCall.arguments } };
|
|
22
|
+
const line = (message, done = false, doneReason) => ({
|
|
23
|
+
model: 'conformance',
|
|
24
|
+
created_at: '2026-10-20T00:00:00Z',
|
|
25
|
+
message: { role: 'assistant', content: '', ...message },
|
|
26
|
+
done,
|
|
27
|
+
...(done && {
|
|
28
|
+
done_reason: doneReason,
|
|
29
|
+
prompt_eval_count: F.usage.inputTokens,
|
|
30
|
+
eval_count: F.usage.outputTokens,
|
|
31
|
+
}),
|
|
32
|
+
});
|
|
33
|
+
function stream(turn) {
|
|
34
|
+
const lines = [];
|
|
35
|
+
if (turn.text)
|
|
36
|
+
for (const content of F.textChunks)
|
|
37
|
+
lines.push(ndjson(line({ content })));
|
|
38
|
+
if (turn.call)
|
|
39
|
+
lines.push(ndjson(line({ tool_calls: [TOOL_CALL] })));
|
|
40
|
+
lines.push(ndjson(line({}, true, turn.doneReason)));
|
|
41
|
+
return lines;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Ollama's native `/api/chat`, streaming newline-delimited JSON. Ollama has no content filter,
|
|
45
|
+
* and it hands out tool calls only once they are whole, so a cut-off call never reaches the
|
|
46
|
+
* client.
|
|
47
|
+
*/
|
|
48
|
+
export const ollamaWire = {
|
|
49
|
+
name: 'Ollama',
|
|
50
|
+
path: () => /^\/api\/chat$/,
|
|
51
|
+
reply(scenario, mode) {
|
|
52
|
+
const error = OLLAMA_ERRORS[scenario];
|
|
53
|
+
if (error)
|
|
54
|
+
return error;
|
|
55
|
+
const streamed = (chunks, hang = false) => ({
|
|
56
|
+
type: 'stream',
|
|
57
|
+
contentType: 'application/x-ndjson',
|
|
58
|
+
chunks,
|
|
59
|
+
hang,
|
|
60
|
+
});
|
|
61
|
+
const started = ndjson(line({ content: F.partialText }));
|
|
62
|
+
switch (scenario) {
|
|
63
|
+
case 'stream-error':
|
|
64
|
+
return streamed([
|
|
65
|
+
started,
|
|
66
|
+
ndjson({ error: 'server busy, please try again. maximum pending requests exceeded' }),
|
|
67
|
+
]);
|
|
68
|
+
case 'abort':
|
|
69
|
+
return mode === 'chat' ? { type: 'hang' } : streamed([started], true);
|
|
70
|
+
default: {
|
|
71
|
+
const turn = TURNS[scenario];
|
|
72
|
+
if (!turn)
|
|
73
|
+
return undefined;
|
|
74
|
+
if (mode === 'stream')
|
|
75
|
+
return streamed(stream(turn));
|
|
76
|
+
return {
|
|
77
|
+
type: 'json',
|
|
78
|
+
body: line({ content: turn.text ?? '', ...(turn.call && { tool_calls: [TOOL_CALL] }) }, true, turn.doneReason),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type ProviderWire } from '../scenarios.js';
|
|
2
|
+
/**
|
|
3
|
+
* The Chat Completions API of OpenAI and of the servers compatible with it, at any path ending
|
|
4
|
+
* in `/chat/completions`. Errors inside a stream come as a `data: {"error": ...}` chunk.
|
|
5
|
+
*/
|
|
6
|
+
export declare const openAIChatWire: ProviderWire;
|
|
7
|
+
/**
|
|
8
|
+
* The OpenAI Responses API at `/responses`. An error inside a stream is an `error` event.
|
|
9
|
+
*/
|
|
10
|
+
export declare const openAIResponsesWire: ProviderWire;
|
|
11
|
+
//# sourceMappingURL=openai.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/openai.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AA2LtB;;;GAGG;AACH,eAAO,MAAM,cAAc,EAAE,YA+B5B,CAAC;AA8GF;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,YA2DjC,CAAC"}
|