@ultimat3/ai 27.9.0 → 27.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +104 -108
- package/README.md +21 -5
- package/package.json +11 -11
- package/src/agent-continue.ts +434 -0
- package/src/agent-declared.ts +157 -0
- package/src/agent-def.ts +28 -5
- package/src/agent-errors.ts +20 -1
- package/src/agent-turn.ts +54 -0
- package/src/agent-usage.ts +49 -13
- package/src/agent.ts +163 -193
- package/src/answer-errors.ts +88 -0
- package/src/billed.ts +87 -0
- package/src/budget.ts +37 -10
- package/src/cost.ts +146 -0
- package/src/echo-provider.ts +3 -1
- package/src/errors-retry.ts +8 -0
- package/src/errors.ts +22 -19
- package/src/failure-disclosure.ts +37 -1
- package/src/gateway.ts +53 -7
- package/src/in-process-tool.ts +77 -0
- package/src/index.ts +29 -3
- package/src/llm-def.ts +97 -0
- package/src/llm.ts +93 -75
- package/src/models.ts +45 -1
- package/src/openai-body.ts +6 -3
- package/src/openai-provider.ts +11 -2
- package/src/openai-wire.ts +168 -62
- package/src/provider.ts +32 -34
- package/src/run-meter.ts +147 -0
- package/src/sampling.ts +113 -0
- package/src/tool-arguments.ts +88 -0
- package/src/tools.ts +51 -6
- package/src/wire.ts +15 -14
package/src/models.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { assert, finiteCount, renderCauseValue, renderFixLiteral } from '@ultimat3/core';
|
|
7
7
|
import type { Money } from '@ultimat3/money';
|
|
8
|
+
import { exponentOf } from '@ultimat3/money';
|
|
8
9
|
import { AiModelUnknownError, AiRequestInvalidError } from './errors';
|
|
9
10
|
|
|
10
11
|
/**
|
|
@@ -59,11 +60,18 @@ export interface ModelReasoning {
|
|
|
59
60
|
readonly disabledThinking?: 'disabled' | 'between_tools';
|
|
60
61
|
}
|
|
61
62
|
|
|
63
|
+
/** Digits a price — and the cost computed from it — may carry beyond the currency's minor unit. */
|
|
64
|
+
export const PRICE_SCALE_DIGITS = 6;
|
|
65
|
+
|
|
62
66
|
export interface ModelSpec {
|
|
63
67
|
readonly id: ModelId;
|
|
64
68
|
readonly contextWindow: number;
|
|
65
69
|
readonly maxOutput: number;
|
|
66
|
-
/**
|
|
70
|
+
/**
|
|
71
|
+
* Cost of one million input tokens: minor units, or — for a price below one minor unit per
|
|
72
|
+
* million — `Money` at a finer `scale`: `{ minor: 75, currency: 'USD', scale: 3 }` is $0.075.
|
|
73
|
+
* The same holds for the three prices below.
|
|
74
|
+
*/
|
|
67
75
|
readonly inputPerMillion: Money;
|
|
68
76
|
/** Cost of one million output tokens, in minor units. */
|
|
69
77
|
readonly outputPerMillion: Money;
|
|
@@ -82,6 +90,18 @@ export interface ModelSpec {
|
|
|
82
90
|
/** Minimum cacheable prefix; a shorter prefix silently does not cache. */
|
|
83
91
|
readonly cacheMinimumTokens: number;
|
|
84
92
|
readonly reasoning: ModelReasoning;
|
|
93
|
+
/**
|
|
94
|
+
* The sampling knobs this model TAKES (`./sampling`). Absent is none: a `temperature` or `seed` a
|
|
95
|
+
* request asks for is then not sent, because a reasoning model answers one with a 400. A row for
|
|
96
|
+
* a model that does take them says `{ temperature: true, seed: true }`.
|
|
97
|
+
*/
|
|
98
|
+
readonly sampling?: { readonly temperature?: boolean; readonly seed?: boolean };
|
|
99
|
+
/**
|
|
100
|
+
* `false` for a model that rejects a FORCED tool choice (a 400 naming `tool_choice`, which
|
|
101
|
+
* always-thinking models answer). `llm()`'s one `respond` tool is then offered under the
|
|
102
|
+
* endpoint's default, and a prose answer takes the repair turn. Absent is forced, as before.
|
|
103
|
+
*/
|
|
104
|
+
readonly forcedToolChoice?: boolean;
|
|
85
105
|
/**
|
|
86
106
|
* The non-text blocks a message to this model may carry. Absent is TEXT ONLY, deliberately: an
|
|
87
107
|
* app's own row for a text-only endpoint must not have an image sent to it, and the refusal
|
|
@@ -147,10 +167,34 @@ export function registerModel(spec: ModelSpec): ModelSpec {
|
|
|
147
167
|
if (spec.cacheWritePerMillion !== undefined) {
|
|
148
168
|
finiteCount(SUBJECT, `${spec.id} cacheWritePerMillion.minor`, spec.cacheWritePerMillion.minor);
|
|
149
169
|
}
|
|
170
|
+
assertPriceScales(spec);
|
|
150
171
|
registry.set(spec.id, spec);
|
|
151
172
|
return spec;
|
|
152
173
|
}
|
|
153
174
|
|
|
175
|
+
/**
|
|
176
|
+
* A price may state a `scale` finer than the currency's minor unit — `{ minor: 75, currency:
|
|
177
|
+
* 'USD', scale: 3 }` is $0.075 per million — up to the scale a COST is carried at. Finer than that
|
|
178
|
+
* cannot be priced exactly, so the row is refused at boot instead of rounded at every call.
|
|
179
|
+
*/
|
|
180
|
+
function assertPriceScales(spec: ModelSpec): void {
|
|
181
|
+
const prices = {
|
|
182
|
+
inputPerMillion: spec.inputPerMillion,
|
|
183
|
+
outputPerMillion: spec.outputPerMillion,
|
|
184
|
+
cacheReadPerMillion: spec.cacheReadPerMillion,
|
|
185
|
+
cacheWritePerMillion: spec.cacheWritePerMillion,
|
|
186
|
+
};
|
|
187
|
+
for (const [field, price] of Object.entries(prices)) {
|
|
188
|
+
if (price?.scale === undefined) continue;
|
|
189
|
+
const finest = exponentOf(price.currency) + PRICE_SCALE_DIGITS;
|
|
190
|
+
assert(
|
|
191
|
+
Number.isInteger(price.scale) && price.scale >= 0 && price.scale <= finest,
|
|
192
|
+
`${SUBJECT} row "${spec.id}" states ${field} at scale ${renderCauseValue(price.scale)}; a price in ${renderCauseValue(price.currency)} may be stated at scale 0 to ${finest}`,
|
|
193
|
+
`registerModel({ ...spec, ${field}: { minor, currency: ${renderFixLiteral(price.currency, '<currency>')}, scale: ${finest} } }) # at most ${PRICE_SCALE_DIGITS} digits finer than the minor unit`,
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
154
198
|
/**
|
|
155
199
|
* Every price on a row in ONE currency. `costOf` sums them into a single `Money` stamped with the
|
|
156
200
|
* input price's currency, so a cache rate in another one would be added as if it were the same.
|
package/src/openai-body.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { modelSpec } from './models';
|
|
|
11
11
|
import { toOpenAiMessages, toOpenAiTools, toolChoiceFor } from './openai-messages';
|
|
12
12
|
import { OPENAI_CACHE_KEY_MAX, promptCacheRule, SESSION_KEY_MAX } from './prompt-cache';
|
|
13
13
|
import type { GenerateRequest } from './provider';
|
|
14
|
+
import { samplingBody } from './sampling';
|
|
14
15
|
|
|
15
16
|
export interface ChatCompletionBodyInput {
|
|
16
17
|
readonly request: GenerateRequest;
|
|
@@ -25,8 +26,8 @@ export interface ChatCompletionBodyInput {
|
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* The body. What is deliberately absent is as load-bearing as what is present:
|
|
28
|
-
* - no `
|
|
29
|
-
* knob on a reasoning model in this family is a 400.
|
|
29
|
+
* - no `top_p` / `presence_penalty`, and `temperature` / `seed` only for a model whose row takes
|
|
30
|
+
* them (`ModelSpec.sampling`): a sampling knob on a reasoning model in this family is a 400.
|
|
30
31
|
* - no `response_format`. The output schema is projected into the `respond` tool by `llm()`, and
|
|
31
32
|
* that projection is the framework's ONE structured-output path — see the README.
|
|
32
33
|
* - `max_completion_tokens`, never the deprecated `max_tokens`, which current reasoning models
|
|
@@ -44,10 +45,12 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record<strin
|
|
|
44
45
|
messages: toOpenAiMessages(request.system, request.messages, { provider, model }, marked),
|
|
45
46
|
max_completion_tokens: Math.min(request.maxTokens, spec.maxOutput),
|
|
46
47
|
...reasoningFields(model, request.effort, request.thinking),
|
|
48
|
+
...samplingBody('openai', model, request),
|
|
47
49
|
};
|
|
48
50
|
if (request.tools !== undefined && request.tools.length > 0) {
|
|
49
51
|
body['tools'] = toOpenAiTools(request.tools);
|
|
50
|
-
|
|
52
|
+
// Forced unless the row says this model rejects a forced choice (`forcedToolChoice: false`).
|
|
53
|
+
const choice = spec.forcedToolChoice === false ? undefined : toolChoiceFor(request.tools);
|
|
51
54
|
if (choice !== undefined) body['tool_choice'] = choice;
|
|
52
55
|
}
|
|
53
56
|
if (request.stopSequences !== undefined && request.stopSequences.length > 0) {
|
package/src/openai-provider.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import type { Secret } from '@ultimat3/core';
|
|
11
11
|
import { isSecret, revealSecret } from '@ultimat3/core';
|
|
12
|
+
import { costOf, exactCostOf } from './cost';
|
|
12
13
|
import { detailOf, withoutKey } from './error-body';
|
|
13
14
|
import { AiKeyMissingError, AiRequestInvalidError, AiTransportError } from './errors';
|
|
14
15
|
import type { AiFetch } from './fetch-seam';
|
|
@@ -25,8 +26,9 @@ import type {
|
|
|
25
26
|
StreamChunk,
|
|
26
27
|
TokenUsage,
|
|
27
28
|
} from './provider';
|
|
28
|
-
import {
|
|
29
|
+
import { estimateInputTokens, estimateTextTokens, requiresStreaming } from './provider';
|
|
29
30
|
import { readSse } from './sse';
|
|
31
|
+
import { assertToolArguments } from './tool-arguments';
|
|
30
32
|
import type { LlmToolCall } from './tools';
|
|
31
33
|
|
|
32
34
|
const API_KEY_ENV = 'OPENAI_API_KEY';
|
|
@@ -85,6 +87,7 @@ export function openAiProvider(input: OpenAiProviderInput): Provider {
|
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
class OpenAiProvider implements Provider {
|
|
90
|
+
readonly wire = 'openai' as const;
|
|
88
91
|
readonly name: string;
|
|
89
92
|
readonly models: readonly ModelId[];
|
|
90
93
|
private readonly config: OpenAiProviderInput;
|
|
@@ -151,6 +154,8 @@ class OpenAiProvider implements Provider {
|
|
|
151
154
|
for await (const frame of readSse(response.body, this.name)) {
|
|
152
155
|
for (const chunk of completion.push(frame)) yield chunk;
|
|
153
156
|
}
|
|
157
|
+
// The socket closed: the calls of a finished turn, for a server that never sends `[DONE]`.
|
|
158
|
+
for (const chunk of completion.end()) yield chunk;
|
|
154
159
|
// A connection cut mid-answer must fail, not resolve: partial text reads as a complete answer,
|
|
155
160
|
// and `end_turn` would be a lie the caller has no way to detect.
|
|
156
161
|
if (!completion.isComplete()) {
|
|
@@ -176,6 +181,9 @@ class OpenAiProvider implements Provider {
|
|
|
176
181
|
/** One parsed answer, priced. The only place `cost` is applied — the provider owns prices. */
|
|
177
182
|
private result(request: GenerateRequest, model: ModelId, answer: ChatAnswer): GenerateResult {
|
|
178
183
|
const usage = answer.usage ?? estimatedUsage(request, answer.text, answer.toolCalls);
|
|
184
|
+
const cost = costOf(model, usage);
|
|
185
|
+
// After usage is known, so a finished turn with unreadable arguments fails WITH its bill.
|
|
186
|
+
assertToolArguments({ provider: this.name, model, ...answer, usage, cost });
|
|
179
187
|
return {
|
|
180
188
|
model,
|
|
181
189
|
text: answer.text,
|
|
@@ -183,7 +191,8 @@ class OpenAiProvider implements Provider {
|
|
|
183
191
|
stopReason: answer.stopReason,
|
|
184
192
|
stopDetails: answer.stopDetails,
|
|
185
193
|
usage,
|
|
186
|
-
cost
|
|
194
|
+
cost,
|
|
195
|
+
costExact: exactCostOf(model, usage),
|
|
187
196
|
};
|
|
188
197
|
}
|
|
189
198
|
|
package/src/openai-wire.ts
CHANGED
|
@@ -10,6 +10,8 @@ import { renderThrowable } from '@ultimat3/core';
|
|
|
10
10
|
import { AiTransportError } from './errors';
|
|
11
11
|
import type { StopDetails, StopReason, StreamChunk, TokenUsage } from './provider';
|
|
12
12
|
import type { SseFrame } from './sse';
|
|
13
|
+
import type { MalformedToolCall } from './tool-arguments';
|
|
14
|
+
import { parseToolArguments } from './tool-arguments';
|
|
13
15
|
import type { LlmToolCall } from './tools';
|
|
14
16
|
|
|
15
17
|
/**
|
|
@@ -20,9 +22,16 @@ import type { LlmToolCall } from './tools';
|
|
|
20
22
|
export interface ChatAnswer {
|
|
21
23
|
readonly text: string;
|
|
22
24
|
readonly toolCalls: readonly LlmToolCall[];
|
|
25
|
+
/**
|
|
26
|
+
* Calls whose arguments were not JSON. Never among `toolCalls` — a fragment is never a call —
|
|
27
|
+
* and what they mean is the provider's decision (`assertToolArguments`), once usage is known.
|
|
28
|
+
*/
|
|
29
|
+
readonly malformed: readonly MalformedToolCall[];
|
|
23
30
|
readonly stopReason: StopReason;
|
|
24
31
|
readonly stopDetails: StopDetails | undefined;
|
|
25
32
|
readonly usage: TokenUsage | undefined;
|
|
33
|
+
/** The raw finish reason when this build does not know it — `stopReason` is then read off the content. */
|
|
34
|
+
readonly unknownFinish?: string;
|
|
26
35
|
}
|
|
27
36
|
|
|
28
37
|
// A `Map`, not an object literal: `raw` is the PROVIDER's string on every read, and
|
|
@@ -33,14 +42,46 @@ export interface ChatAnswer {
|
|
|
33
42
|
const FINISH_REASONS: ReadonlyMap<string, StopReason> = new Map(
|
|
34
43
|
Object.entries({
|
|
35
44
|
stop: 'end_turn',
|
|
45
|
+
// Synonyms for "the model finished", as endpoints in the family spell it: the Messages API's
|
|
46
|
+
// word passed through by a gateway, a tokenizer's end-of-sequence, Gemini's `STOP`.
|
|
47
|
+
end_turn: 'end_turn',
|
|
48
|
+
eos: 'end_turn',
|
|
49
|
+
eos_token: 'end_turn',
|
|
50
|
+
stop_sequence: 'stop_sequence',
|
|
36
51
|
length: 'max_tokens',
|
|
52
|
+
max_tokens: 'max_tokens',
|
|
53
|
+
model_context_window_exceeded: 'model_context_window_exceeded',
|
|
37
54
|
tool_calls: 'tool_use',
|
|
38
55
|
// The legacy name for the same event; LiteLLM and older self-hosted servers still send it.
|
|
39
56
|
function_call: 'tool_use',
|
|
57
|
+
tool_use: 'tool_use',
|
|
40
58
|
content_filter: 'refusal',
|
|
41
59
|
} as const),
|
|
42
60
|
);
|
|
43
61
|
|
|
62
|
+
/** What a `finish_reason` field said: nothing yet, a reason this build knows, or one it does not. */
|
|
63
|
+
export type FinishReading =
|
|
64
|
+
| { readonly kind: 'none' }
|
|
65
|
+
| { readonly kind: 'known'; readonly reason: StopReason }
|
|
66
|
+
| { readonly kind: 'unknown'; readonly raw: string };
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Read a finish reason, case-insensitively (`STOP`, `MAX_TOKENS`). A non-string (`null` on a
|
|
70
|
+
* delta) is "still going". A string this table does not list is UNKNOWN, and deliberately not
|
|
71
|
+
* truncation: `llm()` and `agent()` refuse a truncated turn before reading it, so guessing
|
|
72
|
+
* "cut off" fails every call of an endpoint that merely spells "done" its own way. What an unknown
|
|
73
|
+
* reason amounts to is decided from the answer itself (`unknownFinishReason`), by the provider.
|
|
74
|
+
*/
|
|
75
|
+
export function readFinishReason(raw: unknown): FinishReading {
|
|
76
|
+
if (typeof raw !== 'string') return { kind: 'none' };
|
|
77
|
+
const reason = FINISH_REASONS.get(raw) ?? FINISH_REASONS.get(raw.toLowerCase());
|
|
78
|
+
return reason === undefined ? { kind: 'unknown', raw } : { kind: 'known', reason };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The stop reason of a turn whose finish reason is unknown: what its content shows it to be. */
|
|
82
|
+
const settledAs = (calls: readonly LlmToolCall[]): StopReason =>
|
|
83
|
+
calls.length > 0 ? 'tool_use' : 'end_turn';
|
|
84
|
+
|
|
44
85
|
/**
|
|
45
86
|
* In-band error frames carry a type, not a status, and the gateway's retry rule reads a status.
|
|
46
87
|
* Same mapping job as wire.ts's, over this format's own vocabulary.
|
|
@@ -63,13 +104,13 @@ const ERROR_STATUS: ReadonlyMap<string, number> = new Map(
|
|
|
63
104
|
);
|
|
64
105
|
|
|
65
106
|
/**
|
|
66
|
-
* The finish reason
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* likelier to be a way of stopping early than of finishing.
|
|
107
|
+
* The finish reason as a `StopReason`, or `undefined` for a non-string. An UNKNOWN string reads
|
|
108
|
+
* `end_turn` here — the bare lookup has no answer to judge it by; the two readers below do, and
|
|
109
|
+
* report the raw reason as `unknownFinish` so the provider can refuse a turn that is not whole.
|
|
70
110
|
*/
|
|
71
111
|
export function parseFinishReason(raw: unknown): StopReason | undefined {
|
|
72
|
-
|
|
112
|
+
const read = readFinishReason(raw);
|
|
113
|
+
return read.kind === 'none' ? undefined : read.kind === 'known' ? read.reason : 'end_turn';
|
|
73
114
|
}
|
|
74
115
|
|
|
75
116
|
/**
|
|
@@ -117,15 +158,24 @@ export function parseChatCompletion(raw: unknown, provider: string): ChatAnswer
|
|
|
117
158
|
// `"refusal": ""` beside a real answer, and that body must not refuse here and answer streamed.
|
|
118
159
|
const declined = message['refusal'];
|
|
119
160
|
const refusal = typeof declined === 'string' && declined !== '' ? declined : undefined;
|
|
120
|
-
const
|
|
161
|
+
const read = parseToolCalls(message['tool_calls']);
|
|
162
|
+
const reading = readFinishReason(choice['finish_reason']);
|
|
163
|
+
const finish =
|
|
164
|
+
reading.kind === 'none'
|
|
165
|
+
? undefined
|
|
166
|
+
: reading.kind === 'known'
|
|
167
|
+
? reading.reason
|
|
168
|
+
: settledAs(read.calls);
|
|
121
169
|
return {
|
|
122
170
|
text: typeof message['content'] === 'string' ? message['content'] : '',
|
|
123
|
-
toolCalls:
|
|
171
|
+
toolCalls: read.calls,
|
|
172
|
+
malformed: read.malformed,
|
|
124
173
|
// A refusal string is a refusal whatever the finish reason says: the field only ever appears
|
|
125
174
|
// when the model declined, and `stop` beside it would read downstream as an empty answer.
|
|
126
175
|
stopReason: refusal === undefined ? (finish ?? 'end_turn') : 'refusal',
|
|
127
176
|
stopDetails: refusalDetails(finish, refusal),
|
|
128
177
|
usage: parseOpenAiUsage(record['usage']),
|
|
178
|
+
...(reading.kind === 'unknown' ? { unknownFinish: reading.raw } : {}),
|
|
129
179
|
};
|
|
130
180
|
}
|
|
131
181
|
|
|
@@ -140,41 +190,52 @@ function refusalDetails(
|
|
|
140
190
|
return { type: 'refusal', category: 'content_filter', explanation: undefined };
|
|
141
191
|
}
|
|
142
192
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
193
|
+
interface ReadCalls {
|
|
194
|
+
readonly calls: LlmToolCall[];
|
|
195
|
+
readonly malformed: MalformedToolCall[];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function parseToolCalls(raw: unknown): ReadCalls {
|
|
199
|
+
const read: ReadCalls = { calls: [], malformed: [] };
|
|
200
|
+
if (!Array.isArray(raw)) return read;
|
|
146
201
|
for (const entry of raw) {
|
|
147
202
|
const record = asRecord(entry);
|
|
148
203
|
const fn = asRecord(record?.['function']);
|
|
149
204
|
if (record === undefined || fn === undefined) continue;
|
|
150
205
|
const name = typeof fn['name'] === 'string' ? fn['name'] : '';
|
|
151
206
|
if (name === '') continue;
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
name,
|
|
155
|
-
input: parseArguments(fn['arguments'], name, provider),
|
|
156
|
-
});
|
|
207
|
+
const args = fn['arguments'];
|
|
208
|
+
collect(read, typeof record['id'] === 'string' ? record['id'] : '', name, args);
|
|
157
209
|
}
|
|
158
|
-
return
|
|
210
|
+
return read;
|
|
159
211
|
}
|
|
160
212
|
|
|
161
|
-
/**
|
|
162
|
-
function
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
213
|
+
/** One call, read: among the calls when its arguments parse, among the malformed when not. */
|
|
214
|
+
function collect(
|
|
215
|
+
read: ReadCalls,
|
|
216
|
+
id: string,
|
|
217
|
+
name: string,
|
|
218
|
+
args: unknown,
|
|
219
|
+
): LlmToolCall | undefined {
|
|
220
|
+
const raw = typeof args === 'string' ? args : '';
|
|
221
|
+
const input = parseToolArguments(raw);
|
|
222
|
+
if (input === undefined) {
|
|
223
|
+
read.malformed.push({ name, chars: raw.length });
|
|
224
|
+
return undefined;
|
|
171
225
|
}
|
|
226
|
+
const call: LlmToolCall = { id, name, input };
|
|
227
|
+
read.calls.push(call);
|
|
228
|
+
return call;
|
|
172
229
|
}
|
|
173
230
|
|
|
174
231
|
interface PendingCall {
|
|
232
|
+
/** Where the call sorts among its turn's: the wire's `index`, else its order of arrival. */
|
|
233
|
+
readonly index: number;
|
|
175
234
|
id: string;
|
|
176
235
|
name: string;
|
|
177
236
|
args: string;
|
|
237
|
+
/** A second NAMED fragment arrived with no index and no id: two calls, indistinguishable. */
|
|
238
|
+
unkeyed?: boolean;
|
|
178
239
|
}
|
|
179
240
|
|
|
180
241
|
/**
|
|
@@ -189,11 +250,14 @@ export class ChatCompletionStream {
|
|
|
189
250
|
private text = '';
|
|
190
251
|
private refusal = '';
|
|
191
252
|
private stopReason: StopReason = 'end_turn';
|
|
253
|
+
private unknownFinish: string | undefined;
|
|
192
254
|
private finished = false;
|
|
193
255
|
private done = false;
|
|
194
256
|
private usage: TokenUsage | undefined;
|
|
195
|
-
|
|
196
|
-
private readonly
|
|
257
|
+
/** Every open call, in order of arrival; `byIndex` is which one a wire index currently means. */
|
|
258
|
+
private readonly pending: PendingCall[] = [];
|
|
259
|
+
private readonly byIndex = new Map<number, PendingCall>();
|
|
260
|
+
private readonly read: ReadCalls = { calls: [], malformed: [] };
|
|
197
261
|
private readonly provider: string;
|
|
198
262
|
|
|
199
263
|
constructor(provider: string) {
|
|
@@ -205,7 +269,7 @@ export class ChatCompletionStream {
|
|
|
205
269
|
// The sentinel is not JSON, and parsing it is how a stream reader ends in a syntax error.
|
|
206
270
|
if (frame.data.trim() === '[DONE]') {
|
|
207
271
|
this.done = true;
|
|
208
|
-
return
|
|
272
|
+
return this.end();
|
|
209
273
|
}
|
|
210
274
|
const payload = this.payloadOf(frame);
|
|
211
275
|
throwInBandError(payload, this.provider);
|
|
@@ -214,7 +278,34 @@ export class ChatCompletionStream {
|
|
|
214
278
|
const choice = asRecord(Array.isArray(payload['choices']) ? payload['choices'][0] : undefined);
|
|
215
279
|
if (choice === undefined) return [];
|
|
216
280
|
const chunks = this.onDelta(asRecord(choice['delta']) ?? {});
|
|
217
|
-
|
|
281
|
+
this.onFinish(choice['finish_reason']);
|
|
282
|
+
return chunks;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* The stream is over: the calls a finished turn made, whole, in index order. Called by the
|
|
287
|
+
* provider when the socket closes, and by `[DONE]` — whichever comes first; the second is empty.
|
|
288
|
+
*
|
|
289
|
+
* Here, and not when the finish reason arrives: this format has no per-call stop event, and
|
|
290
|
+
* nothing in it orders the finish reason against the last argument fragment — a call read at the
|
|
291
|
+
* reason is read half way whenever a gateway sends the reason first. A turn with no finish
|
|
292
|
+
* reason drains nothing: its fragments stay open, and `isComplete()` refuses the stream.
|
|
293
|
+
*/
|
|
294
|
+
end(): readonly StreamChunk[] {
|
|
295
|
+
if (!this.finished) return [];
|
|
296
|
+
const chunks: StreamChunk[] = [];
|
|
297
|
+
for (const pending of this.pending.splice(0).sort((a, b) => a.index - b.index)) {
|
|
298
|
+
if (pending.name === '') continue;
|
|
299
|
+
if (pending.unkeyed === true) {
|
|
300
|
+
// Deliberate: merged arguments are never parsed, even when they happen to be JSON.
|
|
301
|
+
this.read.malformed.push({ name: pending.name, chars: pending.args.length, unkeyed: true });
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
const call = collect(this.read, pending.id, pending.name, pending.args);
|
|
305
|
+
if (call !== undefined) chunks.push({ type: 'tool-call', call });
|
|
306
|
+
}
|
|
307
|
+
this.byIndex.clear();
|
|
308
|
+
return chunks;
|
|
218
309
|
}
|
|
219
310
|
|
|
220
311
|
/** True once the answer is accounted for. False means the connection died mid-answer. */
|
|
@@ -232,20 +323,29 @@ export class ChatCompletionStream {
|
|
|
232
323
|
// half a JSON object, and would report `end_turn` for a turn that stopped to call one. So the
|
|
233
324
|
// stream is refused exactly as the Anthropic half refuses a missing `message_stop`
|
|
234
325
|
// (`provider.ts`), and `openai-provider.ts`'s existing truncation guard is what raises it.
|
|
235
|
-
return this.finished || (this.done && this.pending.
|
|
326
|
+
return this.finished || (this.done && this.pending.length === 0);
|
|
236
327
|
}
|
|
237
328
|
|
|
238
329
|
/** What the stream accumulated. `cost` is applied by the provider, which owns prices. */
|
|
239
330
|
state(): ChatAnswer {
|
|
331
|
+
// Idempotent, so a reader that never called `end()` still sees a finished turn's calls.
|
|
332
|
+
this.end();
|
|
240
333
|
const details = this.refusalDetails();
|
|
241
334
|
return {
|
|
242
335
|
text: this.text,
|
|
243
|
-
toolCalls: [...this.
|
|
336
|
+
toolCalls: [...this.read.calls],
|
|
337
|
+
malformed: [...this.read.malformed],
|
|
244
338
|
// A refusal that arrived as a `refusal` delta finishes with `stop`, so the reason alone would
|
|
245
339
|
// read as a complete answer that happens to be empty.
|
|
246
|
-
stopReason:
|
|
340
|
+
stopReason:
|
|
341
|
+
details !== undefined
|
|
342
|
+
? 'refusal'
|
|
343
|
+
: this.unknownFinish === undefined
|
|
344
|
+
? this.stopReason
|
|
345
|
+
: settledAs(this.read.calls),
|
|
247
346
|
stopDetails: details,
|
|
248
347
|
usage: this.usage,
|
|
348
|
+
...(this.unknownFinish === undefined ? {} : { unknownFinish: this.unknownFinish }),
|
|
249
349
|
};
|
|
250
350
|
}
|
|
251
351
|
|
|
@@ -279,54 +379,60 @@ export class ChatCompletionStream {
|
|
|
279
379
|
}
|
|
280
380
|
|
|
281
381
|
/**
|
|
282
|
-
* Merge one chunk's tool-call fragments into the calls they belong to. `index` is the
|
|
382
|
+
* Merge one chunk's tool-call fragments into the calls they belong to. `index` is the key —
|
|
283
383
|
* `id` and `name` arrive on the first fragment and are absent from every later one, so appending
|
|
284
384
|
* by array position instead would build one call per chunk and lose every argument but the last.
|
|
385
|
+
*
|
|
386
|
+
* It is not the WHOLE key. A gateway translating another vendor's stream may number every call
|
|
387
|
+
* of a turn `0` (or send no index) and tell them apart by id alone; merged by index, two
|
|
388
|
+
* well-formed calls became one string — `{…}{…}` — that is not JSON. So a fragment carrying an id
|
|
389
|
+
* that is not the open call's starts a new call. An id repeated on every fragment, or absent
|
|
390
|
+
* from all but the first, is still one call.
|
|
285
391
|
*/
|
|
286
392
|
private accumulate(raw: unknown): void {
|
|
287
393
|
if (!Array.isArray(raw)) return;
|
|
288
394
|
for (const entry of raw) {
|
|
289
395
|
const record = asRecord(entry);
|
|
290
396
|
if (record === undefined) continue;
|
|
291
|
-
const index = numberOf(record['index'])
|
|
292
|
-
const
|
|
293
|
-
const id =
|
|
294
|
-
|
|
397
|
+
const index = numberOf(record['index']);
|
|
398
|
+
const rawId = record['id'];
|
|
399
|
+
const id = typeof rawId === 'string' && rawId !== '' ? rawId : undefined;
|
|
400
|
+
let call = index === undefined ? this.pending.at(-1) : this.byIndex.get(index);
|
|
401
|
+
if (call === undefined || (id !== undefined && call.id !== '' && call.id !== id)) {
|
|
402
|
+
call = { index: index ?? this.pending.length, id: '', name: '', args: '' };
|
|
403
|
+
this.pending.push(call);
|
|
404
|
+
}
|
|
405
|
+
if (index !== undefined) this.byIndex.set(index, call);
|
|
406
|
+
if (id !== undefined) call.id = id;
|
|
295
407
|
const fn = asRecord(record['function']);
|
|
296
408
|
const name = fn?.['name'];
|
|
409
|
+
// No index, no id, and a tool NAMED again after arguments already arrived: a second call
|
|
410
|
+
// this stream gives no way to tell from the first. A name split across frames arrives
|
|
411
|
+
// before any argument, so that legal case is not this one.
|
|
412
|
+
if (
|
|
413
|
+
index === undefined &&
|
|
414
|
+
id === undefined &&
|
|
415
|
+
call.args !== '' &&
|
|
416
|
+
name !== undefined &&
|
|
417
|
+
name !== ''
|
|
418
|
+
) {
|
|
419
|
+
call.unkeyed = true;
|
|
420
|
+
}
|
|
297
421
|
// Concatenated, not assigned: a server that splits the name across two frames is rare and
|
|
298
422
|
// legal, and assigning would keep only the tail.
|
|
299
423
|
if (typeof name === 'string') call.name += name;
|
|
300
424
|
const args = fn?.['arguments'];
|
|
301
425
|
if (typeof args === 'string') call.args += args;
|
|
302
|
-
this.pending.set(index, call);
|
|
303
426
|
}
|
|
304
427
|
}
|
|
305
428
|
|
|
306
|
-
/**
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const reason = parseFinishReason(raw);
|
|
313
|
-
if (reason === undefined) return [];
|
|
314
|
-
this.stopReason = reason;
|
|
429
|
+
/** The finish reason closes the TURN. The calls are read by `end()`, when the stream is over. */
|
|
430
|
+
private onFinish(raw: unknown): void {
|
|
431
|
+
const reading = readFinishReason(raw);
|
|
432
|
+
if (reading.kind === 'none') return;
|
|
433
|
+
if (reading.kind === 'known') this.stopReason = reading.reason;
|
|
434
|
+
else this.unknownFinish = reading.raw;
|
|
315
435
|
this.finished = true;
|
|
316
|
-
const chunks: StreamChunk[] = [];
|
|
317
|
-
for (const index of [...this.pending.keys()].sort((a, b) => a - b)) {
|
|
318
|
-
const pending = this.pending.get(index);
|
|
319
|
-
if (pending === undefined || pending.name === '') continue;
|
|
320
|
-
const call: LlmToolCall = {
|
|
321
|
-
id: pending.id,
|
|
322
|
-
name: pending.name,
|
|
323
|
-
input: parseArguments(pending.args, pending.name, this.provider),
|
|
324
|
-
};
|
|
325
|
-
this.toolCalls.push(call);
|
|
326
|
-
chunks.push({ type: 'tool-call', call });
|
|
327
|
-
}
|
|
328
|
-
this.pending.clear();
|
|
329
|
-
return chunks;
|
|
330
436
|
}
|
|
331
437
|
|
|
332
438
|
private payloadOf(frame: SseFrame): Record<string, unknown> {
|