@ultimat3/ai 30.9.0 → 31.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +20 -11
- package/README.md +47 -22
- package/package.json +13 -12
- package/src/adapter.ts +21 -0
- package/src/agent-compact.ts +5 -2
- package/src/agent-continue.ts +4 -69
- package/src/agent-def.ts +6 -1
- package/src/agent-session.ts +66 -0
- package/src/agent-system.ts +1 -1
- package/src/agent-tools.ts +123 -0
- package/src/agent-turn.ts +14 -4
- package/src/agent.ts +43 -88
- package/src/budget-store.ts +82 -0
- package/src/budget.ts +7 -78
- package/src/embeddings.ts +1 -1
- package/src/errors.ts +5 -5
- package/src/eval-agent.ts +1 -1
- package/src/gateway-providers.ts +22 -10
- package/src/gateway.ts +20 -21
- package/src/index.ts +6 -71
- package/src/llm-answer.ts +125 -0
- package/src/llm-cache.ts +16 -2
- package/src/llm-def.ts +1 -1
- package/src/llm-instrument.ts +100 -0
- package/src/llm.ts +152 -241
- package/src/openai-provider.ts +2 -1
- package/src/openai-wire-stream.ts +243 -0
- package/src/openai-wire.ts +9 -236
- package/src/pg-vector-sql.ts +30 -2
- package/src/provider.ts +1 -1
- package/src/runtime.ts +83 -38
- package/src/sampling.ts +4 -4
- package/src/scorers.ts +1 -1
- package/src/tools.ts +12 -8
- package/src/vector.ts +3 -1
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// Single responsibility: one streamed OpenAI-format chat completion, assembled frame by frame —
|
|
2
|
+
// fragmented, indexed tool calls and the trailing usage chunk. Split from ./openai-wire (the
|
|
3
|
+
// non-streamed answer and the field readers both share) at its line ceiling.
|
|
4
|
+
|
|
5
|
+
import { renderThrowable } from '@ultimat3/core';
|
|
6
|
+
import type { ChatAnswer, ReadCalls } from './openai-wire';
|
|
7
|
+
import {
|
|
8
|
+
asRecord,
|
|
9
|
+
collect,
|
|
10
|
+
malformed,
|
|
11
|
+
numberOf,
|
|
12
|
+
parseOpenAiUsage,
|
|
13
|
+
readFinishReason,
|
|
14
|
+
settledAs,
|
|
15
|
+
throwInBandError,
|
|
16
|
+
} from './openai-wire';
|
|
17
|
+
import type { StopDetails, StopReason, StreamChunk, TokenUsage } from './provider';
|
|
18
|
+
import type { SseFrame } from './sse';
|
|
19
|
+
|
|
20
|
+
interface PendingCall {
|
|
21
|
+
/** Where the call sorts among its turn's: the wire's `index`, else its order of arrival. */
|
|
22
|
+
readonly index: number;
|
|
23
|
+
id: string;
|
|
24
|
+
name: string;
|
|
25
|
+
args: string;
|
|
26
|
+
/** A second NAMED fragment arrived with no index and no id: two calls, indistinguishable. */
|
|
27
|
+
unkeyed?: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One streamed chat completion, assembled frame by frame.
|
|
32
|
+
*
|
|
33
|
+
* Two things this format does that the Anthropic one does not, and both are easy to get subtly
|
|
34
|
+
* wrong: a tool call arrives FRAGMENTED and INDEXED — its id, its name and successive slices of its
|
|
35
|
+
* arguments spread across chunks, keyed only by `tool_calls[].index` — and `usage` arrives once, in
|
|
36
|
+
* a trailing chunk whose `choices` array is empty, long after the finish reason.
|
|
37
|
+
*/
|
|
38
|
+
export class ChatCompletionStream {
|
|
39
|
+
private text = '';
|
|
40
|
+
private refusal = '';
|
|
41
|
+
private stopReason: StopReason = 'end_turn';
|
|
42
|
+
private unknownFinish: string | undefined;
|
|
43
|
+
private finished = false;
|
|
44
|
+
private done = false;
|
|
45
|
+
private usage: TokenUsage | undefined;
|
|
46
|
+
/** Every open call, in order of arrival; `byIndex` is which one a wire index currently means. */
|
|
47
|
+
private readonly pending: PendingCall[] = [];
|
|
48
|
+
private readonly byIndex = new Map<number, PendingCall>();
|
|
49
|
+
private readonly read: ReadCalls = { calls: [], malformed: [] };
|
|
50
|
+
private readonly provider: string;
|
|
51
|
+
|
|
52
|
+
constructor(provider: string) {
|
|
53
|
+
this.provider = provider;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Chunks this frame produced, in order. An unknown field yields nothing — the format grows. */
|
|
57
|
+
push(frame: SseFrame): readonly StreamChunk[] {
|
|
58
|
+
// The sentinel is not JSON, and parsing it is how a stream reader ends in a syntax error.
|
|
59
|
+
if (frame.data.trim() === '[DONE]') {
|
|
60
|
+
this.done = true;
|
|
61
|
+
return this.end();
|
|
62
|
+
}
|
|
63
|
+
const payload = this.payloadOf(frame);
|
|
64
|
+
throwInBandError(payload, this.provider);
|
|
65
|
+
const usage = parseOpenAiUsage(payload['usage']);
|
|
66
|
+
if (usage !== undefined) this.usage = usage;
|
|
67
|
+
const choice = asRecord(Array.isArray(payload['choices']) ? payload['choices'][0] : undefined);
|
|
68
|
+
if (choice === undefined) return [];
|
|
69
|
+
const chunks = this.onDelta(asRecord(choice['delta']) ?? {});
|
|
70
|
+
this.onFinish(choice['finish_reason']);
|
|
71
|
+
return chunks;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The stream is over: the calls a finished turn made, whole, in index order. Called by the
|
|
76
|
+
* provider when the socket closes, and by `[DONE]` — whichever comes first; the second is empty.
|
|
77
|
+
*
|
|
78
|
+
* Here, and not when the finish reason arrives: this format has no per-call stop event, and
|
|
79
|
+
* nothing in it orders the finish reason against the last argument fragment — a call read at the
|
|
80
|
+
* reason is read half way whenever a gateway sends the reason first. A turn with no finish
|
|
81
|
+
* reason drains nothing: its fragments stay open, and `isComplete()` refuses the stream.
|
|
82
|
+
*/
|
|
83
|
+
end(): readonly StreamChunk[] {
|
|
84
|
+
if (!this.finished) return [];
|
|
85
|
+
const chunks: StreamChunk[] = [];
|
|
86
|
+
for (const pending of this.pending.splice(0).sort((a, b) => a.index - b.index)) {
|
|
87
|
+
if (pending.name === '') continue;
|
|
88
|
+
if (pending.unkeyed === true) {
|
|
89
|
+
// Deliberate: merged arguments are never parsed, even when they happen to be JSON.
|
|
90
|
+
this.read.malformed.push({ name: pending.name, chars: pending.args.length, unkeyed: true });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const call = collect(this.read, pending.id, pending.name, pending.args);
|
|
94
|
+
if (call !== undefined) chunks.push({ type: 'tool-call', call });
|
|
95
|
+
}
|
|
96
|
+
this.byIndex.clear();
|
|
97
|
+
return chunks;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** True once the answer is accounted for. False means the connection died mid-answer. */
|
|
101
|
+
isComplete(): boolean {
|
|
102
|
+
// Either sentinel counts. `[DONE]` is the format's own end marker, but plenty of servers in
|
|
103
|
+
// the family close the socket straight after the finish-reason chunk — and a finish reason IS
|
|
104
|
+
// the model saying why it stopped, which is the fact a truncated stream cannot produce.
|
|
105
|
+
//
|
|
106
|
+
// One exception, and it is REFUSAL rather than a flush: `[DONE]` while tool-call fragments are
|
|
107
|
+
// still open. This format has no per-call stop event, so the finish reason is the only thing
|
|
108
|
+
// that ever closes a call and `onFinish` is the only drain of `pending` — which means `[DONE]`
|
|
109
|
+
// alone cannot tell "the model finished asking" from "the connection died mid-arguments".
|
|
110
|
+
// Reporting complete discarded a whole tool call and answered an empty, successful `end_turn`;
|
|
111
|
+
// emitting the fragments anyway would run a tool's side effects from arguments that may be
|
|
112
|
+
// half a JSON object, and would report `end_turn` for a turn that stopped to call one. So the
|
|
113
|
+
// stream is refused exactly as the Anthropic half refuses a missing `message_stop`
|
|
114
|
+
// (`provider.ts`), and `openai-provider.ts`'s existing truncation guard is what raises it.
|
|
115
|
+
return this.finished || (this.done && this.pending.length === 0);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** What the stream accumulated. `cost` is applied by the provider, which owns prices. */
|
|
119
|
+
state(): ChatAnswer {
|
|
120
|
+
// Idempotent, so a reader that never called `end()` still sees a finished turn's calls.
|
|
121
|
+
this.end();
|
|
122
|
+
const details = this.refusalDetails();
|
|
123
|
+
return {
|
|
124
|
+
text: this.text,
|
|
125
|
+
toolCalls: [...this.read.calls],
|
|
126
|
+
malformed: [...this.read.malformed],
|
|
127
|
+
// A refusal that arrived as a `refusal` delta finishes with `stop`, so the reason alone would
|
|
128
|
+
// read as a complete answer that happens to be empty.
|
|
129
|
+
stopReason:
|
|
130
|
+
details !== undefined
|
|
131
|
+
? 'refusal'
|
|
132
|
+
: this.unknownFinish === undefined
|
|
133
|
+
? this.stopReason
|
|
134
|
+
: settledAs(this.read.calls),
|
|
135
|
+
stopDetails: details,
|
|
136
|
+
usage: this.usage,
|
|
137
|
+
...(this.unknownFinish === undefined ? {} : { unknownFinish: this.unknownFinish }),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private refusalDetails(): StopDetails | undefined {
|
|
142
|
+
if (this.refusal !== '') {
|
|
143
|
+
return { type: 'refusal', category: 'refusal', explanation: this.refusal };
|
|
144
|
+
}
|
|
145
|
+
return this.stopReason === 'refusal'
|
|
146
|
+
? { type: 'refusal', category: 'content_filter', explanation: undefined }
|
|
147
|
+
: undefined;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
private onDelta(delta: Record<string, unknown>): readonly StreamChunk[] {
|
|
151
|
+
const chunks: StreamChunk[] = [];
|
|
152
|
+
const content = delta['content'];
|
|
153
|
+
if (typeof content === 'string' && content !== '') {
|
|
154
|
+
this.text += content;
|
|
155
|
+
chunks.push({ type: 'text', text: content });
|
|
156
|
+
}
|
|
157
|
+
// `reasoning_content` is vLLM's and DeepSeek's; `reasoning` is the OpenRouter spelling. Neither
|
|
158
|
+
// is ever appended to `text`, for the reason thinking deltas are not: a consumer concatenating
|
|
159
|
+
// every chunk must not end up shipping the reasoning to the user.
|
|
160
|
+
const thinking = delta['reasoning_content'] ?? delta['reasoning'];
|
|
161
|
+
if (typeof thinking === 'string' && thinking !== '') {
|
|
162
|
+
chunks.push({ type: 'thinking', text: thinking });
|
|
163
|
+
}
|
|
164
|
+
const refusal = delta['refusal'];
|
|
165
|
+
if (typeof refusal === 'string') this.refusal += refusal;
|
|
166
|
+
this.accumulate(delta['tool_calls']);
|
|
167
|
+
return chunks;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Merge one chunk's tool-call fragments into the calls they belong to. `index` is the key —
|
|
172
|
+
* `id` and `name` arrive on the first fragment and are absent from every later one, so appending
|
|
173
|
+
* by array position instead would build one call per chunk and lose every argument but the last.
|
|
174
|
+
*
|
|
175
|
+
* It is not the WHOLE key. A gateway translating another vendor's stream may number every call
|
|
176
|
+
* of a turn `0` (or send no index) and tell them apart by id alone; merged by index, two
|
|
177
|
+
* well-formed calls became one string — `{…}{…}` — that is not JSON. So a fragment carrying an id
|
|
178
|
+
* that is not the open call's starts a new call. An id repeated on every fragment, or absent
|
|
179
|
+
* from all but the first, is still one call.
|
|
180
|
+
*/
|
|
181
|
+
private accumulate(raw: unknown): void {
|
|
182
|
+
if (!Array.isArray(raw)) return;
|
|
183
|
+
for (const entry of raw) {
|
|
184
|
+
const record = asRecord(entry);
|
|
185
|
+
if (record === undefined) continue;
|
|
186
|
+
const index = numberOf(record['index']);
|
|
187
|
+
const rawId = record['id'];
|
|
188
|
+
const id = typeof rawId === 'string' && rawId !== '' ? rawId : undefined;
|
|
189
|
+
let call = index === undefined ? this.pending.at(-1) : this.byIndex.get(index);
|
|
190
|
+
if (call === undefined || (id !== undefined && call.id !== '' && call.id !== id)) {
|
|
191
|
+
call = { index: index ?? this.pending.length, id: '', name: '', args: '' };
|
|
192
|
+
this.pending.push(call);
|
|
193
|
+
}
|
|
194
|
+
if (index !== undefined) this.byIndex.set(index, call);
|
|
195
|
+
if (id !== undefined) call.id = id;
|
|
196
|
+
const fn = asRecord(record['function']);
|
|
197
|
+
const name = fn?.['name'];
|
|
198
|
+
// No index, no id, and a tool NAMED again after arguments already arrived: a second call
|
|
199
|
+
// this stream gives no way to tell from the first. A name split across frames arrives
|
|
200
|
+
// before any argument, so that legal case is not this one.
|
|
201
|
+
if (
|
|
202
|
+
index === undefined &&
|
|
203
|
+
id === undefined &&
|
|
204
|
+
call.args !== '' &&
|
|
205
|
+
name !== undefined &&
|
|
206
|
+
name !== ''
|
|
207
|
+
) {
|
|
208
|
+
call.unkeyed = true;
|
|
209
|
+
}
|
|
210
|
+
// Concatenated, not assigned: a server that splits the name across two frames is rare and
|
|
211
|
+
// legal, and assigning would keep only the tail.
|
|
212
|
+
if (typeof name === 'string') call.name += name;
|
|
213
|
+
const args = fn?.['arguments'];
|
|
214
|
+
if (typeof args === 'string') call.args += args;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** The finish reason closes the TURN. The calls are read by `end()`, when the stream is over. */
|
|
219
|
+
private onFinish(raw: unknown): void {
|
|
220
|
+
const reading = readFinishReason(raw);
|
|
221
|
+
if (reading.kind === 'none') return;
|
|
222
|
+
if (reading.kind === 'known') this.stopReason = reading.reason;
|
|
223
|
+
else this.unknownFinish = reading.raw;
|
|
224
|
+
this.finished = true;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private payloadOf(frame: SseFrame): Record<string, unknown> {
|
|
228
|
+
const where = `unreadable "${frame.event}" frame`;
|
|
229
|
+
let parsed: unknown;
|
|
230
|
+
try {
|
|
231
|
+
parsed = JSON.parse(frame.data);
|
|
232
|
+
} catch (error) {
|
|
233
|
+
throw malformed(this.provider, `${where}: ${renderThrowable(error)}`);
|
|
234
|
+
}
|
|
235
|
+
const record = asRecord(parsed);
|
|
236
|
+
// The same coded refusal the catch above produces, thrown where the frame is known: a bare
|
|
237
|
+
// `SyntaxError` in shipped source carries no code, no cause and no fix.
|
|
238
|
+
if (record === undefined) {
|
|
239
|
+
throw malformed(this.provider, `${where}: frame data is not an object`);
|
|
240
|
+
}
|
|
241
|
+
return record;
|
|
242
|
+
}
|
|
243
|
+
}
|
package/src/openai-wire.ts
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
// Single responsibility: reading what an OpenAI-format endpoint sends back — one chat completion,
|
|
2
|
-
// and the
|
|
2
|
+
// and the field readers its SSE stream (./openai-wire-stream) shares.
|
|
3
3
|
//
|
|
4
4
|
// Split from the provider for the reason wire.ts is: the assembler can then be driven frame by
|
|
5
5
|
// frame with no socket, which is the only way to cover a stream that arrives fragmented, out of
|
|
6
6
|
// order, or stops half way. An LLM response is untrusted input, so every field is parsed and
|
|
7
7
|
// nothing is cast.
|
|
8
8
|
|
|
9
|
-
import { renderThrowable } from '@ultimat3/core';
|
|
10
9
|
import { AiTransportError } from './errors';
|
|
11
|
-
import type { StopDetails, StopReason,
|
|
12
|
-
import type { SseFrame } from './sse';
|
|
10
|
+
import type { StopDetails, StopReason, TokenUsage } from './provider';
|
|
13
11
|
import type { MalformedToolCall } from './tool-arguments';
|
|
14
12
|
import { parseToolArguments } from './tool-arguments';
|
|
15
13
|
import type { LlmToolCall } from './tools';
|
|
@@ -79,7 +77,7 @@ export function readFinishReason(raw: unknown): FinishReading {
|
|
|
79
77
|
}
|
|
80
78
|
|
|
81
79
|
/** The stop reason of a turn whose finish reason is unknown: what its content shows it to be. */
|
|
82
|
-
const settledAs = (calls: readonly LlmToolCall[]): StopReason =>
|
|
80
|
+
export const settledAs = (calls: readonly LlmToolCall[]): StopReason =>
|
|
83
81
|
calls.length > 0 ? 'tool_use' : 'end_turn';
|
|
84
82
|
|
|
85
83
|
/**
|
|
@@ -190,7 +188,7 @@ function refusalDetails(
|
|
|
190
188
|
return { type: 'refusal', category: 'content_filter', explanation: undefined };
|
|
191
189
|
}
|
|
192
190
|
|
|
193
|
-
interface ReadCalls {
|
|
191
|
+
export interface ReadCalls {
|
|
194
192
|
readonly calls: LlmToolCall[];
|
|
195
193
|
readonly malformed: MalformedToolCall[];
|
|
196
194
|
}
|
|
@@ -211,7 +209,7 @@ function parseToolCalls(raw: unknown): ReadCalls {
|
|
|
211
209
|
}
|
|
212
210
|
|
|
213
211
|
/** One call, read: among the calls when its arguments parse, among the malformed when not. */
|
|
214
|
-
function collect(
|
|
212
|
+
export function collect(
|
|
215
213
|
read: ReadCalls,
|
|
216
214
|
id: string,
|
|
217
215
|
name: string,
|
|
@@ -228,237 +226,12 @@ function collect(
|
|
|
228
226
|
return call;
|
|
229
227
|
}
|
|
230
228
|
|
|
231
|
-
interface PendingCall {
|
|
232
|
-
/** Where the call sorts among its turn's: the wire's `index`, else its order of arrival. */
|
|
233
|
-
readonly index: number;
|
|
234
|
-
id: string;
|
|
235
|
-
name: string;
|
|
236
|
-
args: string;
|
|
237
|
-
/** A second NAMED fragment arrived with no index and no id: two calls, indistinguishable. */
|
|
238
|
-
unkeyed?: boolean;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/**
|
|
242
|
-
* One streamed chat completion, assembled frame by frame.
|
|
243
|
-
*
|
|
244
|
-
* Two things this format does that the Anthropic one does not, and both are easy to get subtly
|
|
245
|
-
* wrong: a tool call arrives FRAGMENTED and INDEXED — its id, its name and successive slices of its
|
|
246
|
-
* arguments spread across chunks, keyed only by `tool_calls[].index` — and `usage` arrives once, in
|
|
247
|
-
* a trailing chunk whose `choices` array is empty, long after the finish reason.
|
|
248
|
-
*/
|
|
249
|
-
export class ChatCompletionStream {
|
|
250
|
-
private text = '';
|
|
251
|
-
private refusal = '';
|
|
252
|
-
private stopReason: StopReason = 'end_turn';
|
|
253
|
-
private unknownFinish: string | undefined;
|
|
254
|
-
private finished = false;
|
|
255
|
-
private done = false;
|
|
256
|
-
private usage: TokenUsage | undefined;
|
|
257
|
-
/** Every open call, in order of arrival; `byIndex` is which one a wire index currently means. */
|
|
258
|
-
private readonly pending: PendingCall[] = [];
|
|
259
|
-
private readonly byIndex = new Map<number, PendingCall>();
|
|
260
|
-
private readonly read: ReadCalls = { calls: [], malformed: [] };
|
|
261
|
-
private readonly provider: string;
|
|
262
|
-
|
|
263
|
-
constructor(provider: string) {
|
|
264
|
-
this.provider = provider;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
/** Chunks this frame produced, in order. An unknown field yields nothing — the format grows. */
|
|
268
|
-
push(frame: SseFrame): readonly StreamChunk[] {
|
|
269
|
-
// The sentinel is not JSON, and parsing it is how a stream reader ends in a syntax error.
|
|
270
|
-
if (frame.data.trim() === '[DONE]') {
|
|
271
|
-
this.done = true;
|
|
272
|
-
return this.end();
|
|
273
|
-
}
|
|
274
|
-
const payload = this.payloadOf(frame);
|
|
275
|
-
throwInBandError(payload, this.provider);
|
|
276
|
-
const usage = parseOpenAiUsage(payload['usage']);
|
|
277
|
-
if (usage !== undefined) this.usage = usage;
|
|
278
|
-
const choice = asRecord(Array.isArray(payload['choices']) ? payload['choices'][0] : undefined);
|
|
279
|
-
if (choice === undefined) return [];
|
|
280
|
-
const chunks = this.onDelta(asRecord(choice['delta']) ?? {});
|
|
281
|
-
this.onFinish(choice['finish_reason']);
|
|
282
|
-
return chunks;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/**
|
|
286
|
-
* The stream is over: the calls a finished turn made, whole, in index order. Called by the
|
|
287
|
-
* provider when the socket closes, and by `[DONE]` — whichever comes first; the second is empty.
|
|
288
|
-
*
|
|
289
|
-
* Here, and not when the finish reason arrives: this format has no per-call stop event, and
|
|
290
|
-
* nothing in it orders the finish reason against the last argument fragment — a call read at the
|
|
291
|
-
* reason is read half way whenever a gateway sends the reason first. A turn with no finish
|
|
292
|
-
* reason drains nothing: its fragments stay open, and `isComplete()` refuses the stream.
|
|
293
|
-
*/
|
|
294
|
-
end(): readonly StreamChunk[] {
|
|
295
|
-
if (!this.finished) return [];
|
|
296
|
-
const chunks: StreamChunk[] = [];
|
|
297
|
-
for (const pending of this.pending.splice(0).sort((a, b) => a.index - b.index)) {
|
|
298
|
-
if (pending.name === '') continue;
|
|
299
|
-
if (pending.unkeyed === true) {
|
|
300
|
-
// Deliberate: merged arguments are never parsed, even when they happen to be JSON.
|
|
301
|
-
this.read.malformed.push({ name: pending.name, chars: pending.args.length, unkeyed: true });
|
|
302
|
-
continue;
|
|
303
|
-
}
|
|
304
|
-
const call = collect(this.read, pending.id, pending.name, pending.args);
|
|
305
|
-
if (call !== undefined) chunks.push({ type: 'tool-call', call });
|
|
306
|
-
}
|
|
307
|
-
this.byIndex.clear();
|
|
308
|
-
return chunks;
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
/** True once the answer is accounted for. False means the connection died mid-answer. */
|
|
312
|
-
isComplete(): boolean {
|
|
313
|
-
// Either sentinel counts. `[DONE]` is the format's own end marker, but plenty of servers in
|
|
314
|
-
// the family close the socket straight after the finish-reason chunk — and a finish reason IS
|
|
315
|
-
// the model saying why it stopped, which is the fact a truncated stream cannot produce.
|
|
316
|
-
//
|
|
317
|
-
// One exception, and it is REFUSAL rather than a flush: `[DONE]` while tool-call fragments are
|
|
318
|
-
// still open. This format has no per-call stop event, so the finish reason is the only thing
|
|
319
|
-
// that ever closes a call and `onFinish` is the only drain of `pending` — which means `[DONE]`
|
|
320
|
-
// alone cannot tell "the model finished asking" from "the connection died mid-arguments".
|
|
321
|
-
// Reporting complete discarded a whole tool call and answered an empty, successful `end_turn`;
|
|
322
|
-
// emitting the fragments anyway would run a tool's side effects from arguments that may be
|
|
323
|
-
// half a JSON object, and would report `end_turn` for a turn that stopped to call one. So the
|
|
324
|
-
// stream is refused exactly as the Anthropic half refuses a missing `message_stop`
|
|
325
|
-
// (`provider.ts`), and `openai-provider.ts`'s existing truncation guard is what raises it.
|
|
326
|
-
return this.finished || (this.done && this.pending.length === 0);
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
/** What the stream accumulated. `cost` is applied by the provider, which owns prices. */
|
|
330
|
-
state(): ChatAnswer {
|
|
331
|
-
// Idempotent, so a reader that never called `end()` still sees a finished turn's calls.
|
|
332
|
-
this.end();
|
|
333
|
-
const details = this.refusalDetails();
|
|
334
|
-
return {
|
|
335
|
-
text: this.text,
|
|
336
|
-
toolCalls: [...this.read.calls],
|
|
337
|
-
malformed: [...this.read.malformed],
|
|
338
|
-
// A refusal that arrived as a `refusal` delta finishes with `stop`, so the reason alone would
|
|
339
|
-
// read as a complete answer that happens to be empty.
|
|
340
|
-
stopReason:
|
|
341
|
-
details !== undefined
|
|
342
|
-
? 'refusal'
|
|
343
|
-
: this.unknownFinish === undefined
|
|
344
|
-
? this.stopReason
|
|
345
|
-
: settledAs(this.read.calls),
|
|
346
|
-
stopDetails: details,
|
|
347
|
-
usage: this.usage,
|
|
348
|
-
...(this.unknownFinish === undefined ? {} : { unknownFinish: this.unknownFinish }),
|
|
349
|
-
};
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
private refusalDetails(): StopDetails | undefined {
|
|
353
|
-
if (this.refusal !== '') {
|
|
354
|
-
return { type: 'refusal', category: 'refusal', explanation: this.refusal };
|
|
355
|
-
}
|
|
356
|
-
return this.stopReason === 'refusal'
|
|
357
|
-
? { type: 'refusal', category: 'content_filter', explanation: undefined }
|
|
358
|
-
: undefined;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
private onDelta(delta: Record<string, unknown>): readonly StreamChunk[] {
|
|
362
|
-
const chunks: StreamChunk[] = [];
|
|
363
|
-
const content = delta['content'];
|
|
364
|
-
if (typeof content === 'string' && content !== '') {
|
|
365
|
-
this.text += content;
|
|
366
|
-
chunks.push({ type: 'text', text: content });
|
|
367
|
-
}
|
|
368
|
-
// `reasoning_content` is vLLM's and DeepSeek's; `reasoning` is the OpenRouter spelling. Neither
|
|
369
|
-
// is ever appended to `text`, for the reason thinking deltas are not: a consumer concatenating
|
|
370
|
-
// every chunk must not end up shipping the reasoning to the user.
|
|
371
|
-
const thinking = delta['reasoning_content'] ?? delta['reasoning'];
|
|
372
|
-
if (typeof thinking === 'string' && thinking !== '') {
|
|
373
|
-
chunks.push({ type: 'thinking', text: thinking });
|
|
374
|
-
}
|
|
375
|
-
const refusal = delta['refusal'];
|
|
376
|
-
if (typeof refusal === 'string') this.refusal += refusal;
|
|
377
|
-
this.accumulate(delta['tool_calls']);
|
|
378
|
-
return chunks;
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/**
|
|
382
|
-
* Merge one chunk's tool-call fragments into the calls they belong to. `index` is the key —
|
|
383
|
-
* `id` and `name` arrive on the first fragment and are absent from every later one, so appending
|
|
384
|
-
* by array position instead would build one call per chunk and lose every argument but the last.
|
|
385
|
-
*
|
|
386
|
-
* It is not the WHOLE key. A gateway translating another vendor's stream may number every call
|
|
387
|
-
* of a turn `0` (or send no index) and tell them apart by id alone; merged by index, two
|
|
388
|
-
* well-formed calls became one string — `{…}{…}` — that is not JSON. So a fragment carrying an id
|
|
389
|
-
* that is not the open call's starts a new call. An id repeated on every fragment, or absent
|
|
390
|
-
* from all but the first, is still one call.
|
|
391
|
-
*/
|
|
392
|
-
private accumulate(raw: unknown): void {
|
|
393
|
-
if (!Array.isArray(raw)) return;
|
|
394
|
-
for (const entry of raw) {
|
|
395
|
-
const record = asRecord(entry);
|
|
396
|
-
if (record === undefined) continue;
|
|
397
|
-
const index = numberOf(record['index']);
|
|
398
|
-
const rawId = record['id'];
|
|
399
|
-
const id = typeof rawId === 'string' && rawId !== '' ? rawId : undefined;
|
|
400
|
-
let call = index === undefined ? this.pending.at(-1) : this.byIndex.get(index);
|
|
401
|
-
if (call === undefined || (id !== undefined && call.id !== '' && call.id !== id)) {
|
|
402
|
-
call = { index: index ?? this.pending.length, id: '', name: '', args: '' };
|
|
403
|
-
this.pending.push(call);
|
|
404
|
-
}
|
|
405
|
-
if (index !== undefined) this.byIndex.set(index, call);
|
|
406
|
-
if (id !== undefined) call.id = id;
|
|
407
|
-
const fn = asRecord(record['function']);
|
|
408
|
-
const name = fn?.['name'];
|
|
409
|
-
// No index, no id, and a tool NAMED again after arguments already arrived: a second call
|
|
410
|
-
// this stream gives no way to tell from the first. A name split across frames arrives
|
|
411
|
-
// before any argument, so that legal case is not this one.
|
|
412
|
-
if (
|
|
413
|
-
index === undefined &&
|
|
414
|
-
id === undefined &&
|
|
415
|
-
call.args !== '' &&
|
|
416
|
-
name !== undefined &&
|
|
417
|
-
name !== ''
|
|
418
|
-
) {
|
|
419
|
-
call.unkeyed = true;
|
|
420
|
-
}
|
|
421
|
-
// Concatenated, not assigned: a server that splits the name across two frames is rare and
|
|
422
|
-
// legal, and assigning would keep only the tail.
|
|
423
|
-
if (typeof name === 'string') call.name += name;
|
|
424
|
-
const args = fn?.['arguments'];
|
|
425
|
-
if (typeof args === 'string') call.args += args;
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
/** The finish reason closes the TURN. The calls are read by `end()`, when the stream is over. */
|
|
430
|
-
private onFinish(raw: unknown): void {
|
|
431
|
-
const reading = readFinishReason(raw);
|
|
432
|
-
if (reading.kind === 'none') return;
|
|
433
|
-
if (reading.kind === 'known') this.stopReason = reading.reason;
|
|
434
|
-
else this.unknownFinish = reading.raw;
|
|
435
|
-
this.finished = true;
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
private payloadOf(frame: SseFrame): Record<string, unknown> {
|
|
439
|
-
const where = `unreadable "${frame.event}" frame`;
|
|
440
|
-
let parsed: unknown;
|
|
441
|
-
try {
|
|
442
|
-
parsed = JSON.parse(frame.data);
|
|
443
|
-
} catch (error) {
|
|
444
|
-
throw malformed(this.provider, `${where}: ${renderThrowable(error)}`);
|
|
445
|
-
}
|
|
446
|
-
const record = asRecord(parsed);
|
|
447
|
-
// The same coded refusal the catch above produces, thrown where the frame is known: a bare
|
|
448
|
-
// `SyntaxError` in shipped source carries no code, no cause and no fix.
|
|
449
|
-
if (record === undefined) {
|
|
450
|
-
throw malformed(this.provider, `${where}: frame data is not an object`);
|
|
451
|
-
}
|
|
452
|
-
return record;
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
229
|
/**
|
|
457
230
|
* A 200 that carries an `error` object instead of an answer — how a gateway in front of a model
|
|
458
231
|
* reports a fault it noticed after the headers were sent. Parsed as a message it would read as an
|
|
459
232
|
* empty, successful answer, which is the one outcome nothing downstream can detect.
|
|
460
233
|
*/
|
|
461
|
-
function throwInBandError(payload: Record<string, unknown>, provider: string): void {
|
|
234
|
+
export function throwInBandError(payload: Record<string, unknown>, provider: string): void {
|
|
462
235
|
const error = asRecord(payload['error']);
|
|
463
236
|
if (error === undefined) return;
|
|
464
237
|
const type = typeof error['type'] === 'string' ? error['type'] : 'api_error';
|
|
@@ -472,16 +245,16 @@ function throwInBandError(payload: Record<string, unknown>, provider: string): v
|
|
|
472
245
|
});
|
|
473
246
|
}
|
|
474
247
|
|
|
475
|
-
function malformed(provider: string, detail: string): AiTransportError {
|
|
248
|
+
export function malformed(provider: string, detail: string): AiTransportError {
|
|
476
249
|
return new AiTransportError({ provider, detail });
|
|
477
250
|
}
|
|
478
251
|
|
|
479
|
-
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
252
|
+
export function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
480
253
|
if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined;
|
|
481
254
|
return value as Record<string, unknown>;
|
|
482
255
|
}
|
|
483
256
|
|
|
484
|
-
function numberOf(value: unknown): number | undefined {
|
|
257
|
+
export function numberOf(value: unknown): number | undefined {
|
|
485
258
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
486
259
|
}
|
|
487
260
|
|
package/src/pg-vector-sql.ts
CHANGED
|
@@ -75,8 +75,36 @@ export function ddlSql(target: PgVectorTable): string {
|
|
|
75
75
|
].join('\n');
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
/** `NAMEDATALEN - 1`, in bytes: past it Postgres truncates an identifier and says nothing. */
|
|
79
|
+
const MAX_NAME_BYTES = 63;
|
|
80
|
+
/** `_<8 hex>_idx`: what a shortened name keeps of its whole spelling. */
|
|
81
|
+
const HASHED_TAIL_BYTES = 13;
|
|
82
|
+
|
|
83
|
+
/** The longest prefix of `text` within `bytes` UTF-8 bytes, never splitting a character. */
|
|
84
|
+
const clip = (text: string, bytes: number): string => {
|
|
85
|
+
const encoder = new TextEncoder();
|
|
86
|
+
let kept = '';
|
|
87
|
+
for (const char of text) {
|
|
88
|
+
if (encoder.encode(kept + char).length > bytes) break;
|
|
89
|
+
kept += char;
|
|
90
|
+
}
|
|
91
|
+
return kept;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* `<table>_<column>_idx`, as always, while it fits in 63 bytes. Past that the server truncated it,
|
|
96
|
+
* and a long table's three names collapsed into one: every `create index if not exists` after the
|
|
97
|
+
* first skipped silently. A longer one keeps its head and a hash of the whole spelling instead —
|
|
98
|
+
* distinct per column, and the same name on every run.
|
|
99
|
+
*/
|
|
100
|
+
const indexName = (table: string, column: string): string => {
|
|
101
|
+
const full = `${table}_${column}`;
|
|
102
|
+
if (new TextEncoder().encode(`${full}_idx`).length <= MAX_NAME_BYTES) {
|
|
103
|
+
return identifier(`${full}_idx`).text;
|
|
104
|
+
}
|
|
105
|
+
const hash = new Bun.CryptoHasher('sha256').update(full).digest('hex').slice(0, 8);
|
|
106
|
+
return identifier(`${clip(full, MAX_NAME_BYTES - HASHED_TAIL_BYTES)}_${hash}_idx`).text;
|
|
107
|
+
};
|
|
80
108
|
|
|
81
109
|
export interface PgVectorRowInput {
|
|
82
110
|
readonly id: string;
|
package/src/provider.ts
CHANGED
|
@@ -131,7 +131,7 @@ export interface GenerateResult {
|
|
|
131
131
|
* from one provider to the next is a gateway concept, and a `Provider` implementation an app
|
|
132
132
|
* wrote cannot be asked to report on a decision it did not make. Optional for exactly that
|
|
133
133
|
* reason — a provider's own result has not been through the router yet. `llm()` puts it on the
|
|
134
|
-
* span as `
|
|
134
|
+
* span as `gen_ai.provider.name`, which is what makes a fallback visible rather than silent.
|
|
135
135
|
*/
|
|
136
136
|
readonly provider?: string;
|
|
137
137
|
readonly text: string;
|