@cubicecho/agent-core 2.24.0 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/agent-loop.d.ts +84 -205
- package/dist/agent-loop.js +288 -597
- package/dist/calibration.d.ts +14 -10
- package/dist/calibration.js +30 -23
- package/dist/capabilities.d.ts +111 -42
- package/dist/capabilities.js +281 -138
- package/dist/client.d.ts +104 -35
- package/dist/client.js +205 -74
- package/dist/compaction.d.ts +103 -60
- package/dist/compaction.js +160 -86
- package/dist/config.d.ts +48 -12
- package/dist/config.js +20 -1
- package/dist/continuation.d.ts +24 -16
- package/dist/continuation.js +70 -44
- package/dist/digest.d.ts +7 -0
- package/dist/digest.js +18 -0
- package/dist/errors.d.ts +24 -15
- package/dist/errors.js +23 -15
- package/dist/events.d.ts +100 -63
- package/dist/events.js +250 -120
- package/dist/guards.d.ts +35 -4
- package/dist/guards.js +38 -5
- package/dist/hook-events.d.ts +21 -5
- package/dist/hook-events.js +23 -9
- package/dist/hooks.d.ts +96 -73
- package/dist/hooks.js +202 -116
- package/dist/index.d.ts +30 -26
- package/dist/index.js +29 -24
- package/dist/ledger.d.ts +62 -30
- package/dist/ledger.js +163 -56
- package/dist/platform.d.ts +13 -0
- package/dist/platform.js +13 -0
- package/dist/preselect.d.ts +222 -0
- package/dist/preselect.js +290 -0
- package/dist/request-body.d.ts +32 -0
- package/dist/request-body.js +47 -0
- package/dist/reset.d.ts +1 -0
- package/dist/reset.js +7 -6
- package/dist/retry.d.ts +29 -138
- package/dist/retry.js +69 -282
- package/dist/run-calls.d.ts +123 -0
- package/dist/run-calls.js +351 -0
- package/dist/run-turn.d.ts +36 -15
- package/dist/run-turn.js +74 -37
- package/dist/runtime.d.ts +23 -17
- package/dist/runtime.js +34 -24
- package/dist/schema-compat.d.ts +35 -18
- package/dist/schema-compat.js +340 -175
- package/dist/scope.d.ts +28 -5
- package/dist/scope.js +37 -10
- package/dist/side-task.d.ts +123 -40
- package/dist/side-task.js +181 -63
- package/dist/snapshot.d.ts +10 -5
- package/dist/snapshot.js +73 -30
- package/dist/spec.d.ts +56 -25
- package/dist/spec.js +494 -215
- package/dist/stream.d.ts +66 -16
- package/dist/stream.js +220 -97
- package/dist/thinking.d.ts +23 -8
- package/dist/thinking.js +92 -44
- package/dist/tokens.d.ts +164 -6
- package/dist/tokens.js +319 -6
- package/dist/tool-calls.d.ts +82 -17
- package/dist/tool-calls.js +198 -83
- package/dist/tool-loading.d.ts +101 -189
- package/dist/tool-loading.js +277 -371
- package/dist/wire.d.ts +81 -0
- package/dist/wire.js +73 -0
- package/llms.txt +76 -39
- package/package.json +4 -4
package/dist/stream.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type OpenAI from
|
|
2
|
-
import { type
|
|
3
|
-
import {
|
|
1
|
+
import type OpenAI from 'openai';
|
|
2
|
+
import { type Fence } from './thinking.ts';
|
|
3
|
+
import type { ContextBreakdown } from './tokens.ts';
|
|
4
|
+
import type { ToolCall } from './tool-calls.ts';
|
|
4
5
|
/**
|
|
5
6
|
* Reading one streamed turn back into a message.
|
|
6
7
|
*
|
|
@@ -11,9 +12,32 @@ import { type Fence } from "./thinking.ts";
|
|
|
11
12
|
* hangs the turn until somebody presses stop. `EndpointSilent` and `timeoutMs` were exported for
|
|
12
13
|
* this loop long before the loop itself was.
|
|
13
14
|
*/
|
|
15
|
+
/**
|
|
16
|
+
* What the loop changed that would explain a prompt cache that did not hold.
|
|
17
|
+
*
|
|
18
|
+
* @remarks
|
|
19
|
+
* In the order `breakReason` looks, the earliest in the prompt first, since a change there is the
|
|
20
|
+
* one that costs the rest.
|
|
21
|
+
*/
|
|
22
|
+
export declare const CacheBreakReason: {
|
|
23
|
+
/** The tool array is not the one the request before declared. */
|
|
24
|
+
readonly ToolsChanged: 'tools-changed';
|
|
25
|
+
/** The system messages the prompt opens with are not the ones it opened with before. */
|
|
26
|
+
readonly SystemChanged: 'system-changed';
|
|
27
|
+
/** A message the request before sent is gone or different. */
|
|
28
|
+
readonly HistoryRewritten: 'history-rewritten';
|
|
29
|
+
/**
|
|
30
|
+
* Nothing the loop can see: the server's doing — an eviction, another client on the slot — or
|
|
31
|
+
* a change made where the loop does not look.
|
|
32
|
+
*/
|
|
33
|
+
readonly NoneKnown: 'none-known';
|
|
34
|
+
};
|
|
35
|
+
/** Any one of the reasons in `CacheBreakReason`. */
|
|
36
|
+
export type CacheBreakReason = (typeof CacheBreakReason)[keyof typeof CacheBreakReason];
|
|
14
37
|
/**
|
|
15
38
|
* What a turn cost, and how it went. Zero throughout the four counts means the server did not say.
|
|
16
39
|
*
|
|
40
|
+
* @remarks
|
|
17
41
|
* The four counts are always there, as they always were. Everything after them is absent rather
|
|
18
42
|
* than zero when nothing reported or measured it, because a zero cache hit and a server that says
|
|
19
43
|
* nothing about its cache are different findings, and a consumer drawing one as the other is
|
|
@@ -28,6 +52,7 @@ export interface TurnUsage {
|
|
|
28
52
|
/**
|
|
29
53
|
* How much of `prompt` came from the endpoint's prompt cache — a part of it, not in addition.
|
|
30
54
|
*
|
|
55
|
+
* @remarks
|
|
31
56
|
* The only way a caller can tell whether the prefix it is careful to keep still is actually
|
|
32
57
|
* being reused: a prefix that stops hitting the cache otherwise shows up as a bill and nothing
|
|
33
58
|
* else. Zero is also what a server that does not report it sends; `uncached` is the field whose
|
|
@@ -83,12 +108,8 @@ export interface TurnUsage {
|
|
|
83
108
|
* reported, so a server that says nothing about its cache is never accused of losing it.
|
|
84
109
|
*/
|
|
85
110
|
cacheBroken?: boolean;
|
|
86
|
-
/**
|
|
87
|
-
|
|
88
|
-
* since a change there is the one that costs the rest. `none-known` is the server's doing — an
|
|
89
|
-
* eviction, another client on the slot — or a change the loop cannot see. Only on a broken turn.
|
|
90
|
-
*/
|
|
91
|
-
cacheBreakReason?: "tools-changed" | "system-changed" | "history-rewritten" | "none-known";
|
|
111
|
+
/** What the loop changed that would explain `cacheBroken`. Only on a broken turn. */
|
|
112
|
+
cacheBreakReason?: CacheBreakReason;
|
|
92
113
|
/** How many tools the request declared, filled by `runAgentLoop`. */
|
|
93
114
|
toolsDeclared?: number;
|
|
94
115
|
/**
|
|
@@ -100,6 +121,7 @@ export interface TurnUsage {
|
|
|
100
121
|
* What the request was made of, as `contextTokens` cuts it, filled by `runAgentLoop`. Shares of
|
|
101
122
|
* the prompt the turn's first request reported, summing to it, or an estimate where none was.
|
|
102
123
|
*
|
|
124
|
+
* @remarks
|
|
103
125
|
* Measured on the request as first built, with the tools as `toolSchemaTokens` measures them. A
|
|
104
126
|
* continued turn's `prompt` is every request's summed, so it is larger than this `total`; and
|
|
105
127
|
* where a refusal was answered with a lesser body, the prompt is that body's and the proportions
|
|
@@ -110,10 +132,25 @@ export interface TurnUsage {
|
|
|
110
132
|
/**
|
|
111
133
|
* A usage with nothing counted yet, a fresh object each call since every caller adds into it.
|
|
112
134
|
*
|
|
135
|
+
* @returns The four counts at zero and no other field.
|
|
136
|
+
*
|
|
137
|
+
* @remarks
|
|
113
138
|
* Only the four counts every turn has: the optional fields stay absent, which is what says
|
|
114
139
|
* nothing reported them.
|
|
115
140
|
*/
|
|
116
141
|
export declare const noUsage: () => TurnUsage;
|
|
142
|
+
/**
|
|
143
|
+
* The four token counts of two usages, added.
|
|
144
|
+
*
|
|
145
|
+
* @param a - One usage. Neither is changed.
|
|
146
|
+
* @param b - The other.
|
|
147
|
+
* @returns A new usage holding the four sums, and none of the optional fields of either.
|
|
148
|
+
*
|
|
149
|
+
* @remarks
|
|
150
|
+
* Only those: the rest are measurements a sum of would mean nothing, or would mean something only
|
|
151
|
+
* `runMetrics` knows how to weigh.
|
|
152
|
+
*/
|
|
153
|
+
export declare const addCounts: (a: TurnUsage, b: TurnUsage) => TurnUsage;
|
|
117
154
|
/** One streamed turn, put back together into the shape a loop and a transcript work with. */
|
|
118
155
|
export interface Turn {
|
|
119
156
|
content: string;
|
|
@@ -123,12 +160,13 @@ export interface Turn {
|
|
|
123
160
|
* belongs to, every caller had to narrow before it could read `.function`, to rule out a
|
|
124
161
|
* custom call that this loop cannot produce. Still assignable wherever the union is wanted.
|
|
125
162
|
*/
|
|
126
|
-
toolCalls:
|
|
163
|
+
toolCalls: ToolCall[];
|
|
127
164
|
usage: TurnUsage;
|
|
128
165
|
/**
|
|
129
166
|
* Why the model stopped, in the endpoint's own words — `stop`, `length`, `tool_calls`, or `""`
|
|
130
167
|
* where it never said.
|
|
131
168
|
*
|
|
169
|
+
* @remarks
|
|
132
170
|
* Reported because `length` is otherwise invisible. A turn cut off at the token ceiling comes
|
|
133
171
|
* back as a well-formed `Turn` with truncated `content`, or with a tool call whose `arguments`
|
|
134
172
|
* stop mid-JSON — so the caller meets a parse failure with nothing to attribute it to. Being
|
|
@@ -141,6 +179,7 @@ export interface Turn {
|
|
|
141
179
|
* The model's scratchpad, as `onThinking` was told it, `""` where it deliberated in silence or
|
|
142
180
|
* not at all.
|
|
143
181
|
*
|
|
182
|
+
* @remarks
|
|
144
183
|
* Kept because some models want it back. gpt-oss and DeepSeek in thinking mode read the
|
|
145
184
|
* analysis behind a tool call off the assistant message on the next request, so a caller
|
|
146
185
|
* talking to one stores it as `reasoning_content` on that message for as long as the message
|
|
@@ -152,6 +191,7 @@ export interface Turn {
|
|
|
152
191
|
/**
|
|
153
192
|
* Whether the model has said anything a second attempt would say twice.
|
|
154
193
|
*
|
|
194
|
+
* @remarks
|
|
155
195
|
* A box rather than a return value because it has to be readable *while* the request is in
|
|
156
196
|
* flight: the rules in `retry.ts` are built on the premise that a stream which has already
|
|
157
197
|
* emitted tokens must never be replayed, and by the time a rejected promise is in hand the turn
|
|
@@ -178,8 +218,11 @@ export interface StreamTurnOptions {
|
|
|
178
218
|
*/
|
|
179
219
|
idleMs?: number;
|
|
180
220
|
/**
|
|
181
|
-
* Silence allowed before the first chunk
|
|
221
|
+
* Silence allowed before the first chunk; zero waits forever.
|
|
182
222
|
*
|
|
223
|
+
* @defaultValue `idleMs`
|
|
224
|
+
*
|
|
225
|
+
* @remarks
|
|
183
226
|
* The first wait is prefill, or a server loading the model, and is routinely many times the
|
|
184
227
|
* gap between tokens. See `Endpoint.firstTokenSeconds` and `firstTokenMs`.
|
|
185
228
|
*/
|
|
@@ -187,8 +230,11 @@ export interface StreamTurnOptions {
|
|
|
187
230
|
/** Set by the first chunk that carries anything, so a failed call knows if it can be retried. */
|
|
188
231
|
produced?: Produced;
|
|
189
232
|
/**
|
|
190
|
-
* The fences that mark a scratchpad written into `content
|
|
233
|
+
* The fences that mark a scratchpad written into `content`.
|
|
234
|
+
*
|
|
235
|
+
* @defaultValue `DEFAULT_FENCES`
|
|
191
236
|
*
|
|
237
|
+
* @remarks
|
|
192
238
|
* Text inside one goes to `onThinking` and `reasoning` rather than `onOutput` and `content`.
|
|
193
239
|
* `ALL_FENCES` adds `<thinking>` and `<reasoning>`, which a model can also be quoting; an
|
|
194
240
|
* empty list reads `content` as all answer.
|
|
@@ -209,6 +255,14 @@ export interface StreamTurnOptions {
|
|
|
209
255
|
* Runs one turn as a stream, reporting tokens as they arrive and assembling them back into a
|
|
210
256
|
* message.
|
|
211
257
|
*
|
|
258
|
+
* @param client - The pooled client for this endpoint.
|
|
259
|
+
* @param body - The request, which must set `stream: true`.
|
|
260
|
+
* @param [options] - Cancellation, the idle watchdog, and the token callbacks.
|
|
261
|
+
* @returns The assembled turn, once the stream has ended. Rejects with `EndpointSilent` where the
|
|
262
|
+
* endpoint was silent past its allowance, and with whatever the abort raised where `signal`
|
|
263
|
+
* stopped it — an aborted stream never comes back as a turn.
|
|
264
|
+
*
|
|
265
|
+
* @remarks
|
|
212
266
|
* Streaming buys no speed — nothing waits on the reply but the loop itself. It is what makes a
|
|
213
267
|
* run watchable: a run that stalls, loops, or reaches for the wrong tool says so while it is
|
|
214
268
|
* happening instead of only in the row it leaves behind.
|
|
@@ -216,9 +270,5 @@ export interface StreamTurnOptions {
|
|
|
216
270
|
* Two token callbacks rather than an event input, because a turn does not know which step of
|
|
217
271
|
* which run it is: `step` is the caller's flow concept, and wrapping these into an `emit` is one
|
|
218
272
|
* line at the call site.
|
|
219
|
-
*
|
|
220
|
-
* @param client The pooled client for this endpoint.
|
|
221
|
-
* @param body The request, which must set `stream: true`.
|
|
222
|
-
* @param options Cancellation, the idle watchdog, and the token callbacks.
|
|
223
273
|
*/
|
|
224
274
|
export declare function streamTurn(client: OpenAI, body: OpenAI.ChatCompletionCreateParamsStreaming, { signal, idleMs, firstChunkMs, produced, fences, startInReasoning, onThinking, onOutput, }?: StreamTurnOptions): Promise<Turn>;
|
package/dist/stream.js
CHANGED
|
@@ -1,31 +1,205 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { MS_PER_SECOND } from './platform.js';
|
|
2
|
+
import { EndpointSilent } from './retry.js';
|
|
3
|
+
import { DEFAULT_FENCES, FenceSplitter, SPLIT_REASONING } from './thinking.js';
|
|
4
|
+
import { FUNCTION_TOOL } from './wire.js';
|
|
5
|
+
/**
|
|
6
|
+
* Reading one streamed turn back into a message.
|
|
7
|
+
*
|
|
8
|
+
* The rest of a request is the caller's — which model, which tools, which transcript — but the
|
|
9
|
+
* reading of the answer is the same everywhere, and it is fiddly in ways the API does not
|
|
10
|
+
* advertise: tool calls arrive in pieces, an aborted stream ends rather than throws, reasoning
|
|
11
|
+
* has two spellings and is in no published type, and an endpoint that stops answering mid-stream
|
|
12
|
+
* hangs the turn until somebody presses stop. `EndpointSilent` and `timeoutMs` were exported for
|
|
13
|
+
* this loop long before the loop itself was.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* What the loop changed that would explain a prompt cache that did not hold.
|
|
17
|
+
*
|
|
18
|
+
* @remarks
|
|
19
|
+
* In the order `breakReason` looks, the earliest in the prompt first, since a change there is the
|
|
20
|
+
* one that costs the rest.
|
|
21
|
+
*/
|
|
22
|
+
export const CacheBreakReason = {
|
|
23
|
+
/** The tool array is not the one the request before declared. */
|
|
24
|
+
ToolsChanged: 'tools-changed',
|
|
25
|
+
/** The system messages the prompt opens with are not the ones it opened with before. */
|
|
26
|
+
SystemChanged: 'system-changed',
|
|
27
|
+
/** A message the request before sent is gone or different. */
|
|
28
|
+
HistoryRewritten: 'history-rewritten',
|
|
29
|
+
/**
|
|
30
|
+
* Nothing the loop can see: the server's doing — an eviction, another client on the slot — or
|
|
31
|
+
* a change made where the loop does not look.
|
|
32
|
+
*/
|
|
33
|
+
NoneKnown: 'none-known',
|
|
34
|
+
};
|
|
3
35
|
/**
|
|
4
36
|
* A usage with nothing counted yet, a fresh object each call since every caller adds into it.
|
|
5
37
|
*
|
|
38
|
+
* @returns The four counts at zero and no other field.
|
|
39
|
+
*
|
|
40
|
+
* @remarks
|
|
6
41
|
* Only the four counts every turn has: the optional fields stay absent, which is what says
|
|
7
42
|
* nothing reported them.
|
|
8
43
|
*/
|
|
9
44
|
export const noUsage = () => ({ prompt: 0, completion: 0, total: 0, cached: 0 });
|
|
45
|
+
/**
|
|
46
|
+
* The four token counts of two usages, added.
|
|
47
|
+
*
|
|
48
|
+
* @param a - One usage. Neither is changed.
|
|
49
|
+
* @param b - The other.
|
|
50
|
+
* @returns A new usage holding the four sums, and none of the optional fields of either.
|
|
51
|
+
*
|
|
52
|
+
* @remarks
|
|
53
|
+
* Only those: the rest are measurements a sum of would mean nothing, or would mean something only
|
|
54
|
+
* `runMetrics` knows how to weigh.
|
|
55
|
+
*/
|
|
56
|
+
export const addCounts = (a, b) => ({
|
|
57
|
+
prompt: a.prompt + b.prompt,
|
|
58
|
+
completion: a.completion + b.completion,
|
|
59
|
+
total: a.total + b.total,
|
|
60
|
+
cached: a.cached + b.cached,
|
|
61
|
+
});
|
|
10
62
|
/**
|
|
11
63
|
* The fields of `timings` a turn reports, under the names `TurnUsage` gives them. A value that is
|
|
12
64
|
* not a finite number is left out rather than guessed at.
|
|
13
65
|
*/
|
|
14
66
|
const TIMINGS = [
|
|
15
|
-
[
|
|
16
|
-
[
|
|
17
|
-
[
|
|
18
|
-
[
|
|
19
|
-
[
|
|
20
|
-
[
|
|
67
|
+
['prompt_ms', 'promptMs'],
|
|
68
|
+
['prompt_per_second', 'promptTokensPerSecond'],
|
|
69
|
+
['predicted_ms', 'predictedMs'],
|
|
70
|
+
['predicted_per_second', 'tokensPerSecond'],
|
|
71
|
+
['draft_n', 'draftTotal'],
|
|
72
|
+
['draft_n_accepted', 'draftAccepted'],
|
|
21
73
|
];
|
|
22
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Whether a value is a finite number, which is what a count or a timing off the wire has to be.
|
|
76
|
+
*
|
|
77
|
+
* @param value - Whatever a server put in a numeric field.
|
|
78
|
+
* @returns True for a number that is neither `NaN` nor infinite. Zero and a negative pass.
|
|
79
|
+
*/
|
|
80
|
+
const isCount = (value) => typeof value === 'number' && Number.isFinite(value);
|
|
81
|
+
/**
|
|
82
|
+
* Reads a chunk's usage block and its `timings`, either of which most chunks carry neither of.
|
|
83
|
+
*
|
|
84
|
+
* @param chunk - One chunk of the stream. Not written to.
|
|
85
|
+
* @returns Only what this chunk said. `counts` is empty for a chunk with neither block; a usage
|
|
86
|
+
* block always sets the three token counts, zero for one it left out; and a cache count, a
|
|
87
|
+
* reasoning count or a timing is there only where the server sent a finite number.
|
|
88
|
+
*/
|
|
89
|
+
function chunkReport(chunk) {
|
|
90
|
+
const counts = {};
|
|
91
|
+
const report = { counts };
|
|
92
|
+
if (chunk.usage) {
|
|
93
|
+
counts.prompt = chunk.usage.prompt_tokens ?? 0;
|
|
94
|
+
counts.completion = chunk.usage.completion_tokens ?? 0;
|
|
95
|
+
counts.total = chunk.usage.total_tokens ?? 0;
|
|
96
|
+
const reported = chunk.usage;
|
|
97
|
+
const hit = reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens;
|
|
98
|
+
if (isCount(hit)) {
|
|
99
|
+
report.cacheReport = hit;
|
|
100
|
+
}
|
|
101
|
+
const reasoned = reported.completion_tokens_details?.reasoning_tokens;
|
|
102
|
+
if (isCount(reasoned)) {
|
|
103
|
+
counts.reasoningTokens = reasoned;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
const { timings } = chunk;
|
|
107
|
+
if (timings) {
|
|
108
|
+
for (const [from, to] of TIMINGS) {
|
|
109
|
+
const value = timings[from];
|
|
110
|
+
if (isCount(value)) {
|
|
111
|
+
counts[to] = value;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (isCount(timings.cache_n)) {
|
|
115
|
+
report.cacheTimings = timings.cache_n;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return report;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The position a fragment says its call is at, where the server sent a number.
|
|
122
|
+
*
|
|
123
|
+
* @param part - One tool-call fragment of a delta.
|
|
124
|
+
* @returns Its `index`, or `undefined` where the server sent none or something not a number.
|
|
125
|
+
*/
|
|
126
|
+
const fragmentIndex = (part) => typeof part.index === 'number' ? part.index : undefined;
|
|
127
|
+
/**
|
|
128
|
+
* The call a fragment belongs to, of those begun so far, or none where it opens a new one.
|
|
129
|
+
*
|
|
130
|
+
* @param calls - The calls begun so far, in arrival order. Not written to.
|
|
131
|
+
* @param part - The fragment to place.
|
|
132
|
+
* @returns The call to add the fragment to — the object in `calls`, not a copy — or `undefined`
|
|
133
|
+
* where the caller is to begin a new one.
|
|
134
|
+
*
|
|
135
|
+
* @remarks
|
|
136
|
+
* By `index` where the server sent a number, which the SDK types as required and servers have
|
|
137
|
+
* nonetheless left out: keyed on `undefined`, every call joined into one whose name and
|
|
138
|
+
* arguments were all of theirs run together. Without one, by `id`; failing that, a fragment
|
|
139
|
+
* naming a function opens a call and a bare run of arguments continues the latest. A server
|
|
140
|
+
* that sends whole calls one per chunk at index `0` makes the same mistake the other way, so
|
|
141
|
+
* a different id, or a name after arguments have begun, opens a new call under that index.
|
|
142
|
+
*/
|
|
143
|
+
function ownerOf(calls, part) {
|
|
144
|
+
const index = fragmentIndex(part);
|
|
145
|
+
const name = part.function?.name;
|
|
146
|
+
let known;
|
|
147
|
+
if (index !== undefined) {
|
|
148
|
+
known = calls.findLast((call) => call.index === index);
|
|
149
|
+
}
|
|
150
|
+
else if (part.id) {
|
|
151
|
+
known = calls.find((call) => call.id === part.id);
|
|
152
|
+
}
|
|
153
|
+
else if (!name) {
|
|
154
|
+
known = calls.at(-1);
|
|
155
|
+
}
|
|
156
|
+
const another = known && ((part.id && known.id && part.id !== known.id) || (name && known.arguments));
|
|
157
|
+
return another ? undefined : known;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* A turn's calls as they are handed back: in the order their indexes give, arrival order where a
|
|
161
|
+
* server sent none, each under an id of its own.
|
|
162
|
+
*
|
|
163
|
+
* @param calls - The calls as the stream left them, in arrival order. Not written to or reordered.
|
|
164
|
+
* @returns One function call per entry. An id is made up for a call the server gave none, and for
|
|
165
|
+
* one whose id an earlier call in the result already has.
|
|
166
|
+
*/
|
|
167
|
+
function assembledCalls(calls) {
|
|
168
|
+
const ordered = calls
|
|
169
|
+
.map((call, order) => ({ call, order }))
|
|
170
|
+
.sort((a, b) => (a.call.index ?? a.order) - (b.call.index ?? b.order) || a.order - b.order);
|
|
171
|
+
const minted = new Set();
|
|
172
|
+
const assembled = [];
|
|
173
|
+
for (const [position, { call }] of ordered.entries()) {
|
|
174
|
+
// A server that streams a call without an id still needs one for the result to answer,
|
|
175
|
+
// and two calls it put under one index must not be answered as one.
|
|
176
|
+
let id = call.id || `call_${call.index ?? position}`;
|
|
177
|
+
if (minted.has(id)) {
|
|
178
|
+
id = `call_${position}_${minted.size}`;
|
|
179
|
+
}
|
|
180
|
+
minted.add(id);
|
|
181
|
+
assembled.push({
|
|
182
|
+
id,
|
|
183
|
+
type: FUNCTION_TOOL,
|
|
184
|
+
function: { name: call.name, arguments: call.arguments },
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
return assembled;
|
|
188
|
+
}
|
|
23
189
|
/** The largest delay a timer takes, which is as close to none as the SDK's timeout option goes. */
|
|
24
|
-
const NO_SDK_TIMEOUT =
|
|
190
|
+
const NO_SDK_TIMEOUT = 2_147_483_647;
|
|
25
191
|
/**
|
|
26
192
|
* Runs one turn as a stream, reporting tokens as they arrive and assembling them back into a
|
|
27
193
|
* message.
|
|
28
194
|
*
|
|
195
|
+
* @param client - The pooled client for this endpoint.
|
|
196
|
+
* @param body - The request, which must set `stream: true`.
|
|
197
|
+
* @param [options] - Cancellation, the idle watchdog, and the token callbacks.
|
|
198
|
+
* @returns The assembled turn, once the stream has ended. Rejects with `EndpointSilent` where the
|
|
199
|
+
* endpoint was silent past its allowance, and with whatever the abort raised where `signal`
|
|
200
|
+
* stopped it — an aborted stream never comes back as a turn.
|
|
201
|
+
*
|
|
202
|
+
* @remarks
|
|
29
203
|
* Streaming buys no speed — nothing waits on the reply but the loop itself. It is what makes a
|
|
30
204
|
* run watchable: a run that stalls, loops, or reaches for the wrong tool says so while it is
|
|
31
205
|
* happening instead of only in the row it leaves behind.
|
|
@@ -33,10 +207,6 @@ const NO_SDK_TIMEOUT = 2 ** 31 - 1;
|
|
|
33
207
|
* Two token callbacks rather than an event input, because a turn does not know which step of
|
|
34
208
|
* which run it is: `step` is the caller's flow concept, and wrapping these into an `emit` is one
|
|
35
209
|
* line at the call site.
|
|
36
|
-
*
|
|
37
|
-
* @param client The pooled client for this endpoint.
|
|
38
|
-
* @param body The request, which must set `stream: true`.
|
|
39
|
-
* @param options Cancellation, the idle watchdog, and the token callbacks.
|
|
40
210
|
*/
|
|
41
211
|
export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, produced, fences = DEFAULT_FENCES, startInReasoning, onThinking, onOutput, } = {}) {
|
|
42
212
|
// Silence, not duration: the timer is rearmed on every chunk, so a model that is still
|
|
@@ -55,8 +225,9 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
55
225
|
talking ||= carried;
|
|
56
226
|
const ms = talking ? idleMs : first;
|
|
57
227
|
clearTimeout(idle);
|
|
58
|
-
if (ms)
|
|
228
|
+
if (ms) {
|
|
59
229
|
idle = setTimeout(() => watchdog.abort(), ms);
|
|
230
|
+
}
|
|
60
231
|
};
|
|
61
232
|
try {
|
|
62
233
|
rearm(false);
|
|
@@ -70,8 +241,8 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
70
241
|
// they read the row and disbelieve it.
|
|
71
242
|
if (watchdog.signal.aborted && !signal?.aborted) {
|
|
72
243
|
const waited = talking ? idleMs : first;
|
|
73
|
-
const before = talking || first === idleMs ?
|
|
74
|
-
throw new EndpointSilent(`the model endpoint sent nothing for ${(waited ?? 0) /
|
|
244
|
+
const before = talking || first === idleMs ? '' : ' before its first token';
|
|
245
|
+
throw new EndpointSilent(`the model endpoint sent nothing for ${(waited ?? 0) / MS_PER_SECOND}s${before}`);
|
|
75
246
|
}
|
|
76
247
|
throw error;
|
|
77
248
|
}
|
|
@@ -92,8 +263,9 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
92
263
|
const reasoning = [];
|
|
93
264
|
const splitter = new FenceSplitter(fences, { startInside: startInReasoning });
|
|
94
265
|
const report = (parts) => {
|
|
95
|
-
for (const part of parts)
|
|
96
|
-
(part.kind ===
|
|
266
|
+
for (const part of parts) {
|
|
267
|
+
(part.kind === SPLIT_REASONING ? onThinking : onOutput)?.(part.text);
|
|
268
|
+
}
|
|
97
269
|
};
|
|
98
270
|
// In arrival order, sorted by index at the end; a call from a server that sent none keeps
|
|
99
271
|
// its place in the order they arrived.
|
|
@@ -103,7 +275,7 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
103
275
|
// servers that send both; kept apart so the order the two arrive in does not decide.
|
|
104
276
|
let cacheReport;
|
|
105
277
|
let cacheTimings;
|
|
106
|
-
let finishReason =
|
|
278
|
+
let finishReason = '';
|
|
107
279
|
for await (const chunk of stream) {
|
|
108
280
|
// Rearmed on every chunk, latched below on only some: a priming chunk is the endpoint
|
|
109
281
|
// being alive, which is all the watchdog is asking about. Which allowance it rearms with
|
|
@@ -113,40 +285,24 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
113
285
|
// and the two agree there, but a server that reports cumulatively per chunk makes a sum
|
|
114
286
|
// of sums out of an accumulator — and a token count wrong by a factor of the chunk count
|
|
115
287
|
// is not a number anyone would attribute to the stream reader.
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const reported = chunk.usage;
|
|
121
|
-
const hit = reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens;
|
|
122
|
-
if (isCount(hit))
|
|
123
|
-
cacheReport = hit;
|
|
124
|
-
const reasoned = reported.completion_tokens_details?.reasoning_tokens;
|
|
125
|
-
if (isCount(reasoned))
|
|
126
|
-
usage.reasoningTokens = reasoned;
|
|
127
|
-
}
|
|
128
|
-
const { timings } = chunk;
|
|
129
|
-
if (timings) {
|
|
130
|
-
for (const [from, to] of TIMINGS) {
|
|
131
|
-
const value = timings[from];
|
|
132
|
-
if (isCount(value))
|
|
133
|
-
usage[to] = value;
|
|
134
|
-
}
|
|
135
|
-
if (isCount(timings.cache_n))
|
|
136
|
-
cacheTimings = timings.cache_n;
|
|
137
|
-
}
|
|
288
|
+
const reported = chunkReport(chunk);
|
|
289
|
+
Object.assign(usage, reported.counts);
|
|
290
|
+
cacheReport = reported.cacheReport ?? cacheReport;
|
|
291
|
+
cacheTimings = reported.cacheTimings ?? cacheTimings;
|
|
138
292
|
// One choice, because that is what an agent loop asks for. A body with `n` above one
|
|
139
293
|
// keeps only the first; nothing here is built to reassemble several at once.
|
|
140
294
|
const choice = chunk.choices[0];
|
|
141
295
|
// Read before the delta guard rather than beside the content. The chunk that carries the
|
|
142
296
|
// reason usually carries an empty delta, and some servers send it with no delta at all —
|
|
143
297
|
// either of which the guard below skips, taking the reason with it.
|
|
144
|
-
if (choice?.finish_reason)
|
|
298
|
+
if (choice?.finish_reason) {
|
|
145
299
|
finishReason = choice.finish_reason;
|
|
300
|
+
}
|
|
146
301
|
const delta = choice?.delta;
|
|
147
|
-
if (!delta)
|
|
302
|
+
if (!delta) {
|
|
148
303
|
continue;
|
|
149
|
-
|
|
304
|
+
}
|
|
305
|
+
const thinking = delta.reasoning_content || delta.reasoning || '';
|
|
150
306
|
// Latched on what the chunk carried, not on its having arrived. Most OpenAI-compatible
|
|
151
307
|
// servers open a stream with a content-free `{"role":"assistant"}` before the first
|
|
152
308
|
// token; latching on that made an endpoint that primes and then wedges unrepeatable,
|
|
@@ -159,53 +315,35 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
159
315
|
rearm(true);
|
|
160
316
|
usage.firstTokenMs = Date.now() - started;
|
|
161
317
|
}
|
|
162
|
-
if (produced && carried)
|
|
318
|
+
if (produced && carried) {
|
|
163
319
|
produced.any = true;
|
|
320
|
+
}
|
|
164
321
|
if (thinking) {
|
|
165
322
|
reasoning.push(thinking);
|
|
166
323
|
onThinking?.(thinking);
|
|
167
324
|
}
|
|
168
|
-
if (delta.content)
|
|
325
|
+
if (delta.content) {
|
|
169
326
|
report(splitter.push(delta.content));
|
|
327
|
+
}
|
|
170
328
|
// Tool calls arrive in pieces, keyed by position: the id in one chunk, the name in
|
|
171
329
|
// another, the arguments spread across the next several.
|
|
172
330
|
for (const part of delta.tool_calls ?? []) {
|
|
173
|
-
|
|
174
|
-
if (
|
|
331
|
+
let call = ownerOf(calls, part);
|
|
332
|
+
if (!call) {
|
|
333
|
+
call = { index: fragmentIndex(part), id: '', name: '', arguments: '' };
|
|
334
|
+
calls.push(call);
|
|
335
|
+
}
|
|
336
|
+
if (part.id) {
|
|
175
337
|
call.id = part.id;
|
|
176
|
-
|
|
338
|
+
}
|
|
339
|
+
if (part.function?.name) {
|
|
177
340
|
call.name += part.function.name;
|
|
178
|
-
|
|
341
|
+
}
|
|
342
|
+
if (part.function?.arguments) {
|
|
179
343
|
call.arguments += part.function.arguments;
|
|
344
|
+
}
|
|
180
345
|
}
|
|
181
346
|
}
|
|
182
|
-
/**
|
|
183
|
-
* The call a fragment belongs to.
|
|
184
|
-
*
|
|
185
|
-
* By `index` where the server sent a number, which the SDK types as required and servers have
|
|
186
|
-
* nonetheless left out: keyed on `undefined`, every call joined into one whose name and
|
|
187
|
-
* arguments were all of theirs run together. Without one, by `id`; failing that, a fragment
|
|
188
|
-
* naming a function opens a call and a bare run of arguments continues the latest. A server
|
|
189
|
-
* that sends whole calls one per chunk at index `0` makes the same mistake the other way, so
|
|
190
|
-
* a different id, or a name after arguments have begun, opens a new call under that index.
|
|
191
|
-
*/
|
|
192
|
-
function fragmentOf(part) {
|
|
193
|
-
const index = typeof part.index === "number" ? part.index : undefined;
|
|
194
|
-
const name = part.function?.name;
|
|
195
|
-
const known = index !== undefined
|
|
196
|
-
? calls.findLast((call) => call.index === index)
|
|
197
|
-
: part.id
|
|
198
|
-
? calls.find((call) => call.id === part.id)
|
|
199
|
-
: name
|
|
200
|
-
? undefined
|
|
201
|
-
: calls.at(-1);
|
|
202
|
-
const another = known && ((part.id && known.id && part.id !== known.id) || (name && known.arguments));
|
|
203
|
-
if (known && !another)
|
|
204
|
-
return known;
|
|
205
|
-
const call = { index, id: "", name: "", arguments: "" };
|
|
206
|
-
calls.push(call);
|
|
207
|
-
return call;
|
|
208
|
-
}
|
|
209
347
|
// An aborted stream ends its iteration rather than throwing, so without this a turn cut off
|
|
210
348
|
// halfway — by the watchdog or by someone stopping the run — comes back looking like a
|
|
211
349
|
// complete one, and a truncated answer is recorded as the output. Nothing about the API
|
|
@@ -218,31 +356,16 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
|
|
|
218
356
|
const cached = cacheReport ?? cacheTimings;
|
|
219
357
|
if (cached !== undefined) {
|
|
220
358
|
usage.cached = cached;
|
|
221
|
-
if (usage.prompt > 0)
|
|
359
|
+
if (usage.prompt > 0) {
|
|
222
360
|
usage.uncached = Math.max(0, usage.prompt - cached);
|
|
361
|
+
}
|
|
223
362
|
}
|
|
224
|
-
const minted = new Set();
|
|
225
363
|
return {
|
|
226
364
|
content: splitter.output,
|
|
227
|
-
toolCalls: calls
|
|
228
|
-
.map((call, order) => ({ call, order }))
|
|
229
|
-
.sort((a, b) => (a.call.index ?? a.order) - (b.call.index ?? b.order) || a.order - b.order)
|
|
230
|
-
.map(({ call }, position) => {
|
|
231
|
-
// A server that streams a call without an id still needs one for the result to answer,
|
|
232
|
-
// and two calls it put under one index must not be answered as one.
|
|
233
|
-
let id = call.id || `call_${call.index ?? position}`;
|
|
234
|
-
if (minted.has(id))
|
|
235
|
-
id = `call_${position}_${minted.size}`;
|
|
236
|
-
minted.add(id);
|
|
237
|
-
return {
|
|
238
|
-
id,
|
|
239
|
-
type: "function",
|
|
240
|
-
function: { name: call.name, arguments: call.arguments },
|
|
241
|
-
};
|
|
242
|
-
}),
|
|
365
|
+
toolCalls: assembledCalls(calls),
|
|
243
366
|
usage,
|
|
244
367
|
finishReason,
|
|
245
|
-
reasoning: reasoning.join(
|
|
368
|
+
reasoning: reasoning.join('') + splitter.reasoning,
|
|
246
369
|
};
|
|
247
370
|
}
|
|
248
371
|
}
|