@cubicecho/agent-core 2.24.0 → 2.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/agent-loop.d.ts +84 -205
- package/dist/agent-loop.js +288 -597
- package/dist/calibration.d.ts +14 -10
- package/dist/calibration.js +30 -23
- package/dist/capabilities.d.ts +111 -42
- package/dist/capabilities.js +281 -138
- package/dist/client.d.ts +104 -35
- package/dist/client.js +205 -74
- package/dist/compaction.d.ts +103 -60
- package/dist/compaction.js +160 -86
- package/dist/config.d.ts +48 -12
- package/dist/config.js +20 -1
- package/dist/continuation.d.ts +24 -16
- package/dist/continuation.js +70 -44
- package/dist/digest.d.ts +7 -0
- package/dist/digest.js +18 -0
- package/dist/errors.d.ts +24 -15
- package/dist/errors.js +23 -15
- package/dist/events.d.ts +100 -63
- package/dist/events.js +250 -120
- package/dist/guards.d.ts +49 -4
- package/dist/guards.js +60 -5
- package/dist/hook-events.d.ts +21 -5
- package/dist/hook-events.js +23 -9
- package/dist/hooks.d.ts +96 -73
- package/dist/hooks.js +202 -116
- package/dist/index.d.ts +30 -26
- package/dist/index.js +29 -24
- package/dist/ledger.d.ts +62 -30
- package/dist/ledger.js +163 -56
- package/dist/platform.d.ts +15 -0
- package/dist/platform.js +15 -0
- package/dist/preselect.d.ts +223 -0
- package/dist/preselect.js +291 -0
- package/dist/request-body.d.ts +32 -0
- package/dist/request-body.js +47 -0
- package/dist/reset.d.ts +1 -0
- package/dist/reset.js +7 -6
- package/dist/retry.d.ts +44 -138
- package/dist/retry.js +83 -281
- package/dist/run-calls.d.ts +123 -0
- package/dist/run-calls.js +351 -0
- package/dist/run-turn.d.ts +36 -15
- package/dist/run-turn.js +74 -37
- package/dist/runtime.d.ts +23 -17
- package/dist/runtime.js +34 -24
- package/dist/schema-compat.d.ts +35 -18
- package/dist/schema-compat.js +346 -176
- package/dist/scope.d.ts +28 -5
- package/dist/scope.js +37 -10
- package/dist/side-task.d.ts +124 -40
- package/dist/side-task.js +186 -66
- package/dist/snapshot.d.ts +10 -5
- package/dist/snapshot.js +73 -30
- package/dist/spec.d.ts +56 -25
- package/dist/spec.js +494 -215
- package/dist/stream.d.ts +66 -16
- package/dist/stream.js +220 -97
- package/dist/thinking.d.ts +23 -8
- package/dist/thinking.js +92 -44
- package/dist/tokens.d.ts +164 -6
- package/dist/tokens.js +319 -6
- package/dist/tool-calls.d.ts +90 -17
- package/dist/tool-calls.js +207 -84
- package/dist/tool-loading.d.ts +101 -189
- package/dist/tool-loading.js +257 -371
- package/dist/wire.d.ts +81 -0
- package/dist/wire.js +73 -0
- package/llms.txt +76 -39
- package/package.json +4 -4
package/dist/events.js
CHANGED
|
@@ -1,18 +1,28 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { getOrCreate } from './guards.js';
|
|
2
|
+
import { ABORT_EVENT, MS_PER_MINUTE } from './platform.js';
|
|
3
|
+
import { assignSettings, scoped } from './scope.js';
|
|
4
|
+
import { CacheBreakReason } from './stream.js';
|
|
5
|
+
import { LOAD_TOOLS } from './tool-loading.js';
|
|
6
|
+
import { FinishReason } from './wire.js';
|
|
7
|
+
/** How many minutes an unended run's events are kept, absent a `retainUnendedMs`. */
|
|
8
|
+
const RETAIN_UNENDED_MINUTES = 30;
|
|
4
9
|
/** The numbers a run of the shape this bus was written for wants. */
|
|
5
10
|
const DEFAULTS = {
|
|
6
11
|
maxEvents: 1000,
|
|
7
12
|
trimSlack: 256,
|
|
8
|
-
retainMs:
|
|
9
|
-
retainUnendedMs:
|
|
13
|
+
retainMs: MS_PER_MINUTE,
|
|
14
|
+
retainUnendedMs: RETAIN_UNENDED_MINUTES * MS_PER_MINUTE,
|
|
10
15
|
};
|
|
11
16
|
const bus = scoped(() => ({ limits: { ...DEFAULTS }, streams: new Map(), sweeping: null }));
|
|
12
17
|
/**
|
|
13
18
|
* Changes what the bus keeps, for a process whose runs are not shaped like the ones these
|
|
14
19
|
* defaults were chosen for.
|
|
15
20
|
*
|
|
21
|
+
* @param [options] - The bounds to change. A field left out — or given anything that is not a
|
|
22
|
+
* number above zero — keeps what it has, so a partial or a half-built config narrows nothing.
|
|
23
|
+
* @returns Everything in force afterwards, including what this call did not change.
|
|
24
|
+
*
|
|
25
|
+
* @remarks
|
|
16
26
|
* The bus is one thing per runtime rather than an object a caller holds, so this is too: it is
|
|
17
27
|
* a deployment's setting, said once at startup, and not something to move around under a run.
|
|
18
28
|
* Called at the top level that is the process's bus; a runtime made by `createRuntime` has its
|
|
@@ -24,59 +34,83 @@ const bus = scoped(() => ({ limits: { ...DEFAULTS }, streams: new Map(), sweepin
|
|
|
24
34
|
* Changes apply from the next event and the next sweep. Nothing already buffered is trimmed to
|
|
25
35
|
* a cap that has just come down, because the trim happens on push; the backlog settles to the
|
|
26
36
|
* new number as the run goes on.
|
|
27
|
-
*
|
|
28
|
-
* @param options The bounds to change. A field left out — or given anything that is not a
|
|
29
|
-
* number above zero — keeps what it has, so a partial or a half-built config narrows nothing.
|
|
30
|
-
* @returns Everything in force afterwards, including what this call did not change.
|
|
31
37
|
*/
|
|
32
38
|
export function configureEvents(options = {}) {
|
|
33
|
-
|
|
34
|
-
for (const [name, value] of Object.entries(options)) {
|
|
35
|
-
if (isPositive(value))
|
|
36
|
-
limits[name] = value;
|
|
37
|
-
}
|
|
38
|
-
return { ...limits };
|
|
39
|
+
return assignSettings(bus().limits, options);
|
|
39
40
|
}
|
|
41
|
+
/** Which kind of thing happened, and what `text`, `name`, `ok` and `usage` carry for it. */
|
|
42
|
+
export const RunEventKind = {
|
|
43
|
+
/** A step of a caller's own flow began. `name` is the step, `text` its kind. */
|
|
44
|
+
Step: 'step',
|
|
45
|
+
/** A decision step chose an arm. `text` is the arm it took. */
|
|
46
|
+
Decision: 'decision',
|
|
47
|
+
/** A new turn of the agent loop began, inside whichever step is running. */
|
|
48
|
+
Turn: 'turn',
|
|
49
|
+
/** Reasoning tokens, as they arrive. */
|
|
50
|
+
Thinking: 'thinking',
|
|
51
|
+
/** Reply tokens, as they arrive. */
|
|
52
|
+
Output: 'output',
|
|
53
|
+
/** The model asked for a tool, with the arguments it chose. `id` is the call's. */
|
|
54
|
+
ToolCall: 'tool-call',
|
|
55
|
+
/** A tool came back, with what it said. `id` is the id of the call it answers. */
|
|
56
|
+
ToolResult: 'tool-result',
|
|
57
|
+
/** Something the runner did that is not the model's doing — a preselection, a retry. */
|
|
58
|
+
Notice: 'notice',
|
|
59
|
+
/**
|
|
60
|
+
* What the run has cost so far, at the end of every turn, and what that one turn did. Sent
|
|
61
|
+
* whether or not the endpoint reported tokens, since the timings and the cache comparison are
|
|
62
|
+
* measured either way.
|
|
63
|
+
*/
|
|
64
|
+
Usage: 'usage',
|
|
65
|
+
/** The run ended. Always last, and always sent. */
|
|
66
|
+
Done: 'done',
|
|
67
|
+
};
|
|
40
68
|
/**
|
|
41
69
|
* A whole event from what a caller said of it: the empty defaults under it, the run and sequence over it.
|
|
42
70
|
*
|
|
71
|
+
* @param input - What happened. An `at` it carries wins over the one given here.
|
|
72
|
+
* @param runId - The run it belongs to. Written after `input`, which gets no say in it.
|
|
73
|
+
* @param seq - Its place in that run, the same.
|
|
74
|
+
* @param at - When, for an input that does not say.
|
|
75
|
+
* @returns A new event; `input` is not written to.
|
|
76
|
+
*
|
|
77
|
+
* @remarks
|
|
43
78
|
* The one place the defaults are written, so an event the bus stamps, one the loop keeps for its
|
|
44
79
|
* own metrics and the notice a slow watcher is sent cannot come to disagree about what an unset
|
|
45
80
|
* field reads as.
|
|
46
|
-
*
|
|
47
|
-
* @param input What happened. An `at` it carries wins over the one given here.
|
|
48
|
-
* @param runId The run it belongs to. Written after `input`, which gets no say in it.
|
|
49
|
-
* @param seq Its place in that run, the same.
|
|
50
|
-
* @param at When, for an input that does not say.
|
|
51
81
|
*/
|
|
52
82
|
export const stamp = (input, runId, seq, at) => ({
|
|
53
83
|
at,
|
|
54
|
-
text:
|
|
55
|
-
name:
|
|
56
|
-
step:
|
|
84
|
+
text: '',
|
|
85
|
+
name: '',
|
|
86
|
+
step: '',
|
|
57
87
|
ok: null,
|
|
58
88
|
usage: null,
|
|
59
89
|
...input,
|
|
60
90
|
runId,
|
|
61
91
|
seq,
|
|
62
92
|
});
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
};
|
|
93
|
+
/**
|
|
94
|
+
* A run's stream on a bus, begun empty where the run has none.
|
|
95
|
+
*
|
|
96
|
+
* @param held - The bus to look on. Gains an entry on a miss.
|
|
97
|
+
* @param runId - The run whose stream is wanted.
|
|
98
|
+
* @returns The stream the bus holds, not a copy — writing to it is writing to the run.
|
|
99
|
+
*/
|
|
100
|
+
const streamFor = ({ streams }, runId) => getOrCreate(streams, runId, () => ({
|
|
101
|
+
events: [],
|
|
102
|
+
listeners: new Set(),
|
|
103
|
+
seq: 0,
|
|
104
|
+
touched: Date.now(),
|
|
105
|
+
ended: false,
|
|
106
|
+
}));
|
|
77
107
|
/**
|
|
78
108
|
* Drops the streams nobody is reading and nothing is writing to.
|
|
79
109
|
*
|
|
110
|
+
* @param held - The bus to sweep. Loses each unwatched stream untouched for its retention or
|
|
111
|
+
* longer, and has its timer cleared and — while any stream is left — set again.
|
|
112
|
+
*
|
|
113
|
+
* @remarks
|
|
80
114
|
* Cleanup used to hang entirely off `done`, which assumed every run reaches it. A run killed by
|
|
81
115
|
* an uncaught throw, a signal, or a caller that simply forgets pinned its backlog for the life
|
|
82
116
|
* of the process — and in a long-lived server that map only ever grew. The `done` timer had the
|
|
@@ -91,16 +125,25 @@ function sweep(held) {
|
|
|
91
125
|
held.sweeping = null;
|
|
92
126
|
const now = Date.now();
|
|
93
127
|
for (const [runId, stream] of streams) {
|
|
94
|
-
if (stream.listeners.size)
|
|
128
|
+
if (stream.listeners.size) {
|
|
95
129
|
continue;
|
|
96
|
-
|
|
130
|
+
}
|
|
131
|
+
if (stream.touched <= now - (stream.ended ? limits.retainMs : limits.retainUnendedMs)) {
|
|
97
132
|
streams.delete(runId);
|
|
133
|
+
}
|
|
98
134
|
}
|
|
99
135
|
scheduleSweep(held);
|
|
100
136
|
}
|
|
137
|
+
/**
|
|
138
|
+
* Arms a bus's one sweep timer, unless it is armed already or the bus holds no stream to expire.
|
|
139
|
+
*
|
|
140
|
+
* @param held - The bus. Its `sweeping` is set to a timer that fires after `retainMs` as it stands
|
|
141
|
+
* now, and that does not keep the process alive.
|
|
142
|
+
*/
|
|
101
143
|
function scheduleSweep(held) {
|
|
102
|
-
if (held.sweeping || held.streams.size === 0)
|
|
144
|
+
if (held.sweeping || held.streams.size === 0) {
|
|
103
145
|
return;
|
|
146
|
+
}
|
|
104
147
|
held.sweeping = setTimeout(() => sweep(held), held.limits.retainMs);
|
|
105
148
|
held.sweeping.unref?.();
|
|
106
149
|
}
|
|
@@ -109,28 +152,32 @@ function scheduleSweep(held) {
|
|
|
109
152
|
* loop threw where it could not be caught. The sweep gets there on its own; this is for a
|
|
110
153
|
* caller that already knows.
|
|
111
154
|
*
|
|
112
|
-
* @param runId The run to forget. An id nothing was emitted under is ignored.
|
|
155
|
+
* @param runId - The run to forget. An id nothing was emitted under is ignored.
|
|
113
156
|
*/
|
|
114
157
|
export function endRun(runId) {
|
|
115
158
|
const { streams } = bus();
|
|
116
159
|
const stream = streams.get(runId);
|
|
117
|
-
if (!stream)
|
|
160
|
+
if (!stream) {
|
|
118
161
|
return;
|
|
162
|
+
}
|
|
119
163
|
// Watchers are parked on a promise that only an `emit` to *this* stream can resolve, and the
|
|
120
164
|
// delete below puts it beyond the reach of every later one — the next `emit` builds a fresh
|
|
121
165
|
// stream and wakes nobody. So they are told the run is over first: a watcher that is handed
|
|
122
166
|
// `done` completes, runs its `finally` and lets its consumer go, where one left parked holds
|
|
123
167
|
// an open subscription that can never say anything again. An SSE client on the other end of
|
|
124
168
|
// that is a connection that never closes.
|
|
125
|
-
if (stream.listeners.size)
|
|
126
|
-
emit(runId, { kind:
|
|
169
|
+
if (stream.listeners.size) {
|
|
170
|
+
emit(runId, { kind: RunEventKind.Done, ok: false, text: 'run ended' });
|
|
171
|
+
}
|
|
127
172
|
streams.delete(runId);
|
|
128
173
|
}
|
|
129
174
|
/**
|
|
130
175
|
* Records one event and hands it to everyone watching that run. Never throws at the caller.
|
|
131
176
|
*
|
|
132
|
-
* @param runId The run this belongs to. Created on first use.
|
|
133
|
-
* @param input The event. `kind` is required; `runId` and `seq` are not a caller's to set.
|
|
177
|
+
* @param runId - The run this belongs to. Created on first use.
|
|
178
|
+
* @param input - The event. `kind` is required; `runId` and `seq` are not a caller's to set.
|
|
179
|
+
* @returns The event as it was recorded: its `seq` in the run, its time, and every unset field
|
|
180
|
+
* filled in.
|
|
134
181
|
*/
|
|
135
182
|
export function emit(runId, input) {
|
|
136
183
|
const held = bus();
|
|
@@ -146,8 +193,9 @@ export function emit(runId, input) {
|
|
|
146
193
|
}
|
|
147
194
|
// Kept for a moment so a watcher that arrives just after the end still sees how it went,
|
|
148
195
|
// then dropped: a finished run's record is the row, not this.
|
|
149
|
-
if (event.kind ===
|
|
196
|
+
if (event.kind === RunEventKind.Done) {
|
|
150
197
|
stream.ended = true;
|
|
198
|
+
}
|
|
151
199
|
scheduleSweep(held);
|
|
152
200
|
for (const listener of stream.listeners) {
|
|
153
201
|
// The guarantee above is the point of this: a listener that throws must not take out the
|
|
@@ -162,22 +210,57 @@ export function emit(runId, input) {
|
|
|
162
210
|
/**
|
|
163
211
|
* Everything that has happened on a run, then everything that happens next, until it ends.
|
|
164
212
|
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
* @param runId The run to follow. One that has not started yet is waited on, not refused.
|
|
169
|
-
* @param signal Stops following. The only other way out is the run's own `done`, and a watcher
|
|
213
|
+
* @param runId - The run to follow. One that has not started yet is waited on, not refused.
|
|
214
|
+
* @param [signal] - Stops following. The only other way out is the run's own `done`, and a watcher
|
|
170
215
|
* with no way out is a leak rather than a lost backlog: the sweep below skips any stream a
|
|
171
216
|
* listener is on, so a run that dies without `done` pins its backlog for the life of the process.
|
|
172
217
|
* Returning the generator is not that way out — parked on the promise at the foot of this
|
|
173
218
|
* function it is suspended at an `await` rather than at a `yield`, and a `return()` there is
|
|
174
219
|
* queued behind a promise only the next event can settle. An abort resolves that promise itself.
|
|
220
|
+
* @returns The run's events in order, ending after its `done` or at the abort. A watcher that fell
|
|
221
|
+
* too far behind is given one notice in place of the events it missed. Nothing is subscribed until
|
|
222
|
+
* the first event is asked for.
|
|
223
|
+
*
|
|
224
|
+
* @remarks
|
|
225
|
+
* The backlog comes first so a watcher that joins halfway through — or after the run finished,
|
|
226
|
+
* inside the retention window — reads the same story as one that was there from the start.
|
|
175
227
|
*/
|
|
176
228
|
export function watch(runId, signal) {
|
|
177
229
|
// Read here, not in the generator: its body runs when the first event is asked for, and by
|
|
178
230
|
// then the caller may be outside the runtime whose run this is.
|
|
179
231
|
return watching(bus(), runId, signal);
|
|
180
232
|
}
|
|
233
|
+
/**
|
|
234
|
+
* The notice a watcher that fell behind is given in place of the events it missed.
|
|
235
|
+
*
|
|
236
|
+
* @param runId - The run the watcher is following.
|
|
237
|
+
* @param gap - How many events were dropped, for the text.
|
|
238
|
+
* @param next - The event the notice goes ahead of, which gives it its step, its time and — less
|
|
239
|
+
* one — its seq.
|
|
240
|
+
* @returns A `notice` event made for that one watcher: it is in no backlog.
|
|
241
|
+
*
|
|
242
|
+
* @remarks
|
|
243
|
+
* One short of the event it precedes, which is the last seq that went missing. Sharing a seq
|
|
244
|
+
* with the event behind it made the notice indistinguishable from a duplicate, and de-duplicating
|
|
245
|
+
* on `seq` is the one thing the sequence is documented for — so a client doing exactly that
|
|
246
|
+
* dropped either the gap notice or the event it explains. Inside the gap there is nothing to
|
|
247
|
+
* collide with: those seqs reach no watcher.
|
|
248
|
+
*/
|
|
249
|
+
const gapNotice = (runId, gap, next) => stamp({
|
|
250
|
+
kind: RunEventKind.Notice,
|
|
251
|
+
text: `${gap} event(s) dropped: this watcher fell too far behind`,
|
|
252
|
+
step: next.step,
|
|
253
|
+
}, runId, next.seq - 1, next.at);
|
|
254
|
+
/**
|
|
255
|
+
* The generator behind `watch`, on a bus already chosen.
|
|
256
|
+
*
|
|
257
|
+
* @param held - The bus the run is on, kept for the generator's life whichever runtime resumes it.
|
|
258
|
+
* @param runId - The run to follow. A stream is begun for one that has none, and dropped on the way
|
|
259
|
+
* out where this was its last watcher and the run has ended or never emitted.
|
|
260
|
+
* @param [signal] - Stops following. One already aborted yields nothing, the backlog included.
|
|
261
|
+
* @returns The backlog as it stood at the first `next()`, then each event as it is emitted, ending
|
|
262
|
+
* after `done` or at the abort.
|
|
263
|
+
*/
|
|
181
264
|
async function* watching(held, runId, signal) {
|
|
182
265
|
const { limits, streams } = held;
|
|
183
266
|
const stream = streamFor(held, runId);
|
|
@@ -211,7 +294,7 @@ async function* watching(held, runId, signal) {
|
|
|
211
294
|
// The same wake the listener uses: an abort is another reason to stop waiting, and what the
|
|
212
295
|
// loop does about it is decided in one place below rather than here.
|
|
213
296
|
const onAbort = () => wake?.();
|
|
214
|
-
signal?.addEventListener(
|
|
297
|
+
signal?.addEventListener(ABORT_EVENT, onAbort, { once: true });
|
|
215
298
|
try {
|
|
216
299
|
for (;;) {
|
|
217
300
|
// Guarding the drain rather than sitting after it, so an already-aborted signal leaves
|
|
@@ -233,28 +316,20 @@ async function* watching(held, runId, signal) {
|
|
|
233
316
|
if (dropped > 0) {
|
|
234
317
|
// Said once per gap rather than per event, and before the event that follows it, so a
|
|
235
318
|
// client reading `seq` sees why the numbers jump instead of assuming it lost its place.
|
|
236
|
-
//
|
|
237
|
-
// One short of the event it precedes, which is the last seq that went missing. Sharing
|
|
238
|
-
// a seq with the event behind it made the notice indistinguishable from a duplicate,
|
|
239
|
-
// and de-duplicating on `seq` is the one thing the sequence is documented for — so a
|
|
240
|
-
// client doing exactly that dropped either the gap notice or the event it explains.
|
|
241
|
-
// Inside the gap there is nothing to collide with: those seqs reach no watcher.
|
|
242
319
|
const gap = dropped;
|
|
243
320
|
dropped = 0;
|
|
244
|
-
yield
|
|
245
|
-
kind: "notice",
|
|
246
|
-
text: `${gap} event(s) dropped: this watcher fell too far behind`,
|
|
247
|
-
step: event.step,
|
|
248
|
-
}, runId, event.seq - 1, event.at);
|
|
321
|
+
yield gapNotice(runId, gap, event);
|
|
249
322
|
}
|
|
250
323
|
yield event;
|
|
251
324
|
// `done` is the last event a run will ever have, so the subscription completes rather
|
|
252
325
|
// than leaving the client holding an open stream that will never say anything again.
|
|
253
|
-
if (event.kind ===
|
|
326
|
+
if (event.kind === RunEventKind.Done) {
|
|
254
327
|
return;
|
|
328
|
+
}
|
|
255
329
|
}
|
|
256
|
-
if (signal?.aborted)
|
|
330
|
+
if (signal?.aborted) {
|
|
257
331
|
return;
|
|
332
|
+
}
|
|
258
333
|
await new Promise((resolve) => {
|
|
259
334
|
wake = resolve;
|
|
260
335
|
});
|
|
@@ -265,7 +340,7 @@ async function* watching(held, runId, signal) {
|
|
|
265
340
|
// `once` covers the abort that fired; this is for the one that never did, which would
|
|
266
341
|
// otherwise hold this generator and its queue alive for as long as the caller holds the
|
|
267
342
|
// signal — a run's whole backlog kept by a watcher that finished on `done`.
|
|
268
|
-
signal?.removeEventListener(
|
|
343
|
+
signal?.removeEventListener(ABORT_EVENT, onAbort);
|
|
269
344
|
stream.listeners.delete(listener);
|
|
270
345
|
// A watcher can name a run that has not started, or will never start. Nothing was recorded
|
|
271
346
|
// under it, so nothing is left behind either — and a run that has ended has nothing more to
|
|
@@ -273,9 +348,10 @@ async function* watching(held, runId, signal) {
|
|
|
273
348
|
// Against `stream` rather than the id: this generator may have outlived its own entry — a
|
|
274
349
|
// sweep or an `endRun` drops it and a later `emit` files the same run under a new one — and
|
|
275
350
|
// the last watcher of the old stream has no business deleting the new one.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
351
|
+
const stillFiled = streams.get(runId) === stream;
|
|
352
|
+
const unwatched = stream.listeners.size === 0;
|
|
353
|
+
const nothingMoreToSay = stream.ended || stream.events.length === 0;
|
|
354
|
+
if (stillFiled && unwatched && nothingMoreToSay) {
|
|
279
355
|
streams.delete(runId);
|
|
280
356
|
}
|
|
281
357
|
}
|
|
@@ -283,17 +359,20 @@ async function* watching(held, runId, signal) {
|
|
|
283
359
|
/**
|
|
284
360
|
* The backlog alone, for a caller that wants a snapshot rather than a subscription.
|
|
285
361
|
*
|
|
362
|
+
* @param runId - The run to read. An unknown or already-swept run gives an empty array.
|
|
363
|
+
* @returns The events still held, oldest first, in an array of the caller's own.
|
|
364
|
+
*
|
|
365
|
+
* @remarks
|
|
286
366
|
* The array is a copy; the events in it are not. They are the same objects the bus holds and
|
|
287
367
|
* every watcher was handed, so writing to one rewrites the run for everybody — which is what
|
|
288
368
|
* `fold` copies to avoid, and this is the other half of the same warning. Read them, or copy
|
|
289
369
|
* what you mean to change.
|
|
290
|
-
*
|
|
291
|
-
* @param runId The run to read. An unknown or already-swept run gives an empty array.
|
|
292
370
|
*/
|
|
293
371
|
export const history = (runId) => [...(bus().streams.get(runId)?.events ?? [])];
|
|
294
372
|
/**
|
|
295
373
|
* Test seam: forget every run, so one test's events cannot be read by the next.
|
|
296
374
|
*
|
|
375
|
+
* @remarks
|
|
297
376
|
* Named for what it forgets rather than bare `reset`, which sat in a consumer's imports beside
|
|
298
377
|
* `resetAll`, `resetClients`, `resetCapabilities` and `resetHints` saying nothing about which
|
|
299
378
|
* of the five it was — `reset.ts` had to alias it on the way in to stay readable.
|
|
@@ -306,21 +385,27 @@ export const history = (runId) => [...(bus().streams.get(runId)?.events ?? [])];
|
|
|
306
385
|
export const resetEvents = () => {
|
|
307
386
|
const held = bus();
|
|
308
387
|
held.streams.clear();
|
|
309
|
-
if (held.sweeping)
|
|
388
|
+
if (held.sweeping) {
|
|
310
389
|
clearTimeout(held.sweeping);
|
|
390
|
+
}
|
|
311
391
|
held.sweeping = null;
|
|
312
392
|
// In place, so a watcher still open reads the defaults too and not the caps it began under.
|
|
313
393
|
Object.assign(held.limits, DEFAULTS);
|
|
314
394
|
};
|
|
315
395
|
/**
|
|
316
|
-
*
|
|
396
|
+
* A run's events with each stretch of `thinking` or of `output` joined into one block, for a
|
|
397
|
+
* reader that takes a run in snapshots.
|
|
317
398
|
*
|
|
318
|
-
*
|
|
399
|
+
* @param events - Events in `seq` order, from `history` or collected from `watch`. Not written to.
|
|
400
|
+
* @returns One event per block, every one a copy. A run of `thinking` or of `output` within one
|
|
401
|
+
* step is a single block with the texts joined; anything else is a block to itself.
|
|
402
|
+
*
|
|
403
|
+
* @remarks
|
|
404
|
+
* Consecutive tokens of one kind are one thing being said, not hundreds of things. A client that
|
|
405
|
+
* reads a run in snapshots rather than token by token wants it that way: a
|
|
319
406
|
* reasoning model spends ten thousand deltas on a paragraph, and a paragraph is what it meant.
|
|
320
407
|
* Each block carries the `seq` of its last event, so asking for what came after one block
|
|
321
408
|
* picks up exactly where it left off.
|
|
322
|
-
*
|
|
323
|
-
* @param events Events in `seq` order, from `history` or collected from `watch`.
|
|
324
409
|
*/
|
|
325
410
|
export function fold(events) {
|
|
326
411
|
const blocks = [];
|
|
@@ -329,17 +414,20 @@ export function fold(events) {
|
|
|
329
414
|
// built ten thousand times to produce it once.
|
|
330
415
|
let parts = [];
|
|
331
416
|
const close = () => {
|
|
332
|
-
if (!parts.length)
|
|
417
|
+
if (!parts.length) {
|
|
333
418
|
return;
|
|
419
|
+
}
|
|
334
420
|
const last = blocks[blocks.length - 1];
|
|
335
|
-
if (parts.length > 1)
|
|
336
|
-
last.text = parts.join(
|
|
421
|
+
if (parts.length > 1) {
|
|
422
|
+
last.text = parts.join('');
|
|
423
|
+
}
|
|
337
424
|
parts = [];
|
|
338
425
|
};
|
|
339
426
|
for (const event of events) {
|
|
340
427
|
const last = blocks[blocks.length - 1];
|
|
341
|
-
const mergeable = event.kind ===
|
|
342
|
-
|
|
428
|
+
const mergeable = event.kind === RunEventKind.Thinking || event.kind === RunEventKind.Output;
|
|
429
|
+
const continuesLast = last && last.kind === event.kind && last.step === event.step;
|
|
430
|
+
if (mergeable && continuesLast) {
|
|
343
431
|
last.seq = event.seq;
|
|
344
432
|
last.at = event.at;
|
|
345
433
|
parts.push(event.text);
|
|
@@ -351,25 +439,58 @@ export function fold(events) {
|
|
|
351
439
|
// `fold(history(id))[0].text = ...` rewrite the bus, and every watcher after it read the
|
|
352
440
|
// rewrite.
|
|
353
441
|
blocks.push({ ...event });
|
|
354
|
-
if (mergeable)
|
|
442
|
+
if (mergeable) {
|
|
355
443
|
parts.push(event.text);
|
|
444
|
+
}
|
|
356
445
|
}
|
|
357
446
|
}
|
|
358
447
|
close();
|
|
359
448
|
return blocks;
|
|
360
449
|
}
|
|
450
|
+
/** How a run ended, as `runMetrics` reads it. */
|
|
451
|
+
export const RunOutcome = {
|
|
452
|
+
/** The run did not fail, and its last turn was not cut off. */
|
|
453
|
+
Answered: 'answered',
|
|
454
|
+
/** The run did not fail, and its last turn is an answer the reply ceiling cut off. */
|
|
455
|
+
Truncated: 'truncated',
|
|
456
|
+
/**
|
|
457
|
+
* `done` said the run was not ok. Whether that was an error, a stop or the tool budget is in
|
|
458
|
+
* the host's own `done` text and not here.
|
|
459
|
+
*/
|
|
460
|
+
Failed: 'failed',
|
|
461
|
+
};
|
|
462
|
+
/**
|
|
463
|
+
* How a run that reported `done` ended.
|
|
464
|
+
*
|
|
465
|
+
* @param ok - What the `done` event said. Only `false` is a failure; anything else is read off the turn.
|
|
466
|
+
* @param last - The last turn a `usage` event reported, if any did.
|
|
467
|
+
* @returns `failed` for a `done` that was not ok, `truncated` where the last turn ran into the
|
|
468
|
+
* reply ceiling, and `answered` otherwise.
|
|
469
|
+
*/
|
|
470
|
+
function outcomeOf(ok, last) {
|
|
471
|
+
if (ok === false) {
|
|
472
|
+
return RunOutcome.Failed;
|
|
473
|
+
}
|
|
474
|
+
if (last?.finishReason === FinishReason.Length) {
|
|
475
|
+
return RunOutcome.Truncated;
|
|
476
|
+
}
|
|
477
|
+
return RunOutcome.Answered;
|
|
478
|
+
}
|
|
361
479
|
/**
|
|
362
480
|
* A run's totals, timings and cache findings, derived from the events it emitted.
|
|
363
481
|
*
|
|
482
|
+
* @param events - A run's events in `seq` order, from `history` or collected from `watch`. A backlog
|
|
483
|
+
* that has lost its oldest events to the cap sums what it still has.
|
|
484
|
+
* @param [options] - The served window, for how full the run came to it.
|
|
485
|
+
* @returns A summary of the caller's own. The counts are zero for an empty list, and `outcome` is
|
|
486
|
+
* absent for a run that has not reported `done`.
|
|
487
|
+
*
|
|
488
|
+
* @remarks
|
|
364
489
|
* A sibling of `fold` rather than part of it. `fold` hands back events, and a client renders
|
|
365
490
|
* what it returns as blocks; a summary is another shape, and folding one in would give every
|
|
366
491
|
* consumer of `fold` a block it does not know how to draw. Derived from the `usage` reports' own
|
|
367
492
|
* `turn` rather than the running totals, so a run with several loops in it — a question per loop —
|
|
368
493
|
* adds up the same as one with a single loop.
|
|
369
|
-
*
|
|
370
|
-
* @param events A run's events in `seq` order, from `history` or collected from `watch`. A backlog
|
|
371
|
-
* that has lost its oldest events to the cap sums what it still has.
|
|
372
|
-
* @param options The served window, for how full the run came to it.
|
|
373
494
|
*/
|
|
374
495
|
export function runMetrics(events, { contextLength } = {}) {
|
|
375
496
|
const metrics = {
|
|
@@ -388,10 +509,10 @@ export function runMetrics(events, { contextLength } = {}) {
|
|
|
388
509
|
};
|
|
389
510
|
/** Adds to a field that is absent until something reports it. */
|
|
390
511
|
const add = (field, value) => {
|
|
391
|
-
if (value === undefined)
|
|
512
|
+
if (value === undefined) {
|
|
392
513
|
return;
|
|
393
|
-
|
|
394
|
-
|
|
514
|
+
}
|
|
515
|
+
metrics[field] = (metrics[field] ?? 0) + value;
|
|
395
516
|
};
|
|
396
517
|
let reportedPrompt = 0;
|
|
397
518
|
let reportedCached = 0;
|
|
@@ -400,24 +521,26 @@ export function runMetrics(events, { contextLength } = {}) {
|
|
|
400
521
|
const pending = new Map();
|
|
401
522
|
let last;
|
|
402
523
|
for (const event of events) {
|
|
403
|
-
if (event.kind ===
|
|
524
|
+
if (event.kind === RunEventKind.Step) {
|
|
404
525
|
metrics.steps++;
|
|
405
|
-
|
|
406
|
-
|
|
526
|
+
}
|
|
527
|
+
else if (event.kind === RunEventKind.ToolCall) {
|
|
528
|
+
if (event.name === LOAD_TOOLS) {
|
|
407
529
|
metrics.loadCalls++;
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
pending.set(event.name, waiting);
|
|
530
|
+
}
|
|
531
|
+
getOrCreate(pending, event.name, () => []).push(event.at);
|
|
411
532
|
}
|
|
412
|
-
else if (event.kind ===
|
|
533
|
+
else if (event.kind === RunEventKind.ToolResult) {
|
|
413
534
|
metrics.toolCalls++;
|
|
414
|
-
if (event.ok === false)
|
|
535
|
+
if (event.ok === false) {
|
|
415
536
|
metrics.toolErrors[event.name] = (metrics.toolErrors[event.name] ?? 0) + 1;
|
|
537
|
+
}
|
|
416
538
|
const called = pending.get(event.name)?.shift();
|
|
417
|
-
if (called !== undefined)
|
|
418
|
-
add(
|
|
539
|
+
if (called !== undefined) {
|
|
540
|
+
add('toolMs', event.at - called);
|
|
541
|
+
}
|
|
419
542
|
}
|
|
420
|
-
else if (event.kind ===
|
|
543
|
+
else if (event.kind === RunEventKind.Usage && event.usage?.turn) {
|
|
421
544
|
const turn = event.usage.turn;
|
|
422
545
|
last = turn;
|
|
423
546
|
metrics.turns++;
|
|
@@ -426,45 +549,52 @@ export function runMetrics(events, { contextLength } = {}) {
|
|
|
426
549
|
metrics.completionTokens += turn.completion;
|
|
427
550
|
metrics.cachedTokens += turn.cached;
|
|
428
551
|
if (turn.uncached !== undefined) {
|
|
429
|
-
add(
|
|
552
|
+
add('uncachedTokens', turn.uncached);
|
|
430
553
|
reportedPrompt += turn.prompt;
|
|
431
554
|
reportedCached += turn.cached;
|
|
432
555
|
}
|
|
433
|
-
add(
|
|
434
|
-
add(
|
|
435
|
-
add(
|
|
436
|
-
add(
|
|
437
|
-
add(
|
|
556
|
+
add('reasoningTokens', turn.reasoningTokens);
|
|
557
|
+
add('promptMs', turn.promptMs);
|
|
558
|
+
add('predictedMs', turn.predictedMs);
|
|
559
|
+
add('draftTotal', turn.draftTotal);
|
|
560
|
+
add('draftAccepted', turn.draftAccepted);
|
|
438
561
|
if (turn.cacheBroken) {
|
|
439
562
|
metrics.cacheBreaks++;
|
|
440
|
-
const reason = turn.cacheBreakReason ??
|
|
563
|
+
const reason = turn.cacheBreakReason ?? CacheBreakReason.NoneKnown;
|
|
441
564
|
metrics.cacheBreakReasons[reason] = (metrics.cacheBreakReasons[reason] ?? 0) + 1;
|
|
442
565
|
}
|
|
443
|
-
if (turn.finishReason ===
|
|
566
|
+
if (turn.finishReason === FinishReason.Length) {
|
|
444
567
|
metrics.truncatedTurns++;
|
|
445
|
-
|
|
568
|
+
}
|
|
569
|
+
if (turn.wallMs !== undefined) {
|
|
446
570
|
metrics.slowestTurnMs = Math.max(metrics.slowestTurnMs ?? 0, turn.wallMs);
|
|
571
|
+
}
|
|
447
572
|
if (turn.firstTokenMs !== undefined) {
|
|
448
|
-
add(
|
|
573
|
+
add('firstTokenMs', turn.firstTokenMs);
|
|
449
574
|
firstTokens++;
|
|
450
575
|
}
|
|
451
|
-
if (turn.prompt > 0)
|
|
576
|
+
if (turn.prompt > 0) {
|
|
452
577
|
metrics.largestPrompt = Math.max(metrics.largestPrompt ?? 0, turn.prompt);
|
|
578
|
+
}
|
|
453
579
|
}
|
|
454
|
-
else if (event.kind ===
|
|
455
|
-
metrics.outcome =
|
|
456
|
-
event.ok === false ? "failed" : last?.finishReason === "length" ? "truncated" : "answered";
|
|
580
|
+
else if (event.kind === RunEventKind.Done) {
|
|
581
|
+
metrics.outcome = outcomeOf(event.ok, last);
|
|
457
582
|
}
|
|
458
583
|
}
|
|
459
|
-
if (events.length > 1)
|
|
584
|
+
if (events.length > 1) {
|
|
460
585
|
metrics.wallMs = events[events.length - 1].at - events[0].at;
|
|
461
|
-
|
|
586
|
+
}
|
|
587
|
+
if (reportedPrompt > 0) {
|
|
462
588
|
metrics.cacheHitRatio = reportedCached / reportedPrompt;
|
|
463
|
-
|
|
589
|
+
}
|
|
590
|
+
if (metrics.firstTokenMs !== undefined) {
|
|
464
591
|
metrics.firstTokenMs /= firstTokens;
|
|
465
|
-
|
|
592
|
+
}
|
|
593
|
+
if (metrics.draftTotal && metrics.draftAccepted !== undefined) {
|
|
466
594
|
metrics.draftAcceptance = metrics.draftAccepted / metrics.draftTotal;
|
|
467
|
-
|
|
595
|
+
}
|
|
596
|
+
if (metrics.largestPrompt !== undefined && contextLength && contextLength > 0) {
|
|
468
597
|
metrics.largestPromptShare = metrics.largestPrompt / contextLength;
|
|
598
|
+
}
|
|
469
599
|
return metrics;
|
|
470
600
|
}
|