@juno-ai/bind 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +314 -81
- package/completion/defects.d.ts +138 -0
- package/completion/defects.js +128 -0
- package/completion/index.d.ts +2 -0
- package/completion/index.js +2 -0
- package/completion/watchdog.d.ts +165 -0
- package/completion/watchdog.js +211 -0
- package/contracts/index.d.ts +1 -1
- package/contracts/index.js +1 -1
- package/contracts/turn.d.ts +26 -2
- package/contracts/turn.js +45 -0
- package/index.d.ts +13 -7
- package/index.js +13 -7
- package/loop/index.d.ts +1 -0
- package/loop/index.js +1 -0
- package/loop/tool-loop.d.ts +260 -0
- package/loop/tool-loop.js +288 -0
- package/package.json +10 -2
- package/routing/attempt-errors.d.ts +122 -0
- package/routing/attempt-errors.js +176 -0
- package/routing/executor.js +16 -1
- package/routing/index.d.ts +1 -0
- package/routing/index.js +1 -0
- package/run/children.d.ts +204 -0
- package/run/children.js +226 -0
- package/run/index.d.ts +1 -0
- package/run/index.js +1 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transient defects in an assembled completion, and the retry predicates that
|
|
3
|
+
* decide whether to ask the same endpoint again.
|
|
4
|
+
*
|
|
5
|
+
* A completion can come back structurally intact at the transport layer and
|
|
6
|
+
* still be unusable: the provider cut the stream mid-arguments, or closed it
|
|
7
|
+
* having emitted nothing at all. Neither is an HTTP failure — there is no
|
|
8
|
+
* status code to classify — so nothing upstream in the routing taxonomy sees
|
|
9
|
+
* them. Left undetected they reach the agent loop, where a truncated tool call
|
|
10
|
+
* is rejected as "invalid JSON in tool arguments" and burns a whole recovery
|
|
11
|
+
* turn on something a plain retry fixes.
|
|
12
|
+
*
|
|
13
|
+
* The two defect kinds here are exactly the two that feed
|
|
14
|
+
* `InferenceAttemptError`'s `completion_defect` arm, which the disposition
|
|
15
|
+
* matrix routes to a same-endpoint retry before traversing providers.
|
|
16
|
+
*
|
|
17
|
+
* Everything in this module is pure. Types are structural rather than tied to
|
|
18
|
+
* any SDK's message class, so a host assembling chunks by hand and a host
|
|
19
|
+
* handing over an `openai` message both fit without a cast.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* Additional attempts when a structured-output call returns content that will
|
|
23
|
+
* not parse as JSON: the initial call plus this many retries. Providers
|
|
24
|
+
* occasionally truncate or malform JSON even under a strict schema, and a fresh
|
|
25
|
+
* attempt almost always comes back valid.
|
|
26
|
+
*/
|
|
27
|
+
export const DEFAULT_STRUCTURED_OUTPUT_MAX_RETRIES = 3;
|
|
28
|
+
/**
|
|
29
|
+
* Same-endpoint retry budget for a transiently-corrupt completion. A fresh
|
|
30
|
+
* attempt on the SAME endpoint usually recovers an intact response; a
|
|
31
|
+
* persistently-broken endpoint exhausts these, and the route executor then
|
|
32
|
+
* traverses to the next provider or model.
|
|
33
|
+
*/
|
|
34
|
+
export const DEFAULT_COMPLETION_DEFECT_MAX_RETRIES = 2;
|
|
35
|
+
/** True when `s` parses as JSON. */
|
|
36
|
+
export function jsonParses(s) {
|
|
37
|
+
try {
|
|
38
|
+
JSON.parse(s);
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Find the transient defect in an assembled completion, or `null` when it is
|
|
47
|
+
* usable.
|
|
48
|
+
*
|
|
49
|
+
* **Empty completion.** No content, no tool calls, no refusal. A refusal alone
|
|
50
|
+
* is a legitimate output and is not a defect.
|
|
51
|
+
*
|
|
52
|
+
* **Truncated tool call.** Non-empty arguments that do not parse as JSON are
|
|
53
|
+
* always a truncation — the provider cut the stream mid-arguments. *Empty*
|
|
54
|
+
* arguments are the normal zero-arg shape and are not a defect **unless** the
|
|
55
|
+
* stream was `cutByTokenLimit`: a length-truncated empty-args call is an
|
|
56
|
+
* incomplete message, not a deliberate no-arg call, and exempting it would let
|
|
57
|
+
* a dispatcher execute a tool off a half-streamed turn.
|
|
58
|
+
*
|
|
59
|
+
* `cutByTokenLimit` is the *fact*, not the wire spelling of it — OpenAI says
|
|
60
|
+
* `finish_reason: "length"`, Anthropic says `max_tokens`, Gemini says
|
|
61
|
+
* `MAX_TOKENS`. Taking the fact keeps the one behavioural rule in this module
|
|
62
|
+
* from silently reading `false` for every host that isn't on an
|
|
63
|
+
* OpenAI-compatible gateway, which would apply the zero-arg exemption to
|
|
64
|
+
* exactly the truncated calls it exists to exclude.
|
|
65
|
+
*
|
|
66
|
+
* A tool call whose `arguments` are not a readable string is skipped rather
|
|
67
|
+
* than inspected: {@link CompletionOutputs.tool_calls} accepts a host's full
|
|
68
|
+
* SDK union, and a non-function member (OpenAI's `type: "custom"`) has no
|
|
69
|
+
* arguments to truncate.
|
|
70
|
+
*/
|
|
71
|
+
export function detectCompletionDefect(completion, cutByTokenLimit) {
|
|
72
|
+
const toolCalls = completion.tool_calls;
|
|
73
|
+
const hasToolCalls = toolCalls !== undefined && toolCalls.length > 0;
|
|
74
|
+
if (!completion.content && !hasToolCalls && !completion.refusal) {
|
|
75
|
+
return { kind: "empty_completion" };
|
|
76
|
+
}
|
|
77
|
+
if (hasToolCalls) {
|
|
78
|
+
const truncated = toolCalls.find((toolCall) => {
|
|
79
|
+
const args = toolCall?.function?.arguments;
|
|
80
|
+
if (typeof args !== "string")
|
|
81
|
+
return false;
|
|
82
|
+
const rawArgs = args.trim();
|
|
83
|
+
if (rawArgs.length === 0)
|
|
84
|
+
return cutByTokenLimit;
|
|
85
|
+
return !jsonParses(rawArgs);
|
|
86
|
+
});
|
|
87
|
+
if (truncated !== undefined) {
|
|
88
|
+
return { kind: "truncated_tool_call", toolCall: truncated };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* True when `response_format` asks the model for JSON. Free-form output
|
|
95
|
+
* (`text`, or omitted) is never retried on a parse failure — there is nothing
|
|
96
|
+
* to parse, so every completion would look like a defect.
|
|
97
|
+
*/
|
|
98
|
+
export function expectsJsonOutput(responseFormat) {
|
|
99
|
+
return (responseFormat?.type === "json_schema" || responseFormat?.type === "json_object");
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Whether a structured-output completion holds parseable JSON.
|
|
103
|
+
*
|
|
104
|
+
* Tool calls and refusals are valid non-JSON outcomes and pass. A wholly empty
|
|
105
|
+
* completion is a {@link detectCompletionDefect} concern, but a *whitespace-only*
|
|
106
|
+
* one is truthy there and slips through — so this deliberately does NOT
|
|
107
|
+
* short-circuit on empty content: an empty or whitespace string is not valid
|
|
108
|
+
* JSON, fails the parse, and earns a retry.
|
|
109
|
+
*
|
|
110
|
+
* The ``` / ```json fences some models wrap JSON in are stripped first,
|
|
111
|
+
* mirroring what structured consumers do before parsing. A bare `true` / `false`
|
|
112
|
+
* is itself valid JSON, so a model that answers a boolean schema with the bare
|
|
113
|
+
* literal still passes; only genuinely malformed or truncated JSON fails.
|
|
114
|
+
*/
|
|
115
|
+
export function structuredOutputParses(completion) {
|
|
116
|
+
const toolCalls = completion.tool_calls;
|
|
117
|
+
if (toolCalls !== undefined && toolCalls !== null && toolCalls.length > 0) {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
if (completion.refusal)
|
|
121
|
+
return true;
|
|
122
|
+
const content = (completion.content ?? "").trim();
|
|
123
|
+
const unfenced = content
|
|
124
|
+
.replace(/^```(?:json)?\s*/i, "")
|
|
125
|
+
.replace(/\s*```$/, "")
|
|
126
|
+
.trim();
|
|
127
|
+
return jsonParses(unfenced);
|
|
128
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { createStreamWatchdog, DEFAULT_TIME_TO_FIRST_TOKEN_MS, DEFAULT_INTER_CHUNK_MS, DEFAULT_MAX_CALL_DURATION_MS, type ChunkOutput, type StreamStall, type StreamWatchdog, type StreamWatchdogOptions, } from "./watchdog.js";
|
|
2
|
+
export { detectCompletionDefect, expectsJsonOutput, jsonParses, structuredOutputParses, DEFAULT_STRUCTURED_OUTPUT_MAX_RETRIES, DEFAULT_COMPLETION_DEFECT_MAX_RETRIES, type AssembledCompletion, type CompletionDefect, type CompletionOutputs, type ResponseFormatShape, type StreamedToolCall, } from "./defects.js";
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { createStreamWatchdog, DEFAULT_TIME_TO_FIRST_TOKEN_MS, DEFAULT_INTER_CHUNK_MS, DEFAULT_MAX_CALL_DURATION_MS, } from "./watchdog.js";
|
|
2
|
+
export { detectCompletionDefect, expectsJsonOutput, jsonParses, structuredOutputParses, DEFAULT_STRUCTURED_OUTPUT_MAX_RETRIES, DEFAULT_COMPLETION_DEFECT_MAX_RETRIES, } from "./defects.js";
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming-completion idle watchdog.
|
|
3
|
+
*
|
|
4
|
+
* A chat-completions SDK's `timeout` bounds *establishing* the request and (for
|
|
5
|
+
* a non-streaming call) the whole response — but once a stream has started
|
|
6
|
+
* yielding, nothing bounds the gap between chunks. A provider that opens the
|
|
7
|
+
* stream and then stalls leaves the consumer's `for await` awaiting the next
|
|
8
|
+
* chunk forever, holding its worker slot until some outer deadline fires, or
|
|
9
|
+
* never. Downstream that is the agent that says "thinking…" and never answers.
|
|
10
|
+
*
|
|
11
|
+
* The budget is split into three because the phases of a streaming completion
|
|
12
|
+
* have very different normal latencies, and one number cannot serve all of them:
|
|
13
|
+
*
|
|
14
|
+
* - **Time to first token** — from opening the stream to the first *answer*
|
|
15
|
+
* token. Generous: a high-reasoning model can legitimately think for a while
|
|
16
|
+
* before emitting anything, and even once reasoning deltas are flowing it can
|
|
17
|
+
* sit silent for seconds between finishing its reasoning and emitting the
|
|
18
|
+
* first content/tool token. This budget therefore covers the reasoning phase
|
|
19
|
+
* AND the reasoning→answer gap — {@link ChunkOutput} `"reasoning"` does NOT
|
|
20
|
+
* switch to the tight budget. Arming the tight budget on a reasoning token is
|
|
21
|
+
* a real observed failure: the normal reasoning→content pause of a frontier
|
|
22
|
+
* reasoning model routinely approaches several seconds and tripped
|
|
23
|
+
* "no chunk mid-stream" on healthy turns.
|
|
24
|
+
* - **Inter-chunk** — the gap between chunks once the model's ANSWER output
|
|
25
|
+
* (content / tool-call arguments / refusal) is flowing. Tight: answer chunks
|
|
26
|
+
* normally arrive sub-second, so a multi-second silence mid-answer is already
|
|
27
|
+
* abnormal, and a shorter budget recovers a mid-stream stall far sooner.
|
|
28
|
+
* - **Overall cap** — an absolute wall-clock ceiling, armed at construction and
|
|
29
|
+
* never re-armed. Defense in depth, not a duplicate of the two idle budgets:
|
|
30
|
+
* the idle watchdog only bounds the *gap* between chunks, so a provider that
|
|
31
|
+
* drip-feeds a chunk just under the idle budget forever (a reasoning delta a
|
|
32
|
+
* minute, never reaching an answer) never trips it. The proper bound on total
|
|
33
|
+
* work is the caller's own deadline signal, but not every caller has one, so
|
|
34
|
+
* this guarantees a single call cannot hold a slot indefinitely regardless.
|
|
35
|
+
*
|
|
36
|
+
* It is armed by the constructor rather than by {@link StreamWatchdog.open}
|
|
37
|
+
* because it is the guarantee of last resort: a host that forgets `open()`
|
|
38
|
+
* gets a watchdog that silently never fires, with no type error and no test
|
|
39
|
+
* failure, and the symptom appears only under a stalling provider. Counting
|
|
40
|
+
* the connection handshake against a ceiling this generous costs nothing;
|
|
41
|
+
* leaving the ceiling to an imperative call the host may skip costs a worker
|
|
42
|
+
* slot. The two *idle* budgets genuinely must wait for `open()` — arming them
|
|
43
|
+
* earlier would charge connection time to the first-token budget.
|
|
44
|
+
*
|
|
45
|
+
* The watchdog owns timers and abort signals and nothing else: it does not read
|
|
46
|
+
* the stream, does not know the wire format, and does not phrase the error. It
|
|
47
|
+
* reports a structured {@link StreamStall} and the host words it — the same
|
|
48
|
+
* split the routing modules use, since a stall message is usually product copy.
|
|
49
|
+
*/
|
|
50
|
+
/**
|
|
51
|
+
* What a streamed chunk carried, from the watchdog's point of view. The
|
|
52
|
+
* distinction that matters is `"reasoning"` vs `"answer"`: only answer output
|
|
53
|
+
* arms the tight inter-chunk budget. A chunk carrying both is `"answer"`.
|
|
54
|
+
* A role/metadata-only opening chunk is `"none"` — the model has not produced a
|
|
55
|
+
* token yet, so it must not shorten the budget either.
|
|
56
|
+
*/
|
|
57
|
+
export type ChunkOutput = "none" | "reasoning" | "answer";
|
|
58
|
+
/** Why the watchdog tore the stream down. */
|
|
59
|
+
export type StreamStall =
|
|
60
|
+
/** The absolute wall-clock ceiling elapsed before the stream finished. */
|
|
61
|
+
Readonly<{
|
|
62
|
+
kind: "overall_cap";
|
|
63
|
+
limitMs: number;
|
|
64
|
+
}>
|
|
65
|
+
/**
|
|
66
|
+
* No answer token within the generous budget. `sawReasoning` distinguishes
|
|
67
|
+
* "the provider never said anything" from "it streamed reasoning and never
|
|
68
|
+
* reached an answer" — different upstream faults with the same budget.
|
|
69
|
+
*/
|
|
70
|
+
| Readonly<{
|
|
71
|
+
kind: "time_to_first_token";
|
|
72
|
+
limitMs: number;
|
|
73
|
+
sawReasoning: boolean;
|
|
74
|
+
}>
|
|
75
|
+
/** Answer output was flowing and then stopped mid-stream. */
|
|
76
|
+
| Readonly<{
|
|
77
|
+
kind: "inter_chunk";
|
|
78
|
+
limitMs: number;
|
|
79
|
+
}>;
|
|
80
|
+
export interface StreamWatchdogOptions {
|
|
81
|
+
/** Generous first-answer-token budget. Default {@link DEFAULT_TIME_TO_FIRST_TOKEN_MS}. */
|
|
82
|
+
readonly timeToFirstTokenMs?: number;
|
|
83
|
+
/** Tight budget between chunks once answer output flows. Default {@link DEFAULT_INTER_CHUNK_MS}. */
|
|
84
|
+
readonly interChunkMs?: number;
|
|
85
|
+
/**
|
|
86
|
+
* Absolute ceiling on the whole call, measured from **watchdog
|
|
87
|
+
* construction** — not from {@link StreamWatchdog.open}, which only arms the
|
|
88
|
+
* idle budgets. Size it to include whatever the host does between
|
|
89
|
+
* constructing the watchdog and opening the stream (connecting, sending the
|
|
90
|
+
* request, waiting on response headers), since all of that is inside the
|
|
91
|
+
* budget. Default {@link DEFAULT_MAX_CALL_DURATION_MS}.
|
|
92
|
+
*/
|
|
93
|
+
readonly maxCallDurationMs?: number;
|
|
94
|
+
/**
|
|
95
|
+
* The caller's own abort signal (a run deadline, a cancellation). Composed
|
|
96
|
+
* into {@link StreamWatchdog.signal}, and — critically — consulted by
|
|
97
|
+
* {@link StreamWatchdog.stall}: when the caller aborted, the teardown is the
|
|
98
|
+
* caller's, not a stall, and must not be reclassified as a retriable
|
|
99
|
+
* upstream fault.
|
|
100
|
+
*/
|
|
101
|
+
readonly external?: AbortSignal | null;
|
|
102
|
+
/**
|
|
103
|
+
* Timer port, defaulting to the ambient globals. Inject a controllable clock
|
|
104
|
+
* to test budget behaviour exactly, rather than sleeping a real interval and
|
|
105
|
+
* hoping the machine keeps up — a watchdog test asserting "nothing fired yet"
|
|
106
|
+
* against a real timer is a flake waiting for a loaded CI box.
|
|
107
|
+
*
|
|
108
|
+
* The handle is opaque (`unknown`) so a fake can hand back whatever it likes;
|
|
109
|
+
* only {@link StreamTimers.clear} ever consumes it.
|
|
110
|
+
*/
|
|
111
|
+
readonly timers?: StreamTimers;
|
|
112
|
+
}
|
|
113
|
+
/** The subset of the timer API the watchdog needs. */
|
|
114
|
+
export interface StreamTimers {
|
|
115
|
+
set(callback: () => void, ms: number): unknown;
|
|
116
|
+
clear(handle: unknown): void;
|
|
117
|
+
}
|
|
118
|
+
export interface StreamWatchdog {
|
|
119
|
+
/** Hand this to the transport as the request's abort signal. */
|
|
120
|
+
readonly signal: AbortSignal;
|
|
121
|
+
/**
|
|
122
|
+
* The stream is open: arm the time-to-first-token budget. Call immediately
|
|
123
|
+
* after the transport returns the stream — arming before that would count
|
|
124
|
+
* connection time against the first-token budget. The absolute cap is already
|
|
125
|
+
* running (see {@link createStreamWatchdog}); this only starts the idle
|
|
126
|
+
* budgets. Idempotent, and a no-op after {@link dispose}.
|
|
127
|
+
*/
|
|
128
|
+
open(): void;
|
|
129
|
+
/**
|
|
130
|
+
* A chunk arrived: re-arm for the gap to the *next* one. Call for every
|
|
131
|
+
* chunk, including metadata-only ones — a chunk that carried nothing still
|
|
132
|
+
* proves the stream is alive.
|
|
133
|
+
*/
|
|
134
|
+
observedChunk(output: ChunkOutput): void;
|
|
135
|
+
/**
|
|
136
|
+
* Why the stream was torn down, or `null` if this watchdog did not do it.
|
|
137
|
+
* Returns `null` whenever the caller's own signal aborted, even if a budget
|
|
138
|
+
* also elapsed: a cancelled call is cancelled, not stalled.
|
|
139
|
+
*
|
|
140
|
+
* Call from the `catch` that saw the abort, before {@link dispose}.
|
|
141
|
+
*/
|
|
142
|
+
stall(): StreamStall | null;
|
|
143
|
+
/**
|
|
144
|
+
* Clear the timers, forget any budget that elapsed, and settle the composite
|
|
145
|
+
* signal. Idempotent; call from a `finally` on every path.
|
|
146
|
+
*
|
|
147
|
+
* Settling matters as much as clearing: the composite holds a listener on a
|
|
148
|
+
* possibly long-lived caller signal, and one per call across a run's many
|
|
149
|
+
* calls is a leak.
|
|
150
|
+
*
|
|
151
|
+
* Forgetting matters because a timer can fire in the moment between the last
|
|
152
|
+
* chunk and the stream ending — the abort loses the race, the read completes
|
|
153
|
+
* normally, and nothing was torn down. If that stale flag survived, a *later*
|
|
154
|
+
* failure on the same call (a defect found while assembling the message, a
|
|
155
|
+
* billing read) would reach {@link stall} and be reported as an upstream
|
|
156
|
+
* stall it had nothing to do with. Call `stall()` before `dispose()`, which
|
|
157
|
+
* is the documented order and the only order in which a real stall is
|
|
158
|
+
* observable anyway.
|
|
159
|
+
*/
|
|
160
|
+
dispose(): void;
|
|
161
|
+
}
|
|
162
|
+
export declare const DEFAULT_TIME_TO_FIRST_TOKEN_MS = 120000;
|
|
163
|
+
export declare const DEFAULT_INTER_CHUNK_MS = 5000;
|
|
164
|
+
export declare const DEFAULT_MAX_CALL_DURATION_MS = 600000;
|
|
165
|
+
export declare function createStreamWatchdog(options?: StreamWatchdogOptions): StreamWatchdog;
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming-completion idle watchdog.
|
|
3
|
+
*
|
|
4
|
+
* A chat-completions SDK's `timeout` bounds *establishing* the request and (for
|
|
5
|
+
* a non-streaming call) the whole response — but once a stream has started
|
|
6
|
+
* yielding, nothing bounds the gap between chunks. A provider that opens the
|
|
7
|
+
* stream and then stalls leaves the consumer's `for await` awaiting the next
|
|
8
|
+
* chunk forever, holding its worker slot until some outer deadline fires, or
|
|
9
|
+
* never. Downstream that is the agent that says "thinking…" and never answers.
|
|
10
|
+
*
|
|
11
|
+
* The budget is split into three because the phases of a streaming completion
|
|
12
|
+
* have very different normal latencies, and one number cannot serve all of them:
|
|
13
|
+
*
|
|
14
|
+
* - **Time to first token** — from opening the stream to the first *answer*
|
|
15
|
+
* token. Generous: a high-reasoning model can legitimately think for a while
|
|
16
|
+
* before emitting anything, and even once reasoning deltas are flowing it can
|
|
17
|
+
* sit silent for seconds between finishing its reasoning and emitting the
|
|
18
|
+
* first content/tool token. This budget therefore covers the reasoning phase
|
|
19
|
+
* AND the reasoning→answer gap — {@link ChunkOutput} `"reasoning"` does NOT
|
|
20
|
+
* switch to the tight budget. Arming the tight budget on a reasoning token is
|
|
21
|
+
* a real observed failure: the normal reasoning→content pause of a frontier
|
|
22
|
+
* reasoning model routinely approaches several seconds and tripped
|
|
23
|
+
* "no chunk mid-stream" on healthy turns.
|
|
24
|
+
* - **Inter-chunk** — the gap between chunks once the model's ANSWER output
|
|
25
|
+
* (content / tool-call arguments / refusal) is flowing. Tight: answer chunks
|
|
26
|
+
* normally arrive sub-second, so a multi-second silence mid-answer is already
|
|
27
|
+
* abnormal, and a shorter budget recovers a mid-stream stall far sooner.
|
|
28
|
+
* - **Overall cap** — an absolute wall-clock ceiling, armed at construction and
|
|
29
|
+
* never re-armed. Defense in depth, not a duplicate of the two idle budgets:
|
|
30
|
+
* the idle watchdog only bounds the *gap* between chunks, so a provider that
|
|
31
|
+
* drip-feeds a chunk just under the idle budget forever (a reasoning delta a
|
|
32
|
+
* minute, never reaching an answer) never trips it. The proper bound on total
|
|
33
|
+
* work is the caller's own deadline signal, but not every caller has one, so
|
|
34
|
+
* this guarantees a single call cannot hold a slot indefinitely regardless.
|
|
35
|
+
*
|
|
36
|
+
* It is armed by the constructor rather than by {@link StreamWatchdog.open}
|
|
37
|
+
* because it is the guarantee of last resort: a host that forgets `open()`
|
|
38
|
+
* gets a watchdog that silently never fires, with no type error and no test
|
|
39
|
+
* failure, and the symptom appears only under a stalling provider. Counting
|
|
40
|
+
* the connection handshake against a ceiling this generous costs nothing;
|
|
41
|
+
* leaving the ceiling to an imperative call the host may skip costs a worker
|
|
42
|
+
* slot. The two *idle* budgets genuinely must wait for `open()` — arming them
|
|
43
|
+
* earlier would charge connection time to the first-token budget.
|
|
44
|
+
*
|
|
45
|
+
* The watchdog owns timers and abort signals and nothing else: it does not read
|
|
46
|
+
* the stream, does not know the wire format, and does not phrase the error. It
|
|
47
|
+
* reports a structured {@link StreamStall} and the host words it — the same
|
|
48
|
+
* split the routing modules use, since a stall message is usually product copy.
|
|
49
|
+
*/
|
|
50
|
+
export const DEFAULT_TIME_TO_FIRST_TOKEN_MS = 120_000;
|
|
51
|
+
export const DEFAULT_INTER_CHUNK_MS = 5_000;
|
|
52
|
+
export const DEFAULT_MAX_CALL_DURATION_MS = 600_000;
|
|
53
|
+
/**
|
|
54
|
+
* The largest delay `setTimeout` can represent. Above this the delay overflows
|
|
55
|
+
* its 32-bit signed field and the timer fires almost immediately instead.
|
|
56
|
+
*/
|
|
57
|
+
const MAX_TIMEOUT_MS = 2_147_483_647;
|
|
58
|
+
/**
|
|
59
|
+
* A budget must be a delay `setTimeout` can actually honour.
|
|
60
|
+
*
|
|
61
|
+
* This is checked at construction, and it fails closed in the direction that
|
|
62
|
+
* actually bites. Every value rejected here would otherwise make the watchdog
|
|
63
|
+
* fire on the next tick and tear down every *healthy* stream, surfacing as a
|
|
64
|
+
* total inference outage that looks like a provider incident:
|
|
65
|
+
*
|
|
66
|
+
* - `NaN` / `Infinity`: `setTimeout` coerces a non-finite delay to 1ms rather
|
|
67
|
+
* than ignoring it, so a bad budget does not disable the watchdog — it makes
|
|
68
|
+
* it instantaneous.
|
|
69
|
+
* - Zero or negative: fires on the next tick by definition.
|
|
70
|
+
* - Above {@link MAX_TIMEOUT_MS}: overflows the 32-bit delay and, per spec,
|
|
71
|
+
* clamps to 1ms. This is the trap that reads as reasonable — `maxCallDuration`
|
|
72
|
+
* of 30 days is a plausible config value and the units are milliseconds, so
|
|
73
|
+
* it is one `* 1000` away from a healthy-looking number that disables the
|
|
74
|
+
* backstop entirely.
|
|
75
|
+
*
|
|
76
|
+
* Refusing the value at construction turns a silent outage into a stack trace
|
|
77
|
+
* at the call site that supplied it.
|
|
78
|
+
*/
|
|
79
|
+
function requirePositiveMs(value, field) {
|
|
80
|
+
if (!Number.isFinite(value) || value <= 0 || value > MAX_TIMEOUT_MS) {
|
|
81
|
+
throw new RangeError(`StreamWatchdog ${field} must be a positive number of ms no greater than ${MAX_TIMEOUT_MS}, got ${String(value)}`);
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Call `unref()` on a timer when the runtime exposes it (Node/Bun) so a
|
|
87
|
+
* forgotten `dispose()` cannot hold the event loop open; a no-op under a DOM
|
|
88
|
+
* `number` timer or an injected fake. Feature-detected rather than `as`-cast to
|
|
89
|
+
* keep type safety.
|
|
90
|
+
*
|
|
91
|
+
* Duplicated from `../run/harness` deliberately: importing across module
|
|
92
|
+
* folders would make `@juno-ai/bind/completion` drag in the run mechanics for
|
|
93
|
+
* five lines, and a consumer that wants only the watchdog should get only the
|
|
94
|
+
* watchdog.
|
|
95
|
+
*/
|
|
96
|
+
function unrefTimer(timer) {
|
|
97
|
+
if (typeof timer === "object" &&
|
|
98
|
+
timer !== null &&
|
|
99
|
+
"unref" in timer &&
|
|
100
|
+
typeof timer.unref === "function") {
|
|
101
|
+
timer.unref();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
const AMBIENT_TIMERS = {
|
|
105
|
+
set: (callback, ms) => setTimeout(callback, ms),
|
|
106
|
+
clear: (handle) => {
|
|
107
|
+
clearTimeout(handle);
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
export function createStreamWatchdog(options = {}) {
|
|
111
|
+
const timeToFirstTokenMs = requirePositiveMs(options.timeToFirstTokenMs ?? DEFAULT_TIME_TO_FIRST_TOKEN_MS, "timeToFirstTokenMs");
|
|
112
|
+
const interChunkMs = requirePositiveMs(options.interChunkMs ?? DEFAULT_INTER_CHUNK_MS, "interChunkMs");
|
|
113
|
+
const maxCallDurationMs = requirePositiveMs(options.maxCallDurationMs ?? DEFAULT_MAX_CALL_DURATION_MS, "maxCallDurationMs");
|
|
114
|
+
const external = options.external ?? null;
|
|
115
|
+
const timers = options.timers ?? AMBIENT_TIMERS;
|
|
116
|
+
// Two controllers rather than one, so `stall()` can name which bound tripped
|
|
117
|
+
// without the caller having to correlate timers.
|
|
118
|
+
const idleController = new AbortController();
|
|
119
|
+
const overallController = new AbortController();
|
|
120
|
+
let idleFired = false;
|
|
121
|
+
let overallFired = false;
|
|
122
|
+
let disposed = false;
|
|
123
|
+
// True once any chunk carried model output; only phrases the diagnostic.
|
|
124
|
+
let sawOutput = false;
|
|
125
|
+
// True once ANSWER output flowed. Only this arms the tight budget.
|
|
126
|
+
let sawAnswerOutput = false;
|
|
127
|
+
let idleTimer = null;
|
|
128
|
+
let overallTimer = null;
|
|
129
|
+
function armIdle(ms) {
|
|
130
|
+
if (idleTimer !== null)
|
|
131
|
+
timers.clear(idleTimer);
|
|
132
|
+
idleTimer = timers.set(() => {
|
|
133
|
+
idleFired = true;
|
|
134
|
+
idleController.abort();
|
|
135
|
+
}, ms);
|
|
136
|
+
unrefTimer(idleTimer);
|
|
137
|
+
}
|
|
138
|
+
function clearTimers() {
|
|
139
|
+
if (idleTimer !== null) {
|
|
140
|
+
timers.clear(idleTimer);
|
|
141
|
+
idleTimer = null;
|
|
142
|
+
}
|
|
143
|
+
if (overallTimer !== null) {
|
|
144
|
+
timers.clear(overallTimer);
|
|
145
|
+
overallTimer = null;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const signal = AbortSignal.any(external
|
|
149
|
+
? [external, idleController.signal, overallController.signal]
|
|
150
|
+
: [idleController.signal, overallController.signal]);
|
|
151
|
+
// Armed here, not in `open()`, so the ceiling holds even for a host that
|
|
152
|
+
// never calls `open()`. See the module header.
|
|
153
|
+
overallTimer = timers.set(() => {
|
|
154
|
+
overallFired = true;
|
|
155
|
+
overallController.abort();
|
|
156
|
+
}, maxCallDurationMs);
|
|
157
|
+
unrefTimer(overallTimer);
|
|
158
|
+
return {
|
|
159
|
+
signal,
|
|
160
|
+
open() {
|
|
161
|
+
if (disposed)
|
|
162
|
+
return;
|
|
163
|
+
armIdle(timeToFirstTokenMs);
|
|
164
|
+
},
|
|
165
|
+
observedChunk(output) {
|
|
166
|
+
if (disposed)
|
|
167
|
+
return;
|
|
168
|
+
if (output !== "none")
|
|
169
|
+
sawOutput = true;
|
|
170
|
+
if (output === "answer")
|
|
171
|
+
sawAnswerOutput = true;
|
|
172
|
+
armIdle(sawAnswerOutput ? interChunkMs : timeToFirstTokenMs);
|
|
173
|
+
},
|
|
174
|
+
stall() {
|
|
175
|
+
// A caller abort wins over every budget: the stream may well have been
|
|
176
|
+
// idle when cancellation landed, but reporting that as an upstream stall
|
|
177
|
+
// would make a deliberate cancellation look retriable.
|
|
178
|
+
if (external?.aborted)
|
|
179
|
+
return null;
|
|
180
|
+
// Checked before the idle budget because the cap is the stronger claim:
|
|
181
|
+
// when both elapsed, the call ran past its absolute ceiling, and the
|
|
182
|
+
// trailing idle gap is a symptom of that rather than a separate fault.
|
|
183
|
+
if (overallFired) {
|
|
184
|
+
return { kind: "overall_cap", limitMs: maxCallDurationMs };
|
|
185
|
+
}
|
|
186
|
+
if (idleFired) {
|
|
187
|
+
if (sawAnswerOutput)
|
|
188
|
+
return { kind: "inter_chunk", limitMs: interChunkMs };
|
|
189
|
+
return {
|
|
190
|
+
kind: "time_to_first_token",
|
|
191
|
+
limitMs: timeToFirstTokenMs,
|
|
192
|
+
sawReasoning: sawOutput,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return null;
|
|
196
|
+
},
|
|
197
|
+
dispose() {
|
|
198
|
+
disposed = true;
|
|
199
|
+
clearTimers();
|
|
200
|
+
// Drop any budget that elapsed. A timer that fired in the moment before
|
|
201
|
+
// the stream ended lost the race — it tore nothing down — and leaving the
|
|
202
|
+
// flag set would let it be blamed for a later, unrelated failure on the
|
|
203
|
+
// same call. See the `dispose` doc comment.
|
|
204
|
+
idleFired = false;
|
|
205
|
+
overallFired = false;
|
|
206
|
+
// Abort the idle controller (not the overall one) purely to settle the
|
|
207
|
+
// composite so it releases its listener on `external`.
|
|
208
|
+
idleController.abort();
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
package/contracts/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { emptyRunStats, accumulateTurn, accumulateToolCall, type TranscriptMessage, type AssistantTurnMessage, type WireToolDefinition, type WireToolCall, type TurnTimings, type TurnUsage, type ModelTurnResult, type TurnFn, type StopReason, type RunStats, } from "./turn.js";
|
|
1
|
+
export { emptyRunStats, accumulateTurn, accumulateToolCall, accumulateRun, type TranscriptMessage, type AssistantTurnMessage, type WireToolDefinition, type WireToolCall, type TurnTimings, type TurnUsage, type ModelTurnResult, type TurnFn, type StopReason, type RunStats, } from "./turn.js";
|
package/contracts/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { emptyRunStats, accumulateTurn, accumulateToolCall, } from "./turn.js";
|
|
1
|
+
export { emptyRunStats, accumulateTurn, accumulateToolCall, accumulateRun, } from "./turn.js";
|
package/contracts/turn.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type OpenAI from "openai";
|
|
2
2
|
/**
|
|
3
|
-
* Turn vocabulary — the shared language between the turn kernel
|
|
4
|
-
*
|
|
3
|
+
* Turn vocabulary — the shared language between the turn kernel
|
|
4
|
+
* (`@juno-ai/bind/loop`), LLM transports, and hosts.
|
|
5
5
|
*
|
|
6
6
|
* The declared wire format is the OpenAI chat-completions message shape,
|
|
7
7
|
* consumed as **types only** (`openai` is a peer used purely for its type
|
|
@@ -77,3 +77,27 @@ export declare function emptyRunStats(): RunStats;
|
|
|
77
77
|
export declare function accumulateTurn(stats: RunStats, turn: ModelTurnResult): RunStats;
|
|
78
78
|
/** Fold one dispatched tool call's duration into cumulative run stats. */
|
|
79
79
|
export declare function accumulateToolCall(stats: RunStats, toolName: string, durationMs: number): RunStats;
|
|
80
|
+
/**
|
|
81
|
+
* Fold a completed run's totals into another run's — the roll-up for a chain
|
|
82
|
+
* that spawned child runs (`@juno-ai/bind/run`).
|
|
83
|
+
*
|
|
84
|
+
* Two things follow from summing across runs rather than within one, and both
|
|
85
|
+
* are correct rather than artifacts:
|
|
86
|
+
*
|
|
87
|
+
* - **`modelTimeMs` can exceed the chain's wall-clock**, because children that
|
|
88
|
+
* ran concurrently each contribute their own. That is precisely why model
|
|
89
|
+
* time and wall-clock are separate numbers; a chain's *cost* is the sum, its
|
|
90
|
+
* *latency* is not.
|
|
91
|
+
* - **`outputTokensPerSecond` is recomputed from the merged totals**, not
|
|
92
|
+
* averaged from the parts. An average of two rates weights a 10-token run
|
|
93
|
+
* the same as a 10,000-token one and reports a throughput neither run
|
|
94
|
+
* achieved.
|
|
95
|
+
*
|
|
96
|
+
* The fold is associative and order-independent, so a chain reduces cleanly in
|
|
97
|
+
* whatever order its children finish:
|
|
98
|
+
*
|
|
99
|
+
* ```ts
|
|
100
|
+
* const chainTotals = childStats.reduce(accumulateRun, parentStats);
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
export declare function accumulateRun(stats: RunStats, run: RunStats): RunStats;
|
package/contracts/turn.js
CHANGED
|
@@ -37,3 +37,48 @@ export function accumulateToolCall(stats, toolName, durationMs) {
|
|
|
37
37
|
},
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* Fold a completed run's totals into another run's — the roll-up for a chain
|
|
42
|
+
* that spawned child runs (`@juno-ai/bind/run`).
|
|
43
|
+
*
|
|
44
|
+
* Two things follow from summing across runs rather than within one, and both
|
|
45
|
+
* are correct rather than artifacts:
|
|
46
|
+
*
|
|
47
|
+
* - **`modelTimeMs` can exceed the chain's wall-clock**, because children that
|
|
48
|
+
* ran concurrently each contribute their own. That is precisely why model
|
|
49
|
+
* time and wall-clock are separate numbers; a chain's *cost* is the sum, its
|
|
50
|
+
* *latency* is not.
|
|
51
|
+
* - **`outputTokensPerSecond` is recomputed from the merged totals**, not
|
|
52
|
+
* averaged from the parts. An average of two rates weights a 10-token run
|
|
53
|
+
* the same as a 10,000-token one and reports a throughput neither run
|
|
54
|
+
* achieved.
|
|
55
|
+
*
|
|
56
|
+
* The fold is associative and order-independent, so a chain reduces cleanly in
|
|
57
|
+
* whatever order its children finish:
|
|
58
|
+
*
|
|
59
|
+
* ```ts
|
|
60
|
+
* const chainTotals = childStats.reduce(accumulateRun, parentStats);
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
export function accumulateRun(stats, run) {
|
|
64
|
+
const outputTokens = stats.outputTokens + run.outputTokens;
|
|
65
|
+
const modelTimeMs = stats.modelTimeMs + run.modelTimeMs;
|
|
66
|
+
const toolTimeBreakdownMs = {
|
|
67
|
+
...stats.toolTimeBreakdownMs,
|
|
68
|
+
};
|
|
69
|
+
for (const [toolName, durationMs] of Object.entries(run.toolTimeBreakdownMs)) {
|
|
70
|
+
toolTimeBreakdownMs[toolName] =
|
|
71
|
+
(toolTimeBreakdownMs[toolName] ?? 0) + durationMs;
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
turns: stats.turns + run.turns,
|
|
75
|
+
toolCalls: stats.toolCalls + run.toolCalls,
|
|
76
|
+
inputTokens: stats.inputTokens + run.inputTokens,
|
|
77
|
+
outputTokens,
|
|
78
|
+
costCents: stats.costCents + run.costCents,
|
|
79
|
+
modelTimeMs,
|
|
80
|
+
toolTimeMs: stats.toolTimeMs + run.toolTimeMs,
|
|
81
|
+
outputTokensPerSecond: modelTimeMs > 0 ? (outputTokens / modelTimeMs) * 1000 : null,
|
|
82
|
+
toolTimeBreakdownMs,
|
|
83
|
+
};
|
|
84
|
+
}
|
package/index.d.ts
CHANGED
|
@@ -5,17 +5,23 @@
|
|
|
5
5
|
* completion into tool effects into the next turn's context. This package is
|
|
6
6
|
* the harness that runs that chain.
|
|
7
7
|
*
|
|
8
|
-
* Current surface: the
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* and
|
|
13
|
-
*
|
|
14
|
-
*
|
|
8
|
+
* Current surface: the tool-calling turn kernel (`src/loop/` — the iteration
|
|
9
|
+
* engine itself), the deterministic LLM provider-routing core with its
|
|
10
|
+
* transport-error classifier, the turn vocabulary, the run mechanics (deadline,
|
|
11
|
+
* coalesced heartbeat, failure classification, tool-batch pooling, child-run
|
|
12
|
+
* lineage and admission), the streaming-completion watchdog and completion
|
|
13
|
+
* defect detection (`src/completion/`), transcript validation/healing, provider
|
|
14
|
+
* tool-schema sanitization, and the plugin/tool vocabulary with its registry and
|
|
15
|
+
* progressive-disclosure activation — generic over the host's invocation
|
|
16
|
+
* context. What is NOT here is
|
|
17
|
+
* the run driver: starting a run, recording what it did, and delivering its
|
|
18
|
+
* output. See the README for the rest of what is deliberately absent.
|
|
15
19
|
*/
|
|
16
20
|
export * from "./routing/index.js";
|
|
21
|
+
export * from "./completion/index.js";
|
|
17
22
|
export * from "./contracts/index.js";
|
|
18
23
|
export * from "./run/index.js";
|
|
19
24
|
export * from "./transcript/index.js";
|
|
20
25
|
export * from "./tools/index.js";
|
|
21
26
|
export * from "./plugins/index.js";
|
|
27
|
+
export * from "./loop/index.js";
|