@meridian-flow/runtime 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +67 -2
- package/dist/gateway/adapters/anthropic/adapter.d.ts +9 -0
- package/dist/gateway/adapters/anthropic/adapter.js +52 -0
- package/dist/gateway/adapters/anthropic/request-map.d.ts +46 -0
- package/dist/gateway/adapters/anthropic/request-map.js +285 -0
- package/dist/gateway/adapters/anthropic/stream-collect.d.ts +67 -0
- package/dist/gateway/adapters/anthropic/stream-collect.js +295 -0
- package/dist/gateway/adapters/anthropic/tool-schema.d.ts +8 -0
- package/dist/gateway/adapters/anthropic/tool-schema.js +68 -0
- package/dist/gateway/adapters/credentials.d.ts +25 -0
- package/dist/gateway/adapters/credentials.js +118 -0
- package/dist/gateway/adapters/gemini/adapter.d.ts +10 -0
- package/dist/gateway/adapters/gemini/adapter.js +90 -0
- package/dist/gateway/adapters/gemini/request-map.d.ts +29 -0
- package/dist/gateway/adapters/gemini/request-map.js +178 -0
- package/dist/gateway/adapters/gemini/stream-collect.d.ts +29 -0
- package/dist/gateway/adapters/gemini/stream-collect.js +116 -0
- package/dist/gateway/adapters/mock/response.d.ts +11 -0
- package/dist/gateway/adapters/mock/response.js +160 -0
- package/dist/gateway/adapters/mock/samples.d.ts +8 -0
- package/dist/gateway/adapters/mock/samples.js +133 -0
- package/dist/gateway/adapters/mock/script-queue.d.ts +10 -0
- package/dist/gateway/adapters/mock/script-queue.js +60 -0
- package/dist/gateway/adapters/mock/server.d.ts +16 -0
- package/dist/gateway/adapters/mock/server.js +342 -0
- package/dist/gateway/adapters/mock/write-directive.d.ts +15 -0
- package/dist/gateway/adapters/mock/write-directive.js +21 -0
- package/dist/gateway/adapters/openai/adapter.d.ts +3 -0
- package/dist/gateway/adapters/openai/adapter.js +35 -0
- package/dist/gateway/adapters/openai/request-map.d.ts +31 -0
- package/dist/gateway/adapters/openai/request-map.js +228 -0
- package/dist/gateway/adapters/openai/stream-collect.d.ts +63 -0
- package/dist/gateway/adapters/openai/stream-collect.js +490 -0
- package/dist/gateway/adapters/openai-compatible/adapter.d.ts +3 -0
- package/dist/gateway/adapters/openai-compatible/adapter.js +35 -0
- package/dist/gateway/adapters/openai-compatible/request-map.d.ts +31 -0
- package/dist/gateway/adapters/openai-compatible/request-map.js +180 -0
- package/dist/gateway/adapters/openai-compatible/stream-collect.d.ts +84 -0
- package/dist/gateway/adapters/openai-compatible/stream-collect.js +215 -0
- package/dist/gateway/adapters/provider-http-error.d.ts +65 -0
- package/dist/gateway/adapters/provider-http-error.js +188 -0
- package/dist/gateway/adapters/provider-stream.d.ts +23 -0
- package/dist/gateway/adapters/provider-stream.js +58 -0
- package/dist/gateway/attempt-stream.d.ts +27 -0
- package/dist/gateway/attempt-stream.js +329 -0
- package/dist/gateway/create-gateway.d.ts +25 -0
- package/dist/gateway/create-gateway.js +88 -0
- package/dist/gateway/deadline.d.ts +56 -0
- package/dist/gateway/deadline.js +120 -0
- package/dist/gateway/domain/index.d.ts +10 -0
- package/dist/gateway/domain/index.js +10 -0
- package/dist/gateway/domain/media.d.ts +62 -0
- package/dist/gateway/domain/media.js +126 -0
- package/dist/gateway/domain/metering.d.ts +7 -0
- package/dist/gateway/domain/metering.js +6 -0
- package/dist/gateway/domain/resolve-call.d.ts +24 -0
- package/dist/gateway/domain/resolve-call.js +39 -0
- package/dist/gateway/domain/types.d.ts +457 -0
- package/dist/gateway/domain/types.js +10 -0
- package/dist/gateway/helpers/messages.d.ts +17 -0
- package/dist/gateway/helpers/messages.js +29 -0
- package/dist/gateway/helpers/parse-tool-arguments.d.ts +9 -0
- package/dist/gateway/helpers/parse-tool-arguments.js +38 -0
- package/dist/gateway/helpers/tool-call-pairing.d.ts +13 -0
- package/dist/gateway/helpers/tool-call-pairing.js +25 -0
- package/dist/gateway/index.d.ts +12 -0
- package/dist/gateway/index.js +11 -0
- package/dist/gateway/mock-script.d.ts +129 -0
- package/dist/gateway/mock-script.js +129 -0
- package/dist/gateway/mock.d.ts +11 -0
- package/dist/gateway/mock.js +22 -0
- package/dist/gateway/ports/gateway.d.ts +18 -0
- package/dist/gateway/ports/gateway.js +1 -0
- package/dist/gateway/ports/provider-adapter.d.ts +21 -0
- package/dist/gateway/ports/provider-adapter.js +1 -0
- package/dist/gateway/routing.d.ts +28 -0
- package/dist/gateway/routing.js +48 -0
- package/dist/gateway/shapes/index.d.ts +7 -0
- package/dist/gateway/shapes/index.js +1 -0
- package/dist/gateway/shapes/json.d.ts +4 -0
- package/dist/gateway/shapes/json.js +1 -0
- package/dist/gateway/shapes/provider-error.d.ts +10 -0
- package/dist/gateway/shapes/provider-error.js +1 -0
- package/dist/gateway/shapes/usage.d.ts +41 -0
- package/dist/gateway/shapes/usage.js +29 -0
- package/dist/gateway/stream-events.d.ts +21 -0
- package/dist/gateway/stream-events.js +26 -0
- package/package.json +62 -3
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/** Every provider's context-window and content-filter wording; an adapter overrides only what differs. */
|
|
2
|
+
const DEFAULT_PATTERNS = {
|
|
3
|
+
contextOverflow: /context[_ ](?:length|window)|exceeds? context limit|prompt is too long|maximum context length|too many (?:input )?tokens|input.*exceeds.*token/,
|
|
4
|
+
contentFiltered: /content[\s\S]*filter|filter[\s\S]*content/,
|
|
5
|
+
};
|
|
6
|
+
function record(value) {
|
|
7
|
+
return typeof value === "object" && value !== null
|
|
8
|
+
? value
|
|
9
|
+
: undefined;
|
|
10
|
+
}
|
|
11
|
+
function header(headers, name) {
|
|
12
|
+
if (headers instanceof Headers)
|
|
13
|
+
return headers.get(name) ?? undefined;
|
|
14
|
+
const values = record(headers);
|
|
15
|
+
if (!values)
|
|
16
|
+
return undefined;
|
|
17
|
+
const key = Object.keys(values).find((candidate) => candidate.toLowerCase() === name);
|
|
18
|
+
const value = key ? values[key] : undefined;
|
|
19
|
+
return typeof value === "string" || typeof value === "number" ? String(value) : undefined;
|
|
20
|
+
}
|
|
21
|
+
function retryAfterMs(headers, now) {
|
|
22
|
+
const milliseconds = header(headers, "retry-after-ms");
|
|
23
|
+
if (milliseconds !== undefined) {
|
|
24
|
+
const value = Number(milliseconds);
|
|
25
|
+
if (Number.isFinite(value) && value >= 0)
|
|
26
|
+
return Math.round(value);
|
|
27
|
+
}
|
|
28
|
+
const retryAfter = header(headers, "retry-after");
|
|
29
|
+
if (retryAfter === undefined)
|
|
30
|
+
return undefined;
|
|
31
|
+
const seconds = Number(retryAfter);
|
|
32
|
+
if (Number.isFinite(seconds))
|
|
33
|
+
return seconds >= 0 ? Math.round(seconds * 1_000) : undefined;
|
|
34
|
+
const date = Date.parse(retryAfter);
|
|
35
|
+
return Number.isFinite(date) ? Math.max(0, date - now) : undefined;
|
|
36
|
+
}
|
|
37
|
+
function responseHeaders(err) {
|
|
38
|
+
const errorRecord = record(err);
|
|
39
|
+
return errorRecord?.headers ?? record(errorRecord?.response)?.headers;
|
|
40
|
+
}
|
|
41
|
+
function httpStatus(err) {
|
|
42
|
+
const status = record(err)?.status;
|
|
43
|
+
return typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599
|
|
44
|
+
? status
|
|
45
|
+
: undefined;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A provider's error can echo a media link it failed to fetch, and a signed link's query string
|
|
49
|
+
* is a credential, so error text never keeps one.
|
|
50
|
+
*/
|
|
51
|
+
export function stripUrlQueries(text) {
|
|
52
|
+
return text.replace(/(https?:\/\/[^\s"'<>?#]*)[?#][^\s"'<>]*/g, "$1");
|
|
53
|
+
}
|
|
54
|
+
function errorMessage(err) {
|
|
55
|
+
if (err instanceof Error)
|
|
56
|
+
return stripUrlQueries(err.message);
|
|
57
|
+
const message = record(err)?.message;
|
|
58
|
+
return stripUrlQueries(message !== undefined ? String(message) : String(err));
|
|
59
|
+
}
|
|
60
|
+
// SDKs parse failure bodies and keep only part (OpenAI keeps `body.error`). The
|
|
61
|
+
// SDK error carries the same Headers object as the Response, so it keys the text.
|
|
62
|
+
const failureBodies = new WeakMap();
|
|
63
|
+
/**
|
|
64
|
+
* `fetch` for provider SDK clients: records each failed response's exact body text, so
|
|
65
|
+
* the provider's message survives where the SDK's parse drops it (OpenAI's reads a body
|
|
66
|
+
* without `error` as "402 status code (no body)").
|
|
67
|
+
*/
|
|
68
|
+
export const providerFetch = async (input, init) => {
|
|
69
|
+
const response = await fetch(input, init);
|
|
70
|
+
if (!response.ok) {
|
|
71
|
+
try {
|
|
72
|
+
failureBodies.set(response.headers, await response.clone().text());
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
// An unreadable body leaves the SDK's parsed view as the evidence.
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return response;
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* Constructor options every adapter's SDK client shares. `maxRetries: 0` because the
|
|
82
|
+
* gateway owns retries (`streamWithRetry`); SDK retries would multiply them.
|
|
83
|
+
*
|
|
84
|
+
* Both SDKs fill every option left `undefined` from process environment variables
|
|
85
|
+
* (`*_BASE_URL`, `*_AUTH_TOKEN`, `*_ORG_ID`, `*_LOG`, ...), which would redirect requests or add
|
|
86
|
+
* credentials the config never named. So each option is passed explicitly: the adapter
|
|
87
|
+
* adds the SDK-specific ones with `null` (see `ANTHROPIC_ENV_OPTOUT`, `OPENAI_ENV_OPTOUT`).
|
|
88
|
+
* The SDKs' `*_CUSTOM_HEADERS` has no opt-out option and still applies.
|
|
89
|
+
*/
|
|
90
|
+
export function providerClientOptions(config, defaultBaseUrl) {
|
|
91
|
+
return {
|
|
92
|
+
baseURL: config.baseUrl ?? defaultBaseUrl,
|
|
93
|
+
defaultHeaders: config.headers,
|
|
94
|
+
logLevel: "warn",
|
|
95
|
+
maxRetries: 0,
|
|
96
|
+
fetch: providerFetch,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/** Anthropic client options that default from the environment, set to "none". */
|
|
100
|
+
export const ANTHROPIC_ENV_OPTOUT = { authToken: null, webhookKey: null };
|
|
101
|
+
/** OpenAI client options that default from the environment, set to "none". */
|
|
102
|
+
export const OPENAI_ENV_OPTOUT = {
|
|
103
|
+
adminAPIKey: null,
|
|
104
|
+
organization: null,
|
|
105
|
+
project: null,
|
|
106
|
+
webhookSecret: null,
|
|
107
|
+
};
|
|
108
|
+
function exactFailureBody(err) {
|
|
109
|
+
const headers = record(err)?.headers;
|
|
110
|
+
return headers instanceof Headers ? failureBodies.get(headers) : undefined;
|
|
111
|
+
}
|
|
112
|
+
function bodyMessage(body) {
|
|
113
|
+
const fields = record(body);
|
|
114
|
+
if (typeof fields?.message === "string")
|
|
115
|
+
return fields.message;
|
|
116
|
+
const nested = record(fields?.error)?.message;
|
|
117
|
+
return typeof nested === "string" ? nested : undefined;
|
|
118
|
+
}
|
|
119
|
+
function parseJson(text) {
|
|
120
|
+
try {
|
|
121
|
+
return JSON.parse(text);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The provider's status and message, read from the exact body when the client used
|
|
129
|
+
* `providerFetch`, otherwise from the SDK's parsed view (OpenAI-shaped SDKs keep only
|
|
130
|
+
* the body's `error`). Absent when the SDK never received a response.
|
|
131
|
+
*/
|
|
132
|
+
function providerErrorOf(err) {
|
|
133
|
+
const status = httpStatus(err);
|
|
134
|
+
const exact = exactFailureBody(err);
|
|
135
|
+
const parsed = exact === undefined ? record(err)?.error : parseJson(exact);
|
|
136
|
+
if (status === undefined && parsed === undefined && exact === undefined)
|
|
137
|
+
return undefined;
|
|
138
|
+
const message = bodyMessage(parsed) ?? (exact?.trim() || errorMessage(err));
|
|
139
|
+
return { status: status ?? null, message: stripUrlQueries(message) };
|
|
140
|
+
}
|
|
141
|
+
export function mapProviderHttpError(err, overrides = {}) {
|
|
142
|
+
const patterns = { ...DEFAULT_PATTERNS, ...overrides };
|
|
143
|
+
// A token provider that fails inside the SDK's request comes back wrapped (`APIConnectionError`,
|
|
144
|
+
// `OpenAIError`) with the provider's own error as the cause: it is classified by its status.
|
|
145
|
+
const cause = record(err)?.cause;
|
|
146
|
+
const failure = httpStatus(err) === undefined && httpStatus(cause) !== undefined ? cause : err;
|
|
147
|
+
const status = httpStatus(failure);
|
|
148
|
+
const message = errorMessage(failure);
|
|
149
|
+
const mapped = (() => {
|
|
150
|
+
if (status === 401 || status === 403) {
|
|
151
|
+
return { code: "auth_error", message, retryable: false };
|
|
152
|
+
}
|
|
153
|
+
if (status === 429)
|
|
154
|
+
return { code: "rate_limited", message, retryable: true };
|
|
155
|
+
if (status === 408)
|
|
156
|
+
return { code: "network_error", message, retryable: true };
|
|
157
|
+
if (status === 400) {
|
|
158
|
+
const lower = message.toLowerCase();
|
|
159
|
+
if (patterns.contextOverflow.test(lower)) {
|
|
160
|
+
return { code: "context_overflow", message, retryable: false };
|
|
161
|
+
}
|
|
162
|
+
if (patterns.contentFiltered.test(lower)) {
|
|
163
|
+
return { code: "content_filtered", message, retryable: false };
|
|
164
|
+
}
|
|
165
|
+
return { code: "invalid_request", message, retryable: false };
|
|
166
|
+
}
|
|
167
|
+
if (status !== undefined && status >= 500) {
|
|
168
|
+
return { code: "server_error", message, retryable: true };
|
|
169
|
+
}
|
|
170
|
+
if (status !== undefined && status >= 400) {
|
|
171
|
+
return { code: "provider_error", message, retryable: false };
|
|
172
|
+
}
|
|
173
|
+
if (err instanceof TypeError || message.includes("fetch")) {
|
|
174
|
+
return { code: "network_error", message, retryable: true };
|
|
175
|
+
}
|
|
176
|
+
return { code: "provider_error", message, retryable: true };
|
|
177
|
+
})();
|
|
178
|
+
const providerError = providerErrorOf(err);
|
|
179
|
+
const headers = responseHeaders(err);
|
|
180
|
+
const retryAfter = retryAfterMs(headers, Date.now());
|
|
181
|
+
const shouldRetry = header(headers, "x-should-retry")?.trim().toLowerCase();
|
|
182
|
+
return {
|
|
183
|
+
...mapped,
|
|
184
|
+
...(retryAfter === undefined ? {} : { retryAfterMs: retryAfter }),
|
|
185
|
+
...(shouldRetry === "false" ? { retryable: false } : {}),
|
|
186
|
+
...(providerError ? { providerError } : {}),
|
|
187
|
+
};
|
|
188
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The streaming envelope every provider adapter shares: `start`, open the SDK stream,
|
|
3
|
+
* map each frame to canonical events, then `end`. Adapters pass closures over their own
|
|
4
|
+
* accumulator, so this envelope knows nothing of provider wire shapes.
|
|
5
|
+
*
|
|
6
|
+
* An abort ends the stream at once with an `end` built from what accumulated, possibly
|
|
7
|
+
* nothing, even when the SDK ignores the abort. That `end` is not success: `streamWithRetry`
|
|
8
|
+
* owns what a stop or timeout means.
|
|
9
|
+
*/
|
|
10
|
+
import type { GenerateResult, StreamEvent } from "../domain/index.js";
|
|
11
|
+
import { type ProviderErrorPatterns } from "./provider-http-error.js";
|
|
12
|
+
export declare function streamProvider<Frame>(spec: {
|
|
13
|
+
start: Extract<StreamEvent, {
|
|
14
|
+
type: "start";
|
|
15
|
+
}>;
|
|
16
|
+
signal: AbortSignal | undefined;
|
|
17
|
+
errorPatterns?: Partial<ProviderErrorPatterns>;
|
|
18
|
+
/** Reshapes an SDK error whose status, message or headers `mapProviderHttpError` can't read as is. */
|
|
19
|
+
normalizeError?: (err: unknown) => unknown;
|
|
20
|
+
open: () => Promise<AsyncIterable<Frame>>;
|
|
21
|
+
events: (frame: Frame) => Iterable<StreamEvent>;
|
|
22
|
+
result: () => GenerateResult;
|
|
23
|
+
}): AsyncIterable<StreamEvent>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { mapProviderHttpError } from "./provider-http-error.js";
|
|
2
|
+
const ABORTED = Symbol("aborted");
|
|
3
|
+
/** Closes an iterator without waiting on it: a hung SDK iterator must not block the exit. */
|
|
4
|
+
async function release(iterator) {
|
|
5
|
+
// allSettled observes a failed open or close; nothing is waiting on either.
|
|
6
|
+
await Promise.allSettled([iterator.then((it) => it?.return?.())]);
|
|
7
|
+
}
|
|
8
|
+
export async function* streamProvider(spec) {
|
|
9
|
+
yield spec.start;
|
|
10
|
+
// After an abort the SDK owes nothing more: the accumulator already holds all there is, so an
|
|
11
|
+
// abort ends here whether the SDK rejects, returns or hangs. A timeout aborts the same signal;
|
|
12
|
+
// `streamWithRetry` tells the two apart.
|
|
13
|
+
const signal = spec.signal;
|
|
14
|
+
let onAbort;
|
|
15
|
+
const aborted = new Promise((resolve) => {
|
|
16
|
+
if (signal?.aborted)
|
|
17
|
+
return resolve(ABORTED);
|
|
18
|
+
onAbort = () => resolve(ABORTED);
|
|
19
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
20
|
+
});
|
|
21
|
+
let frames;
|
|
22
|
+
try {
|
|
23
|
+
const opening = spec.open();
|
|
24
|
+
const opened = await Promise.race([opening, aborted]);
|
|
25
|
+
if (opened === ABORTED) {
|
|
26
|
+
void release(opening.then((stream) => stream[Symbol.asyncIterator]()));
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
frames = opened[Symbol.asyncIterator]();
|
|
30
|
+
while (true) {
|
|
31
|
+
const next = await Promise.race([frames.next(), aborted]);
|
|
32
|
+
if (next === ABORTED || next.done)
|
|
33
|
+
break;
|
|
34
|
+
for (const event of spec.events(next.value)) {
|
|
35
|
+
yield event;
|
|
36
|
+
if (event.type === "error")
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
if (!signal?.aborted) {
|
|
44
|
+
yield {
|
|
45
|
+
type: "error",
|
|
46
|
+
...mapProviderHttpError(spec.normalizeError?.(err) ?? err, spec.errorPatterns),
|
|
47
|
+
};
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
if (onAbort)
|
|
53
|
+
signal?.removeEventListener("abort", onAbort);
|
|
54
|
+
// Every exit closes the SDK stream, as `for await` did: error frame, consumer drop, abort.
|
|
55
|
+
void release(Promise.resolve(frames));
|
|
56
|
+
}
|
|
57
|
+
yield { type: "end", result: spec.result() };
|
|
58
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-attempt stream driver: runs one provider adapter with retry/backoff and
|
|
3
|
+
* per-attempt timeouts. Isolated from gateway composition (routing,
|
|
4
|
+
* settlement) so the retry/deadline contract has one testable home.
|
|
5
|
+
*
|
|
6
|
+
* Timeout policy:
|
|
7
|
+
* - Stall timeout aborts only when the stream makes no progress; every event
|
|
8
|
+
* re-arms it, so a slow-but-streaming model is never killed.
|
|
9
|
+
* - Absolute ceiling bounds the pathological streaming-forever case.
|
|
10
|
+
*
|
|
11
|
+
* Retry gate: retry only before committed output (visible text or a tool call)
|
|
12
|
+
* has reached the caller. Reasoning-only attempts are retryable, which is what
|
|
13
|
+
* makes `maxAttempts` real for reasoning-first providers.
|
|
14
|
+
*/
|
|
15
|
+
import { type ModelAttemptTimeouts } from "./deadline.js";
|
|
16
|
+
import { type GatewayConfig, type GenerateRequest, type ModelInfo, type StreamEvent } from "./domain/index.js";
|
|
17
|
+
import type { ProviderAdapter } from "./ports/provider-adapter.js";
|
|
18
|
+
/** Gateway-level timeout policy. */
|
|
19
|
+
export interface ModelAttemptTimeoutOverrides {
|
|
20
|
+
attemptStallMs?: number;
|
|
21
|
+
attemptCeilingMs?: number;
|
|
22
|
+
}
|
|
23
|
+
/** The effective timeouts for an attempt: gateway policy, then the process default. */
|
|
24
|
+
export declare function resolveAttemptTimeouts(gateway: ModelAttemptTimeoutOverrides): ModelAttemptTimeouts;
|
|
25
|
+
export declare function streamWithRetry(adapter: ProviderAdapter, request: GenerateRequest, model: ModelInfo, retry: GatewayConfig["retry"], timeouts: ModelAttemptTimeouts, options?: {
|
|
26
|
+
now?: () => number;
|
|
27
|
+
}): AsyncGenerator<StreamEvent>;
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-attempt stream driver: runs one provider adapter with retry/backoff and
|
|
3
|
+
* per-attempt timeouts. Isolated from gateway composition (routing,
|
|
4
|
+
* settlement) so the retry/deadline contract has one testable home.
|
|
5
|
+
*
|
|
6
|
+
* Timeout policy:
|
|
7
|
+
* - Stall timeout aborts only when the stream makes no progress; every event
|
|
8
|
+
* re-arms it, so a slow-but-streaming model is never killed.
|
|
9
|
+
* - Absolute ceiling bounds the pathological streaming-forever case.
|
|
10
|
+
*
|
|
11
|
+
* Retry gate: retry only before committed output (visible text or a tool call)
|
|
12
|
+
* has reached the caller. Reasoning-only attempts are retryable, which is what
|
|
13
|
+
* makes `maxAttempts` real for reasoning-first providers.
|
|
14
|
+
*/
|
|
15
|
+
import { createModelAttemptSignal, DEFAULT_ATTEMPT_CEILING_MS, DEFAULT_ATTEMPT_STALL_MS, getModelAttemptTimeout, modelAttemptTimeoutEvent, } from "./deadline.js";
|
|
16
|
+
import { resolveCall, } from "./domain/index.js";
|
|
17
|
+
import { hasBillableTokenUsage, withMissingUsageMetering } from "./domain/metering.js";
|
|
18
|
+
import { isCommittedOutputEvent, isFirstTokenEvent } from "./stream-events.js";
|
|
19
|
+
const MAX_BUFFERED_BYTES = 1_048_576;
|
|
20
|
+
/**
|
|
21
|
+
* Eagerly collected provider events are bounded so persistence lag cannot
|
|
22
|
+
* retain an unbounded stream. One oversized event is allowed through an empty
|
|
23
|
+
* buffer so a large provider chunk can always make progress.
|
|
24
|
+
*/
|
|
25
|
+
class EventBuffer {
|
|
26
|
+
events = [];
|
|
27
|
+
bytes = 0;
|
|
28
|
+
closed = false;
|
|
29
|
+
failure;
|
|
30
|
+
reader;
|
|
31
|
+
spaceAvailable;
|
|
32
|
+
async push(event, onBackpressure) {
|
|
33
|
+
let waiting = false;
|
|
34
|
+
while (!this.closed &&
|
|
35
|
+
this.events.length > 0 &&
|
|
36
|
+
this.bytes + event.estimatedBytes > MAX_BUFFERED_BYTES) {
|
|
37
|
+
if (!waiting) {
|
|
38
|
+
waiting = true;
|
|
39
|
+
onBackpressure();
|
|
40
|
+
}
|
|
41
|
+
await new Promise((resolve) => {
|
|
42
|
+
this.spaceAvailable = resolve;
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
if (this.closed)
|
|
46
|
+
return false;
|
|
47
|
+
if (this.reader) {
|
|
48
|
+
const reader = this.reader;
|
|
49
|
+
this.reader = undefined;
|
|
50
|
+
reader.resolve({ done: false, value: event });
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
this.events.push(event);
|
|
54
|
+
this.bytes += event.estimatedBytes;
|
|
55
|
+
}
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
next() {
|
|
59
|
+
const event = this.events.shift();
|
|
60
|
+
if (event) {
|
|
61
|
+
this.bytes -= event.estimatedBytes;
|
|
62
|
+
this.spaceAvailable?.();
|
|
63
|
+
this.spaceAvailable = undefined;
|
|
64
|
+
return Promise.resolve({ done: false, value: event });
|
|
65
|
+
}
|
|
66
|
+
if (this.closed) {
|
|
67
|
+
return this.failure === undefined
|
|
68
|
+
? Promise.resolve({ done: true, value: undefined })
|
|
69
|
+
: Promise.reject(this.failure);
|
|
70
|
+
}
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
this.reader = { resolve, reject };
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
finish(error) {
|
|
76
|
+
if (this.closed)
|
|
77
|
+
return;
|
|
78
|
+
this.closed = true;
|
|
79
|
+
this.failure = error;
|
|
80
|
+
this.spaceAvailable?.();
|
|
81
|
+
this.spaceAvailable = undefined;
|
|
82
|
+
if (this.events.length === 0 && this.reader) {
|
|
83
|
+
const reader = this.reader;
|
|
84
|
+
this.reader = undefined;
|
|
85
|
+
if (error === undefined)
|
|
86
|
+
reader.resolve({ done: true, value: undefined });
|
|
87
|
+
else
|
|
88
|
+
reader.reject(error);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function estimatedEventBytes(event) {
|
|
93
|
+
switch (event.type) {
|
|
94
|
+
case "text.delta":
|
|
95
|
+
case "reasoning.delta":
|
|
96
|
+
return 64 + event.text.length * 2;
|
|
97
|
+
case "tool_call.delta":
|
|
98
|
+
return 64 + (event.id.length + event.name.length + event.argumentsDelta.length) * 2;
|
|
99
|
+
case "start":
|
|
100
|
+
return 64 + (event.provider.length + event.model.length) * 2;
|
|
101
|
+
case "usage":
|
|
102
|
+
return 128;
|
|
103
|
+
case "error":
|
|
104
|
+
return 128 + estimatedUnknownBytes(event.result);
|
|
105
|
+
case "end":
|
|
106
|
+
return (256 +
|
|
107
|
+
estimatedUnknownBytes(event.result.content) +
|
|
108
|
+
estimatedUnknownBytes(event.result.toolCalls));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function estimatedUnknownBytes(value) {
|
|
112
|
+
try {
|
|
113
|
+
const serialized = JSON.stringify(value);
|
|
114
|
+
return serialized === undefined ? 0 : serialized.length * 2;
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// An unmeasurable provider payload is treated as oversized and admitted
|
|
118
|
+
// only when the queue is empty, rather than escaping the queue's budget.
|
|
119
|
+
return MAX_BUFFERED_BYTES + 1;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function elapsedMs(startedAt, endedAt) {
|
|
123
|
+
return Math.max(0, Math.round(endedAt - startedAt));
|
|
124
|
+
}
|
|
125
|
+
/** The effective timeouts for an attempt: gateway policy, then the process default. */
|
|
126
|
+
export function resolveAttemptTimeouts(gateway) {
|
|
127
|
+
return {
|
|
128
|
+
stallMs: gateway.attemptStallMs ?? DEFAULT_ATTEMPT_STALL_MS,
|
|
129
|
+
ceilingMs: gateway.attemptCeilingMs ?? DEFAULT_ATTEMPT_CEILING_MS,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
/** Abort-aware sleep for backoff between retry attempts: resolves early on abort. */
|
|
133
|
+
function sleep(ms, signal) {
|
|
134
|
+
return new Promise((resolve) => {
|
|
135
|
+
if (signal?.aborted) {
|
|
136
|
+
resolve();
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const timer = setTimeout(done, ms);
|
|
140
|
+
function done() {
|
|
141
|
+
clearTimeout(timer);
|
|
142
|
+
signal?.removeEventListener("abort", done);
|
|
143
|
+
resolve();
|
|
144
|
+
}
|
|
145
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
export async function* streamWithRetry(adapter, request, model, retry, timeouts, options = {}) {
|
|
149
|
+
const now = options.now ?? (() => performance.now());
|
|
150
|
+
const maxAttempts = retry?.maxAttempts ?? 1;
|
|
151
|
+
let delay = retry?.initialDelayMs ?? 500;
|
|
152
|
+
const maxDelay = retry?.maxDelayMs ?? 10_000;
|
|
153
|
+
const call = resolveCall(request, model);
|
|
154
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
155
|
+
// A stop that landed before this attempt started made no call, so it leaves no response.
|
|
156
|
+
if (request.signal?.aborted)
|
|
157
|
+
return;
|
|
158
|
+
let sawError;
|
|
159
|
+
let emittedCommittedOutput = false;
|
|
160
|
+
const attemptSignal = createModelAttemptSignal(request.signal, timeouts);
|
|
161
|
+
const startedAt = now();
|
|
162
|
+
const requestStartedAt = new Date().toISOString();
|
|
163
|
+
const iterator = adapter
|
|
164
|
+
.stream({ ...request, signal: attemptSignal.signal }, model, call)[Symbol.asyncIterator]();
|
|
165
|
+
const buffer = new EventBuffer();
|
|
166
|
+
let backpressured = false;
|
|
167
|
+
let firstTokenAt;
|
|
168
|
+
let firstTokenArrivedBeforeBackpressure = false;
|
|
169
|
+
const pump = async () => {
|
|
170
|
+
try {
|
|
171
|
+
while (true) {
|
|
172
|
+
const next = await iterator.next();
|
|
173
|
+
if (next.done) {
|
|
174
|
+
buffer.finish();
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
if (!attemptSignal.signal.aborted)
|
|
178
|
+
attemptSignal.notifyProgress();
|
|
179
|
+
const event = next.value;
|
|
180
|
+
// The provider has finished: a timer firing now would discard a complete result.
|
|
181
|
+
if ((event.type === "end" || event.type === "error") && !attemptSignal.signal.aborted) {
|
|
182
|
+
attemptSignal.cleanup();
|
|
183
|
+
}
|
|
184
|
+
const arrivedAt = now();
|
|
185
|
+
const arrivedBeforeBackpressure = !backpressured;
|
|
186
|
+
if (firstTokenAt === undefined && isFirstTokenEvent(event)) {
|
|
187
|
+
firstTokenAt = arrivedAt;
|
|
188
|
+
firstTokenArrivedBeforeBackpressure = arrivedBeforeBackpressure;
|
|
189
|
+
}
|
|
190
|
+
const accepted = await buffer.push({
|
|
191
|
+
event,
|
|
192
|
+
arrivedAt,
|
|
193
|
+
estimatedBytes: estimatedEventBytes(event),
|
|
194
|
+
arrivedBeforeBackpressure,
|
|
195
|
+
}, () => {
|
|
196
|
+
backpressured = true;
|
|
197
|
+
});
|
|
198
|
+
if (!accepted)
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
buffer.finish(error);
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
void pump();
|
|
207
|
+
// A timeout ends the attempt at once with its error; a caller's stop ends in the adapter's `end`.
|
|
208
|
+
let timedOut;
|
|
209
|
+
const stopped = new Promise((resolve) => {
|
|
210
|
+
timedOut = (error) => resolve({ kind: "timeout", error });
|
|
211
|
+
});
|
|
212
|
+
const onAbort = () => {
|
|
213
|
+
const timeout = getModelAttemptTimeout(attemptSignal.signal);
|
|
214
|
+
if (timeout)
|
|
215
|
+
timedOut(timeout);
|
|
216
|
+
};
|
|
217
|
+
attemptSignal.signal.addEventListener("abort", onAbort, { once: true });
|
|
218
|
+
const read = async () => {
|
|
219
|
+
const next = await Promise.race([stopped, buffer.next()]);
|
|
220
|
+
if ("kind" in next)
|
|
221
|
+
throw next.error;
|
|
222
|
+
return next;
|
|
223
|
+
};
|
|
224
|
+
const timeTerminalEvent = (arrival) => {
|
|
225
|
+
if (arrival.event.type !== "end" && arrival.event.type !== "error")
|
|
226
|
+
return arrival.event;
|
|
227
|
+
const terminal = arrival.event.result;
|
|
228
|
+
if (!terminal)
|
|
229
|
+
return arrival.event;
|
|
230
|
+
// A stopped call's finish reason is the adapter's default, not what happened. A stop can
|
|
231
|
+
// also land before the provider's usage arrives (OpenAI-compatible sends it in the final
|
|
232
|
+
// chunk). Mark both here so every adapter's partial result is settled and persisted.
|
|
233
|
+
let result = terminal;
|
|
234
|
+
if (arrival.event.type === "end" && request.signal?.aborted) {
|
|
235
|
+
result = { ...terminal, finishReason: null };
|
|
236
|
+
if (!hasBillableTokenUsage(terminal.usage))
|
|
237
|
+
result = withMissingUsageMetering(result);
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
...arrival.event,
|
|
241
|
+
result: {
|
|
242
|
+
...result,
|
|
243
|
+
timing: {
|
|
244
|
+
requestStartedAt,
|
|
245
|
+
latencyMs: arrival.arrivedBeforeBackpressure
|
|
246
|
+
? elapsedMs(startedAt, arrival.arrivedAt)
|
|
247
|
+
: null,
|
|
248
|
+
timeToFirstTokenMs: firstTokenAt === undefined || !firstTokenArrivedBeforeBackpressure
|
|
249
|
+
? null
|
|
250
|
+
: elapsedMs(startedAt, firstTokenAt),
|
|
251
|
+
generationMs: firstTokenAt === undefined || backpressured
|
|
252
|
+
? null
|
|
253
|
+
: elapsedMs(firstTokenAt, arrival.arrivedAt),
|
|
254
|
+
},
|
|
255
|
+
},
|
|
256
|
+
};
|
|
257
|
+
};
|
|
258
|
+
try {
|
|
259
|
+
while (true) {
|
|
260
|
+
const next = await read();
|
|
261
|
+
if (next.done)
|
|
262
|
+
break;
|
|
263
|
+
if (!attemptSignal.signal.aborted)
|
|
264
|
+
attemptSignal.notifyProgress();
|
|
265
|
+
const { event } = next.value;
|
|
266
|
+
// If the attempt was killed by a timer, surface that timeout event
|
|
267
|
+
// (retryable) instead of whatever the SDK emitted.
|
|
268
|
+
const failure = event.type === "error"
|
|
269
|
+
? (modelAttemptTimeoutEvent(attemptSignal.signal) ?? event)
|
|
270
|
+
: undefined;
|
|
271
|
+
if (failure) {
|
|
272
|
+
sawError = failure;
|
|
273
|
+
if (emittedCommittedOutput || !sawError.retryable || attempt >= maxAttempts) {
|
|
274
|
+
yield timeTerminalEvent({ ...next.value, event: sawError });
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
break;
|
|
278
|
+
}
|
|
279
|
+
emittedCommittedOutput ||= isCommittedOutputEvent(event);
|
|
280
|
+
if (event.type === "end") {
|
|
281
|
+
yield timeTerminalEvent(next.value);
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
yield event;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
catch (error) {
|
|
288
|
+
const timeoutEvent = modelAttemptTimeoutEvent(attemptSignal.signal);
|
|
289
|
+
if (timeoutEvent) {
|
|
290
|
+
sawError = timeoutEvent;
|
|
291
|
+
}
|
|
292
|
+
else {
|
|
293
|
+
sawError = {
|
|
294
|
+
type: "error",
|
|
295
|
+
code: "provider_error",
|
|
296
|
+
message: error instanceof Error ? error.message : String(error),
|
|
297
|
+
retryable: false,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
if (emittedCommittedOutput || !sawError.retryable || attempt >= maxAttempts) {
|
|
301
|
+
yield sawError;
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
finally {
|
|
306
|
+
buffer.finish();
|
|
307
|
+
attemptSignal.signal.removeEventListener("abort", onAbort);
|
|
308
|
+
// Aborting first releases an SDK stream hung on a pending next(), which return() would queue behind.
|
|
309
|
+
attemptSignal.abort();
|
|
310
|
+
attemptSignal.cleanup();
|
|
311
|
+
// A stuck next() can prevent return() from ever settling. Observe teardown
|
|
312
|
+
// rejection without letting provider cleanup hold the run or retry hostage.
|
|
313
|
+
void Promise.resolve()
|
|
314
|
+
.then(() => iterator.return?.())
|
|
315
|
+
.catch(() => undefined);
|
|
316
|
+
}
|
|
317
|
+
if (sawError?.type === "error") {
|
|
318
|
+
if (sawError.retryable && attempt < maxAttempts) {
|
|
319
|
+
// Exponential backoff: wait before retrying, respecting parent abort.
|
|
320
|
+
await sleep(Math.min(Math.max(delay, sawError.retryAfterMs ?? 0), 60_000), request.signal);
|
|
321
|
+
delay = Math.min(delay * 2, maxDelay);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
yield sawError;
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway factory: assembles the provider registry from config, wires the
|
|
3
|
+
* anthropic/openai/openai-compatible/gemini adapters, and returns a Gateway that routes
|
|
4
|
+
* each request to its provider. The gateway's composition root; the per-attempt
|
|
5
|
+
* retry/deadline driver lives in attempt-stream.ts.
|
|
6
|
+
*
|
|
7
|
+
* ── how it works ──
|
|
8
|
+
* 1. createAdapter() maps each ProviderConfig.adapter to its factory.
|
|
9
|
+
* 2. buildProviderRegistry() indexes models by ID so resolveRoute() can look up
|
|
10
|
+
* adapter + model in O(1).
|
|
11
|
+
* 3. gateway.stream() is the main entry point:
|
|
12
|
+
* Resolve the model's route and stream it with retry. Routing errors are
|
|
13
|
+
* non-retryable invalid_request errors; there is no cross-provider fallback.
|
|
14
|
+
* 4. Per-attempt timeout policy is resolved through
|
|
15
|
+
* resolveAttemptTimeouts (gateway config, then the process default) and handed to streamWithRetry.
|
|
16
|
+
*/
|
|
17
|
+
import type { GatewayConfig, ProviderConfig } from "./domain/index.js";
|
|
18
|
+
import type { Gateway } from "./ports/gateway.js";
|
|
19
|
+
/**
|
|
20
|
+
* Reject a provider whose models ask for cache lifetimes its wire cannot express.
|
|
21
|
+
* Separate from adapter construction so the env factory can check every registry
|
|
22
|
+
* row, including providers whose key is not set in this deployment.
|
|
23
|
+
*/
|
|
24
|
+
export declare function assertModelsOnWire(config: Pick<ProviderConfig, "adapter" | "models">): void;
|
|
25
|
+
export declare function createGateway(config: GatewayConfig): Gateway;
|