@meridian-flow/runtime 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +67 -2
- package/dist/gateway/adapters/anthropic/adapter.d.ts +9 -0
- package/dist/gateway/adapters/anthropic/adapter.js +52 -0
- package/dist/gateway/adapters/anthropic/request-map.d.ts +46 -0
- package/dist/gateway/adapters/anthropic/request-map.js +285 -0
- package/dist/gateway/adapters/anthropic/stream-collect.d.ts +67 -0
- package/dist/gateway/adapters/anthropic/stream-collect.js +295 -0
- package/dist/gateway/adapters/anthropic/tool-schema.d.ts +8 -0
- package/dist/gateway/adapters/anthropic/tool-schema.js +68 -0
- package/dist/gateway/adapters/credentials.d.ts +25 -0
- package/dist/gateway/adapters/credentials.js +118 -0
- package/dist/gateway/adapters/gemini/adapter.d.ts +10 -0
- package/dist/gateway/adapters/gemini/adapter.js +90 -0
- package/dist/gateway/adapters/gemini/request-map.d.ts +29 -0
- package/dist/gateway/adapters/gemini/request-map.js +178 -0
- package/dist/gateway/adapters/gemini/stream-collect.d.ts +29 -0
- package/dist/gateway/adapters/gemini/stream-collect.js +116 -0
- package/dist/gateway/adapters/mock/response.d.ts +11 -0
- package/dist/gateway/adapters/mock/response.js +160 -0
- package/dist/gateway/adapters/mock/samples.d.ts +8 -0
- package/dist/gateway/adapters/mock/samples.js +133 -0
- package/dist/gateway/adapters/mock/script-queue.d.ts +10 -0
- package/dist/gateway/adapters/mock/script-queue.js +60 -0
- package/dist/gateway/adapters/mock/server.d.ts +16 -0
- package/dist/gateway/adapters/mock/server.js +342 -0
- package/dist/gateway/adapters/mock/write-directive.d.ts +15 -0
- package/dist/gateway/adapters/mock/write-directive.js +21 -0
- package/dist/gateway/adapters/openai/adapter.d.ts +3 -0
- package/dist/gateway/adapters/openai/adapter.js +35 -0
- package/dist/gateway/adapters/openai/request-map.d.ts +31 -0
- package/dist/gateway/adapters/openai/request-map.js +228 -0
- package/dist/gateway/adapters/openai/stream-collect.d.ts +63 -0
- package/dist/gateway/adapters/openai/stream-collect.js +490 -0
- package/dist/gateway/adapters/openai-compatible/adapter.d.ts +3 -0
- package/dist/gateway/adapters/openai-compatible/adapter.js +35 -0
- package/dist/gateway/adapters/openai-compatible/request-map.d.ts +31 -0
- package/dist/gateway/adapters/openai-compatible/request-map.js +180 -0
- package/dist/gateway/adapters/openai-compatible/stream-collect.d.ts +84 -0
- package/dist/gateway/adapters/openai-compatible/stream-collect.js +215 -0
- package/dist/gateway/adapters/provider-http-error.d.ts +65 -0
- package/dist/gateway/adapters/provider-http-error.js +188 -0
- package/dist/gateway/adapters/provider-stream.d.ts +23 -0
- package/dist/gateway/adapters/provider-stream.js +58 -0
- package/dist/gateway/attempt-stream.d.ts +27 -0
- package/dist/gateway/attempt-stream.js +329 -0
- package/dist/gateway/create-gateway.d.ts +25 -0
- package/dist/gateway/create-gateway.js +88 -0
- package/dist/gateway/deadline.d.ts +56 -0
- package/dist/gateway/deadline.js +120 -0
- package/dist/gateway/domain/index.d.ts +10 -0
- package/dist/gateway/domain/index.js +10 -0
- package/dist/gateway/domain/media.d.ts +62 -0
- package/dist/gateway/domain/media.js +126 -0
- package/dist/gateway/domain/metering.d.ts +7 -0
- package/dist/gateway/domain/metering.js +6 -0
- package/dist/gateway/domain/resolve-call.d.ts +24 -0
- package/dist/gateway/domain/resolve-call.js +39 -0
- package/dist/gateway/domain/types.d.ts +457 -0
- package/dist/gateway/domain/types.js +10 -0
- package/dist/gateway/helpers/messages.d.ts +17 -0
- package/dist/gateway/helpers/messages.js +29 -0
- package/dist/gateway/helpers/parse-tool-arguments.d.ts +9 -0
- package/dist/gateway/helpers/parse-tool-arguments.js +38 -0
- package/dist/gateway/helpers/tool-call-pairing.d.ts +13 -0
- package/dist/gateway/helpers/tool-call-pairing.js +25 -0
- package/dist/gateway/index.d.ts +12 -0
- package/dist/gateway/index.js +11 -0
- package/dist/gateway/mock-script.d.ts +129 -0
- package/dist/gateway/mock-script.js +129 -0
- package/dist/gateway/mock.d.ts +11 -0
- package/dist/gateway/mock.js +22 -0
- package/dist/gateway/ports/gateway.d.ts +18 -0
- package/dist/gateway/ports/gateway.js +1 -0
- package/dist/gateway/ports/provider-adapter.d.ts +21 -0
- package/dist/gateway/ports/provider-adapter.js +1 -0
- package/dist/gateway/routing.d.ts +28 -0
- package/dist/gateway/routing.js +48 -0
- package/dist/gateway/shapes/index.d.ts +7 -0
- package/dist/gateway/shapes/index.js +1 -0
- package/dist/gateway/shapes/json.d.ts +4 -0
- package/dist/gateway/shapes/json.js +1 -0
- package/dist/gateway/shapes/provider-error.d.ts +10 -0
- package/dist/gateway/shapes/provider-error.js +1 -0
- package/dist/gateway/shapes/usage.d.ts +41 -0
- package/dist/gateway/shapes/usage.js +29 -0
- package/dist/gateway/stream-events.d.ts +21 -0
- package/dist/gateway/stream-events.js +26 -0
- package/package.json +76 -3
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway factory: assembles the provider registry from config, wires the
|
|
3
|
+
* anthropic/openai/openai-compatible/gemini adapters, and returns a Gateway that routes
|
|
4
|
+
* each request to its provider. The gateway's composition root; the per-attempt
|
|
5
|
+
* retry/deadline driver lives in attempt-stream.ts.
|
|
6
|
+
*
|
|
7
|
+
* ── how it works ──
|
|
8
|
+
* 1. createAdapter() maps each ProviderConfig.adapter to its factory.
|
|
9
|
+
* 2. buildProviderRegistry() indexes models by ID so resolveRoute() can look up
|
|
10
|
+
* adapter + model in O(1).
|
|
11
|
+
* 3. gateway.stream() is the main entry point:
|
|
12
|
+
* Resolve the model's route and stream it with retry. Routing errors are
|
|
13
|
+
* non-retryable invalid_request errors; there is no cross-provider fallback.
|
|
14
|
+
* 4. Per-attempt timeout policy is resolved through
|
|
15
|
+
* resolveAttemptTimeouts (gateway config, then the process default) and handed to streamWithRetry.
|
|
16
|
+
*/
|
|
17
|
+
import { createAnthropicAdapter } from "./adapters/anthropic/adapter.js";
|
|
18
|
+
import { assertCacheTtlsOnWire as assertAnthropicCacheTtlsOnWire } from "./adapters/anthropic/request-map.js";
|
|
19
|
+
import { createGeminiAdapter } from "./adapters/gemini/adapter.js";
|
|
20
|
+
import { assertCacheTtlsOnWire as assertGeminiCacheTtlsOnWire } from "./adapters/gemini/request-map.js";
|
|
21
|
+
import { createOpenAIResponsesAdapter } from "./adapters/openai/adapter.js";
|
|
22
|
+
import { assertCacheTtlsOnWire as assertOpenAICacheTtlsOnWire } from "./adapters/openai/request-map.js";
|
|
23
|
+
import { createOpenAICompatibleAdapter } from "./adapters/openai-compatible/adapter.js";
|
|
24
|
+
import { resolveAttemptTimeouts, streamWithRetry, } from "./attempt-stream.js";
|
|
25
|
+
import { buildProviderRegistry, resolveRoute } from "./routing.js";
|
|
26
|
+
/** Maps a ProviderConfig.adapter to its concrete ProviderAdapter factory. */
|
|
27
|
+
function createAdapter(config) {
|
|
28
|
+
switch (config.adapter) {
|
|
29
|
+
case "openai-compatible":
|
|
30
|
+
return createOpenAICompatibleAdapter(config);
|
|
31
|
+
case "anthropic":
|
|
32
|
+
return createAnthropicAdapter(config);
|
|
33
|
+
case "openai":
|
|
34
|
+
return createOpenAIResponsesAdapter(config);
|
|
35
|
+
case "gemini":
|
|
36
|
+
return createGeminiAdapter(config);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Reject a provider whose models ask for cache lifetimes its wire cannot express.
|
|
41
|
+
* Separate from adapter construction so the env factory can check every registry
|
|
42
|
+
* row, including providers whose key is not set in this deployment.
|
|
43
|
+
*/
|
|
44
|
+
export function assertModelsOnWire(config) {
|
|
45
|
+
for (const model of config.models) {
|
|
46
|
+
if (config.adapter === "anthropic")
|
|
47
|
+
assertAnthropicCacheTtlsOnWire(model);
|
|
48
|
+
else if (config.adapter === "openai")
|
|
49
|
+
assertOpenAICacheTtlsOnWire(model);
|
|
50
|
+
else if (config.adapter === "gemini")
|
|
51
|
+
assertGeminiCacheTtlsOnWire(model);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export function createGateway(config) {
|
|
55
|
+
for (const provider of config.providers)
|
|
56
|
+
assertModelsOnWire(provider);
|
|
57
|
+
const registry = buildProviderRegistry(config.providers.map((provider) => ({ config: provider, adapter: createAdapter(provider) })), config.onWarning);
|
|
58
|
+
const gateway = {
|
|
59
|
+
async *stream(request) {
|
|
60
|
+
const retry = config.retry;
|
|
61
|
+
const timeoutOverrides = {
|
|
62
|
+
attemptStallMs: config.attemptStallMs,
|
|
63
|
+
attemptCeilingMs: config.attemptCeilingMs,
|
|
64
|
+
};
|
|
65
|
+
// Routing errors (unknown model, missing adapter) are surfaced as
|
|
66
|
+
// non-retryable invalid_request errors.
|
|
67
|
+
try {
|
|
68
|
+
const route = resolveRoute(registry, request, config.defaultModel);
|
|
69
|
+
yield* streamWithRetry(route.adapter, request, route.model, retry, resolveAttemptTimeouts(timeoutOverrides));
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
yield {
|
|
73
|
+
type: "error",
|
|
74
|
+
code: "invalid_request",
|
|
75
|
+
message: err instanceof Error ? err.message : String(err),
|
|
76
|
+
retryable: false,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
listModels() {
|
|
81
|
+
return [...registry.values()].map((e) => e.model);
|
|
82
|
+
},
|
|
83
|
+
getDefaultModel() {
|
|
84
|
+
return config.defaultModel;
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
return gateway;
|
|
88
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/** Default inactivity window before an attempt with no stream progress is aborted. */
|
|
2
|
+
export declare const DEFAULT_ATTEMPT_STALL_MS = 60000;
|
|
3
|
+
/** Default absolute ceiling for one attempt. 0 disables it (e.g. self-hosted models). */
|
|
4
|
+
export declare const DEFAULT_ATTEMPT_CEILING_MS = 900000;
|
|
5
|
+
export type ModelAttemptTimeoutKind = "stall" | "ceiling";
|
|
6
|
+
export interface ModelAttemptTimeouts {
|
|
7
|
+
/** Abort after this long with no stream progress. 0 disables the stall guard. */
|
|
8
|
+
stallMs: number;
|
|
9
|
+
/** Absolute per-attempt ceiling. 0 disables the ceiling backstop. */
|
|
10
|
+
ceilingMs: number;
|
|
11
|
+
}
|
|
12
|
+
export declare class ModelAttemptTimeoutError extends Error {
|
|
13
|
+
readonly timeoutMs: number;
|
|
14
|
+
readonly kind: ModelAttemptTimeoutKind;
|
|
15
|
+
constructor(timeoutMs: number, kind: ModelAttemptTimeoutKind);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Inspect an AbortSignal's reason to see if it was caused by a model attempt
|
|
19
|
+
* timeout. Returns the ModelAttemptTimeoutError instance if so, null otherwise.
|
|
20
|
+
*/
|
|
21
|
+
export declare function getModelAttemptTimeout(signal: AbortSignal | undefined): ModelAttemptTimeoutError | null;
|
|
22
|
+
export interface ModelAttemptSignal {
|
|
23
|
+
signal: AbortSignal;
|
|
24
|
+
/** Re-arm the inactivity deadline. Call on every stream progress event. */
|
|
25
|
+
notifyProgress: () => void;
|
|
26
|
+
/** Abort the derived signal, releasing a provider call still in flight. */
|
|
27
|
+
abort: () => void;
|
|
28
|
+
/** Clear both timers and the parent listener, and stop re-arming. Idempotent. */
|
|
29
|
+
cleanup: () => void;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Create a derived AbortSignal that combines a parent signal with a resettable
|
|
33
|
+
* inactivity timer and a fixed ceiling timer.
|
|
34
|
+
*
|
|
35
|
+
* Behavior:
|
|
36
|
+
* - Creates an AbortController armed with both timers. `notifyProgress`
|
|
37
|
+
* resets only the stall timer; the ceiling timer is never extended.
|
|
38
|
+
* - A timer value <= 0 disables that timer. With both disabled the attempt has no
|
|
39
|
+
* deadline at all (explicit self-hosted configuration).
|
|
40
|
+
* - If the parent aborts first, clears both timers and propagates the parent's
|
|
41
|
+
* abort reason.
|
|
42
|
+
* - cleanup() is idempotent and safe to call multiple times.
|
|
43
|
+
*/
|
|
44
|
+
export declare function createModelAttemptSignal(parent: AbortSignal | undefined, timeouts: ModelAttemptTimeouts): ModelAttemptSignal;
|
|
45
|
+
/**
|
|
46
|
+
* Check whether an abort signal was triggered by a model attempt timeout.
|
|
47
|
+
* Returns a canonical retryable error event if so, null otherwise.
|
|
48
|
+
* This is called in streamWithRetry to classify abort reasons: timeouts are
|
|
49
|
+
* retryable, user-initiated cancellations (parent signal abort) are not.
|
|
50
|
+
*/
|
|
51
|
+
export declare function modelAttemptTimeoutEvent(signal: AbortSignal | undefined): {
|
|
52
|
+
type: "error";
|
|
53
|
+
code: "provider_error";
|
|
54
|
+
message: string;
|
|
55
|
+
retryable: true;
|
|
56
|
+
} | null;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-attempt model-call timeout: defines ModelAttemptTimeoutError and the
|
|
3
|
+
* AbortSignal helper that surfaces a stall or ceiling timeout as a stream event.
|
|
4
|
+
* Owns the single timeout convention `attempt-stream.ts` applies to every attempt.
|
|
5
|
+
*
|
|
6
|
+
* Design: the gateway guards each attempt with two timers on one derived
|
|
7
|
+
* AbortSignal, not a Promise.race on the full generator:
|
|
8
|
+
* - The inactivity (stall) timer is the primary guard. It is re-armed on every
|
|
9
|
+
* stream progress event, so a slow-but-streaming model is never killed — only
|
|
10
|
+
* a stream that stops making progress is.
|
|
11
|
+
* - The absolute ceiling is a backstop for a stream that never terminates. It is
|
|
12
|
+
* fixed from attempt start and is not the normal terminator; 0 disables it for
|
|
13
|
+
* self-hosted models.
|
|
14
|
+
* - Either timer aborts the underlying provider HTTP stream (via the SDK's own
|
|
15
|
+
* signal support), not just the iterator.
|
|
16
|
+
* - If the provider SDK ignores the signal, the provider-stream envelope
|
|
17
|
+
* races the iterator against the abort anyway.
|
|
18
|
+
* - The timeout error is surfaced as a retryable `provider_error` StreamEvent,
|
|
19
|
+
* so the retry machinery can try the same provider again.
|
|
20
|
+
*/
|
|
21
|
+
const ATTEMPT_TIMEOUT_NAME = "ModelAttemptTimeoutError";
|
|
22
|
+
/** Default inactivity window before an attempt with no stream progress is aborted. */
|
|
23
|
+
export const DEFAULT_ATTEMPT_STALL_MS = 60_000;
|
|
24
|
+
/** Default absolute ceiling for one attempt. 0 disables it (e.g. self-hosted models). */
|
|
25
|
+
export const DEFAULT_ATTEMPT_CEILING_MS = 900_000;
|
|
26
|
+
export class ModelAttemptTimeoutError extends Error {
|
|
27
|
+
timeoutMs;
|
|
28
|
+
kind;
|
|
29
|
+
constructor(timeoutMs, kind) {
|
|
30
|
+
super(kind === "stall"
|
|
31
|
+
? `Model stream stalled for ${timeoutMs}ms`
|
|
32
|
+
: `Model request exceeded its ${timeoutMs}ms ceiling`);
|
|
33
|
+
this.timeoutMs = timeoutMs;
|
|
34
|
+
this.kind = kind;
|
|
35
|
+
this.name = ATTEMPT_TIMEOUT_NAME;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Inspect an AbortSignal's reason to see if it was caused by a model attempt
|
|
40
|
+
* timeout. Returns the ModelAttemptTimeoutError instance if so, null otherwise.
|
|
41
|
+
*/
|
|
42
|
+
export function getModelAttemptTimeout(signal) {
|
|
43
|
+
const reason = signal?.reason;
|
|
44
|
+
return reason instanceof ModelAttemptTimeoutError ? reason : null;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Create a derived AbortSignal that combines a parent signal with a resettable
|
|
48
|
+
* inactivity timer and a fixed ceiling timer.
|
|
49
|
+
*
|
|
50
|
+
* Behavior:
|
|
51
|
+
* - Creates an AbortController armed with both timers. `notifyProgress`
|
|
52
|
+
* resets only the stall timer; the ceiling timer is never extended.
|
|
53
|
+
* - A timer value <= 0 disables that timer. With both disabled the attempt has no
|
|
54
|
+
* deadline at all (explicit self-hosted configuration).
|
|
55
|
+
* - If the parent aborts first, clears both timers and propagates the parent's
|
|
56
|
+
* abort reason.
|
|
57
|
+
* - cleanup() is idempotent and safe to call multiple times.
|
|
58
|
+
*/
|
|
59
|
+
export function createModelAttemptSignal(parent, timeouts) {
|
|
60
|
+
const controller = new AbortController();
|
|
61
|
+
let stallTimer;
|
|
62
|
+
let ceilingTimer;
|
|
63
|
+
let done = false;
|
|
64
|
+
const armStall = () => {
|
|
65
|
+
if (done)
|
|
66
|
+
return;
|
|
67
|
+
if (stallTimer !== undefined)
|
|
68
|
+
clearTimeout(stallTimer);
|
|
69
|
+
stallTimer =
|
|
70
|
+
timeouts.stallMs > 0
|
|
71
|
+
? setTimeout(() => {
|
|
72
|
+
controller.abort(new ModelAttemptTimeoutError(timeouts.stallMs, "stall"));
|
|
73
|
+
}, timeouts.stallMs)
|
|
74
|
+
: undefined;
|
|
75
|
+
};
|
|
76
|
+
armStall();
|
|
77
|
+
if (timeouts.ceilingMs > 0) {
|
|
78
|
+
ceilingTimer = setTimeout(() => {
|
|
79
|
+
controller.abort(new ModelAttemptTimeoutError(timeouts.ceilingMs, "ceiling"));
|
|
80
|
+
}, timeouts.ceilingMs);
|
|
81
|
+
}
|
|
82
|
+
const onParentAbort = () => {
|
|
83
|
+
if (stallTimer !== undefined)
|
|
84
|
+
clearTimeout(stallTimer);
|
|
85
|
+
if (ceilingTimer !== undefined)
|
|
86
|
+
clearTimeout(ceilingTimer);
|
|
87
|
+
controller.abort(parent?.reason ?? new Error("Request aborted"));
|
|
88
|
+
};
|
|
89
|
+
parent?.addEventListener("abort", onParentAbort, { once: true });
|
|
90
|
+
return {
|
|
91
|
+
signal: controller.signal,
|
|
92
|
+
notifyProgress: armStall,
|
|
93
|
+
abort: () => controller.abort(new Error("Attempt finished")),
|
|
94
|
+
cleanup: () => {
|
|
95
|
+
done = true;
|
|
96
|
+
if (stallTimer !== undefined)
|
|
97
|
+
clearTimeout(stallTimer);
|
|
98
|
+
if (ceilingTimer !== undefined)
|
|
99
|
+
clearTimeout(ceilingTimer);
|
|
100
|
+
parent?.removeEventListener("abort", onParentAbort);
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Check whether an abort signal was triggered by a model attempt timeout.
|
|
106
|
+
* Returns a canonical retryable error event if so, null otherwise.
|
|
107
|
+
* This is called in streamWithRetry to classify abort reasons: timeouts are
|
|
108
|
+
* retryable, user-initiated cancellations (parent signal abort) are not.
|
|
109
|
+
*/
|
|
110
|
+
export function modelAttemptTimeoutEvent(signal) {
|
|
111
|
+
const timeout = getModelAttemptTimeout(signal);
|
|
112
|
+
if (!timeout)
|
|
113
|
+
return null;
|
|
114
|
+
return {
|
|
115
|
+
type: "error",
|
|
116
|
+
code: "provider_error",
|
|
117
|
+
message: timeout.message,
|
|
118
|
+
retryable: true,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Barrel: re-exports the gateway's canonical domain types.
|
|
3
|
+
*
|
|
4
|
+
* Consumer modules import `from "./domain/index.js"` (aliased via
|
|
5
|
+
* `"../domain/index.js"` from deeper modules) and get the full type-only
|
|
6
|
+
* contract without reaching into individual files.
|
|
7
|
+
*/
|
|
8
|
+
export { type MediaKind, type MediaPlacement, mediaKind, PDF_PAGE_TOKENS, placeMedia, } from "./media.js";
|
|
9
|
+
export * from "./resolve-call.js";
|
|
10
|
+
export * from "./types.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Barrel: re-exports the gateway's canonical domain types.
|
|
3
|
+
*
|
|
4
|
+
* Consumer modules import `from "./domain/index.js"` (aliased via
|
|
5
|
+
* `"../domain/index.js"` from deeper modules) and get the full type-only
|
|
6
|
+
* contract without reaching into individual files.
|
|
7
|
+
*/
|
|
8
|
+
export { mediaKind, PDF_PAGE_TOKENS, placeMedia, } from "./media.js";
|
|
9
|
+
export * from "./resolve-call.js";
|
|
10
|
+
export * from "./types.js";
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a model can take as media and how each wire format carries it. Provider facts are in
|
|
3
|
+
* `findings/provider-media-support.md` of the gateway-and-media work item.
|
|
4
|
+
*
|
|
5
|
+
* Two levels, because they change independently: the model says which kinds it takes
|
|
6
|
+
* (`inputModalities`, copied from models.dev) and how large a file its provider fetches by link
|
|
7
|
+
* (`maxLinkBytes`); the wire format says whether a link works per kind and where a tool result's
|
|
8
|
+
* media goes. Two models of one provider can differ by wire (OpenAI Chat against Responses).
|
|
9
|
+
* Placement is looked up by model, not wire alone: the Gemini wire keys it on the model's major
|
|
10
|
+
* version (nested `functionResponse` parts need Gemini 3), and an OpenAI-compatible host may or
|
|
11
|
+
* may not fetch image links.
|
|
12
|
+
*/
|
|
13
|
+
import type { MediaPart, ModelInfo } from "./types.js";
|
|
14
|
+
export type MediaKind = "image" | "pdf";
|
|
15
|
+
/**
|
|
16
|
+
* What one PDF page costs a model, text and page image together: Anthropic measured 1,832 tokens
|
|
17
|
+
* a page for short pages, and its docs give 1,500 to 3,000. The one figure for deciding whether a
|
|
18
|
+
* PDF fits (`placeMedia`) and for pricing one in a context estimate.
|
|
19
|
+
*/
|
|
20
|
+
export declare const PDF_PAGE_TOKENS = 2000;
|
|
21
|
+
/** The major version in a Gemini model id (`gemini-3.1-flash-lite` is 3), or null for an id that doesn't say. */
|
|
22
|
+
export declare function geminiMajorVersion(modelId: string): number | null;
|
|
23
|
+
export declare function mediaKind(mediaType: string): MediaKind | null;
|
|
24
|
+
type MediaModel = Pick<ModelInfo, "wire" | "imageLinks">;
|
|
25
|
+
export type MediaPlacement = {
|
|
26
|
+
transport: "link" | "bytes";
|
|
27
|
+
toolResult: "inside" | "after" | "none";
|
|
28
|
+
} | {
|
|
29
|
+
transport: "none";
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* How to send a file of this type (and size, when known) to this model: as a link, as bytes, or
|
|
33
|
+
* not at all. `none` covers a kind the model doesn't take, an image type its wire doesn't
|
|
34
|
+
* accept (the caller can send a preview instead), a file over the provider's link limit, a PDF
|
|
35
|
+
* (when its `pages` are known) too long for the model's page cap or context window, and,
|
|
36
|
+
* when `inToolResult` is set, a model that takes no media in a tool result; the caller then
|
|
37
|
+
* describes the file in text. The wire decides whether a link works, so this never answers
|
|
38
|
+
* `bytes` for a file a link could carry. `toolResult` says where a tool result's media go
|
|
39
|
+
* (`none`: nowhere, for a model that takes media only in a user message).
|
|
40
|
+
*/
|
|
41
|
+
export declare function placeMedia(model: Pick<ModelInfo, "id" | "wire" | "inputModalities" | "maxLinkBytes" | "imageLinks" | "contextWindow">, media: {
|
|
42
|
+
mediaType: string;
|
|
43
|
+
sizeBytes?: number;
|
|
44
|
+
pages?: number;
|
|
45
|
+
inToolResult?: true;
|
|
46
|
+
}): MediaPlacement;
|
|
47
|
+
export type WireMediaSource = {
|
|
48
|
+
kind: MediaKind;
|
|
49
|
+
url: string;
|
|
50
|
+
} | {
|
|
51
|
+
kind: MediaKind;
|
|
52
|
+
data: string;
|
|
53
|
+
mediaType: string;
|
|
54
|
+
filename?: string;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* The part as a wire can carry it, or an error: a mapper never downloads a link, so a source
|
|
58
|
+
* the wire can't take stops the request. A PDF sent as bytes always has a filename (OpenAI
|
|
59
|
+
* requires one); one sent by link needs none.
|
|
60
|
+
*/
|
|
61
|
+
export declare function wireMediaSource(model: MediaModel, part: MediaPart): WireMediaSource;
|
|
62
|
+
export {};
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What one PDF page costs a model, text and page image together: Anthropic measured 1,832 tokens
|
|
3
|
+
* a page for short pages, and its docs give 1,500 to 3,000. The one figure for deciding whether a
|
|
4
|
+
* PDF fits (`placeMedia`) and for pricing one in a context estimate.
|
|
5
|
+
*/
|
|
6
|
+
export const PDF_PAGE_TOKENS = 2_000;
|
|
7
|
+
/** The share of a model's context window a PDF may take as media, so the conversation still fits around it. */
|
|
8
|
+
const PDF_CONTEXT_SHARE = 0.5;
|
|
9
|
+
/** Anthropic takes 600 pages a request, 100 when the context window is under 1M tokens. */
|
|
10
|
+
const ANTHROPIC_PDF_PAGES = { large: 600, small: 100, largeWindowTokens: 1_000_000 };
|
|
11
|
+
const OPENAI_IMAGE_TYPES = ["image/png", "image/jpeg", "image/webp", "image/gif"];
|
|
12
|
+
const WIRES = {
|
|
13
|
+
anthropic: {
|
|
14
|
+
pdfPageCap: (model) => model.contextWindow >= ANTHROPIC_PDF_PAGES.largeWindowTokens
|
|
15
|
+
? ANTHROPIC_PDF_PAGES.large
|
|
16
|
+
: ANTHROPIC_PDF_PAGES.small,
|
|
17
|
+
link: { image: true, pdf: true },
|
|
18
|
+
imageTypes: ["image/jpeg", "image/png", "image/gif", "image/webp"],
|
|
19
|
+
toolResult: () => "inside",
|
|
20
|
+
},
|
|
21
|
+
openai: {
|
|
22
|
+
link: { image: true, pdf: true },
|
|
23
|
+
imageTypes: OPENAI_IMAGE_TYPES,
|
|
24
|
+
toolResult: () => "inside",
|
|
25
|
+
},
|
|
26
|
+
// Chat Completions has no file URL on a `file` part, so a PDF is bytes only; whether a host
|
|
27
|
+
// fetches image links is its own fact (`ModelInfo.imageLinks`).
|
|
28
|
+
"openai-compatible": {
|
|
29
|
+
link: { image: false, pdf: false },
|
|
30
|
+
imageTypes: OPENAI_IMAGE_TYPES,
|
|
31
|
+
toolResult: () => "after",
|
|
32
|
+
},
|
|
33
|
+
// `functionResponse.parts` needs Gemini 3; an older or unknown model can't take media there.
|
|
34
|
+
gemini: {
|
|
35
|
+
link: { image: true, pdf: true },
|
|
36
|
+
imageTypes: ["image/png", "image/jpeg", "image/webp", "image/heic", "image/heif"],
|
|
37
|
+
// Google lists only these inside a function response.
|
|
38
|
+
toolResultImageTypes: ["image/png", "image/jpeg", "image/webp"],
|
|
39
|
+
toolResult: (model) => ((geminiMajorVersion(model.id) ?? 0) >= 3 ? "inside" : "none"),
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
/** The major version in a Gemini model id (`gemini-3.1-flash-lite` is 3), or null for an id that doesn't say. */
|
|
43
|
+
export function geminiMajorVersion(modelId) {
|
|
44
|
+
const match = /^gemini-(\d+)(?:[.-]|$)/.exec(modelId);
|
|
45
|
+
return match ? Number(match[1]) : null;
|
|
46
|
+
}
|
|
47
|
+
export function mediaKind(mediaType) {
|
|
48
|
+
if (mediaType.startsWith("image/"))
|
|
49
|
+
return "image";
|
|
50
|
+
return mediaType === "application/pdf" ? "pdf" : null;
|
|
51
|
+
}
|
|
52
|
+
function linksWork(model, kind) {
|
|
53
|
+
return WIRES[model.wire].link[kind] || (kind === "image" && model.imageLinks === true);
|
|
54
|
+
}
|
|
55
|
+
/** Whether the wire accepts this file type at all (an image type outside its table is refused). */
|
|
56
|
+
function wireAccepts(wire, kind, mediaType) {
|
|
57
|
+
return kind === "pdf" || WIRES[wire].imageTypes.includes(mediaType);
|
|
58
|
+
}
|
|
59
|
+
/** Whether a PDF of this many pages fits the model as media: under the provider's page cap and a share of its window. */
|
|
60
|
+
function pdfFits(model, pages) {
|
|
61
|
+
const cap = WIRES[model.wire].pdfPageCap?.(model) ?? Number.POSITIVE_INFINITY;
|
|
62
|
+
return pages <= cap && pages * PDF_PAGE_TOKENS <= model.contextWindow * PDF_CONTEXT_SHARE;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* How to send a file of this type (and size, when known) to this model: as a link, as bytes, or
|
|
66
|
+
* not at all. `none` covers a kind the model doesn't take, an image type its wire doesn't
|
|
67
|
+
* accept (the caller can send a preview instead), a file over the provider's link limit, a PDF
|
|
68
|
+
* (when its `pages` are known) too long for the model's page cap or context window, and,
|
|
69
|
+
* when `inToolResult` is set, a model that takes no media in a tool result; the caller then
|
|
70
|
+
* describes the file in text. The wire decides whether a link works, so this never answers
|
|
71
|
+
* `bytes` for a file a link could carry. `toolResult` says where a tool result's media go
|
|
72
|
+
* (`none`: nowhere, for a model that takes media only in a user message).
|
|
73
|
+
*/
|
|
74
|
+
export function placeMedia(model, media) {
|
|
75
|
+
const kind = mediaKind(media.mediaType);
|
|
76
|
+
if (kind === null || !model.inputModalities.includes(kind))
|
|
77
|
+
return { transport: "none" };
|
|
78
|
+
if (!wireAccepts(model.wire, kind, media.mediaType))
|
|
79
|
+
return { transport: "none" };
|
|
80
|
+
if (kind === "pdf" && media.pages !== undefined && !pdfFits(model, media.pages))
|
|
81
|
+
return { transport: "none" };
|
|
82
|
+
const inToolResultTypes = WIRES[model.wire].toolResultImageTypes;
|
|
83
|
+
if (media.inToolResult &&
|
|
84
|
+
kind === "image" &&
|
|
85
|
+
inToolResultTypes?.includes(media.mediaType) === false)
|
|
86
|
+
return { transport: "none" };
|
|
87
|
+
const toolResult = WIRES[model.wire].toolResult(model);
|
|
88
|
+
if (toolResult === "none" && media.inToolResult)
|
|
89
|
+
return { transport: "none" };
|
|
90
|
+
if (!linksWork(model, kind))
|
|
91
|
+
return { transport: "bytes", toolResult };
|
|
92
|
+
const { maxLinkBytes } = model;
|
|
93
|
+
if (maxLinkBytes !== undefined && media.sizeBytes !== undefined && media.sizeBytes > maxLinkBytes)
|
|
94
|
+
return { transport: "none" };
|
|
95
|
+
return { transport: "link", toolResult };
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The part as a wire can carry it, or an error: a mapper never downloads a link, so a source
|
|
99
|
+
* the wire can't take stops the request. A PDF sent as bytes always has a filename (OpenAI
|
|
100
|
+
* requires one); one sent by link needs none.
|
|
101
|
+
*/
|
|
102
|
+
export function wireMediaSource(model, part) {
|
|
103
|
+
const kind = mediaKind(part.mediaType);
|
|
104
|
+
if (kind === null || !wireAccepts(model.wire, kind, part.mediaType)) {
|
|
105
|
+
throw new Error(`${model.wire} cannot carry media of type ${part.mediaType}`);
|
|
106
|
+
}
|
|
107
|
+
const { source } = part;
|
|
108
|
+
if ("reference" in source) {
|
|
109
|
+
throw new Error(`${model.wire} does not send provider file references (${source.reference.provider})`);
|
|
110
|
+
}
|
|
111
|
+
if ("data" in source) {
|
|
112
|
+
return {
|
|
113
|
+
kind,
|
|
114
|
+
data: source.data,
|
|
115
|
+
mediaType: part.mediaType,
|
|
116
|
+
...(kind === "pdf" ? { filename: part.filename ?? "document.pdf" } : {}),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
if (!source.url.startsWith("https://")) {
|
|
120
|
+
throw new Error("A media link must be an https URL; send bytes in `data`, not a data URL");
|
|
121
|
+
}
|
|
122
|
+
if (!linksWork(model, kind)) {
|
|
123
|
+
throw new Error(`${model.wire} takes ${kind} media as bytes only, not by link`);
|
|
124
|
+
}
|
|
125
|
+
return { kind, url: source.url };
|
|
126
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metering helpers on GenerateResult. `attempt-stream.ts` flags a cancelled stream
|
|
3
|
+
* that ended without provider usage with `meteringStatus: "missing_usage"`.
|
|
4
|
+
*/
|
|
5
|
+
import type { GenerateResult } from "./types.js";
|
|
6
|
+
export declare function withMissingUsageMetering(result: GenerateResult): GenerateResult;
|
|
7
|
+
export declare function hasBillableTokenUsage(usage: GenerateResult["usage"]): boolean;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-call model decisions, made once by the gateway core where the routed
|
|
3
|
+
* model is known. Adapters only map a `ResolvedCall` to their wire fields; they
|
|
4
|
+
* never re-derive cache lifetimes or clamp efforts themselves.
|
|
5
|
+
*/
|
|
6
|
+
import { type GenerateRequest, type ModelInfo, type ReasoningEffort, type ReasoningSupport } from "./types.js";
|
|
7
|
+
/** `default` sends nothing and leaves the model's own default; `off` is only resolved for models that can disable reasoning. */
|
|
8
|
+
export type ResolvedReasoning = "default" | "off" | {
|
|
9
|
+
effort: ReasoningEffort;
|
|
10
|
+
};
|
|
11
|
+
export interface ResolvedCall {
|
|
12
|
+
/** Lifetime to put on the wire for cache breakpoints; null when the provider publishes none or the retention is `none` (no mark sent). */
|
|
13
|
+
cacheTtlSeconds: number | null;
|
|
14
|
+
/** Whether cache writes are billed at the model's long lifetime. */
|
|
15
|
+
longCacheWrites: boolean;
|
|
16
|
+
reasoning: ResolvedReasoning;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Resolve a canonical reasoning request against the model's facts. `adaptive` and
|
|
20
|
+
* an absent request mean the model's default; `disabled` turns reasoning off only
|
|
21
|
+
* when the model can, otherwise it takes the lowest supported effort.
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveReasoning(requested: GenerateRequest["reasoning"], support: ReasoningSupport): ResolvedReasoning;
|
|
24
|
+
export declare function resolveCall(request: Pick<GenerateRequest, "promptCacheRetention" | "reasoning">, model: Pick<ModelInfo, "promptCache" | "reasoning">): ResolvedCall;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-call model decisions, made once by the gateway core where the routed
|
|
3
|
+
* model is known. Adapters only map a `ResolvedCall` to their wire fields; they
|
|
4
|
+
* never re-derive cache lifetimes or clamp efforts themselves.
|
|
5
|
+
*/
|
|
6
|
+
import { REASONING_EFFORTS, resolvePromptCacheTtlSeconds, } from "./types.js";
|
|
7
|
+
/** The supported effort nearest the requested one; a tie goes to the higher effort. */
|
|
8
|
+
function nearestEffort(requested, supported) {
|
|
9
|
+
const rank = (effort) => REASONING_EFFORTS.indexOf(effort);
|
|
10
|
+
return supported.reduce((best, effort) => {
|
|
11
|
+
const distance = Math.abs(rank(effort) - rank(requested));
|
|
12
|
+
const bestDistance = Math.abs(rank(best) - rank(requested));
|
|
13
|
+
return distance < bestDistance || (distance === bestDistance && rank(effort) > rank(best))
|
|
14
|
+
? effort
|
|
15
|
+
: best;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Resolve a canonical reasoning request against the model's facts. `adaptive` and
|
|
20
|
+
* an absent request mean the model's default; `disabled` turns reasoning off only
|
|
21
|
+
* when the model can, otherwise it takes the lowest supported effort.
|
|
22
|
+
*/
|
|
23
|
+
export function resolveReasoning(requested, support) {
|
|
24
|
+
if (requested === undefined || requested === "adaptive")
|
|
25
|
+
return "default";
|
|
26
|
+
if (requested === "disabled") {
|
|
27
|
+
return support.canDisable ? "off" : { effort: nearestEffort("low", support.efforts) };
|
|
28
|
+
}
|
|
29
|
+
return { effort: nearestEffort(requested.effort, support.efforts) };
|
|
30
|
+
}
|
|
31
|
+
export function resolveCall(request, model) {
|
|
32
|
+
const { promptCache } = model;
|
|
33
|
+
return {
|
|
34
|
+
cacheTtlSeconds: resolvePromptCacheTtlSeconds(promptCache, request.promptCacheRetention),
|
|
35
|
+
longCacheWrites: request.promptCacheRetention === "long" &&
|
|
36
|
+
promptCache.longTtlSeconds !== promptCache.shortTtlSeconds,
|
|
37
|
+
reasoning: resolveReasoning(request.reasoning, model.reasoning),
|
|
38
|
+
};
|
|
39
|
+
}
|