@meridian-flow/runtime 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/dist/gateway/adapters/anthropic/request-map.d.ts +1 -2
- package/dist/gateway/adapters/anthropic/stream-collect.d.ts +4 -4
- package/dist/gateway/adapters/anthropic/stream-collect.js +4 -4
- package/dist/gateway/adapters/gemini/request-map.d.ts +1 -1
- package/dist/gateway/adapters/openai/request-map.d.ts +2 -2
- package/dist/gateway/adapters/openai/request-map.js +1 -1
- package/dist/gateway/adapters/openai/stream-collect.d.ts +5 -6
- package/dist/gateway/adapters/openai/stream-collect.js +10 -11
- package/dist/gateway/adapters/openai-compatible/stream-collect.d.ts +1 -1
- package/dist/gateway/adapters/openai-compatible/stream-collect.js +2 -2
- package/dist/gateway/domain/media.d.ts +3 -2
- package/dist/gateway/domain/types.d.ts +23 -25
- package/dist/gateway/domain/types.js +1 -1
- package/dist/gateway/helpers/messages.d.ts +1 -2
- package/dist/gateway/helpers/messages.js +1 -2
- package/dist/gateway/mock-script.d.ts +3 -2
- package/dist/gateway/mock-script.js +4 -3
- package/dist/gateway/ports/gateway.d.ts +6 -8
- package/dist/gateway/ports/provider-adapter.d.ts +1 -1
- package/dist/gateway/shapes/provider-error.d.ts +2 -3
- package/dist/gateway/shapes/usage.d.ts +3 -4
- package/dist/gateway/shapes/usage.js +3 -4
- package/dist/gateway/stream-events.d.ts +1 -1
- package/dist/gateway/stream-events.js +1 -1
- package/package.json +15 -1
package/README.md
CHANGED
|
@@ -22,7 +22,9 @@ The package reads no environment variable: keys, URLs and policy come in through
|
|
|
22
22
|
|
|
23
23
|
## Example
|
|
24
24
|
|
|
25
|
-
A gateway with one provider and a credential chosen per call
|
|
25
|
+
A gateway with one provider and a credential chosen per call. The model row (limits, input
|
|
26
|
+
kinds, reasoning efforts, cache lifetimes) is yours to declare; this one follows
|
|
27
|
+
[models.dev](https://models.dev) for Claude Sonnet 5.5.
|
|
26
28
|
|
|
27
29
|
```ts
|
|
28
30
|
import { createGateway, user } from "@meridian-flow/runtime/gateway";
|
|
@@ -34,14 +36,14 @@ const gateway = createGateway({
|
|
|
34
36
|
adapter: "anthropic",
|
|
35
37
|
models: [
|
|
36
38
|
{
|
|
37
|
-
id: "claude-sonnet-
|
|
39
|
+
id: "claude-sonnet-5-5",
|
|
38
40
|
provider: "anthropic",
|
|
39
|
-
displayName: "Claude Sonnet
|
|
40
|
-
contextWindow:
|
|
41
|
-
maxOutputTokens:
|
|
41
|
+
displayName: "Claude Sonnet 5.5",
|
|
42
|
+
contextWindow: 1_000_000,
|
|
43
|
+
maxOutputTokens: 128_000,
|
|
42
44
|
tokenizer: "anthropic",
|
|
43
|
-
inputModalities: ["text", "image"],
|
|
44
|
-
reasoning: { efforts: ["low", "medium", "high"
|
|
45
|
+
inputModalities: ["text", "image", "pdf"],
|
|
46
|
+
reasoning: { efforts: ["low", "medium", "high", "max"] },
|
|
45
47
|
promptCache: { kind: "explicit", shortTtlSeconds: 300, longTtlSeconds: 3600 },
|
|
46
48
|
},
|
|
47
49
|
],
|
|
@@ -52,7 +54,7 @@ const gateway = createGateway({
|
|
|
52
54
|
const apiKey = "key-for-this-tenant"; // read it wherever your product keeps keys
|
|
53
55
|
|
|
54
56
|
for await (const event of gateway.stream({
|
|
55
|
-
model: "claude-sonnet-
|
|
57
|
+
model: "claude-sonnet-5-5",
|
|
56
58
|
messages: [user("Say hello.")],
|
|
57
59
|
promptCacheRetention: "none",
|
|
58
60
|
auth: { anthropic: { apiKey } },
|
|
@@ -36,8 +36,7 @@
|
|
|
36
36
|
* landing beside a tool result. Marks go on wire user messages, whose last block
|
|
37
37
|
* always accepts `cache_control`. Marks move as the tail
|
|
38
38
|
* advances; providers hash content, not the marker, so that never invalidates
|
|
39
|
-
* a cached prefix. A 1h write costs 2x input tokens (5m: 1.25x)
|
|
40
|
-
* registry's `cacheWriteUsdPerMillionTokens` pinned rates.
|
|
39
|
+
* a cached prefix. A 1h write costs 2x input tokens (5m: 1.25x).
|
|
41
40
|
*/
|
|
42
41
|
import type Anthropic from "@anthropic-ai/sdk";
|
|
43
42
|
import type { GenerateRequest, ModelInfo, ResolvedCall } from "../../domain/index.js";
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* when building the final result, so persisted content follows the order Claude
|
|
11
11
|
* streamed instead of grouping all reasoning, then all text, then all tools.
|
|
12
12
|
*
|
|
13
|
-
* Provider-protocol
|
|
13
|
+
* Provider-protocol facts this file relies on (checked against the `@anthropic-ai/sdk` types and docs):
|
|
14
14
|
* - Anthropic's documented stream flow is strictly serial per block:
|
|
15
15
|
* `message_start` → (content_block_start → delta(s) → content_block_stop)*
|
|
16
16
|
* → `message_delta` → `message_stop`. Each block's `index` corresponds to
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* `input_json_delta` (partial JSON for tool input — must be concatenated
|
|
24
24
|
* before parsing).
|
|
25
25
|
* - StopReason mismatch: current Anthropic docs list
|
|
26
|
-
* `model_context_window_exceeded`, but the
|
|
26
|
+
* `model_context_window_exceeded`, but the SDK's
|
|
27
27
|
* `StopReason` union is `'end_turn' | 'max_tokens' | 'stop_sequence' |
|
|
28
28
|
* 'tool_use' | 'pause_turn' | 'refusal'` — that value is not present.
|
|
29
29
|
*/
|
|
@@ -53,9 +53,9 @@ interface ReasoningBlockEntry {
|
|
|
53
53
|
}
|
|
54
54
|
export declare function createStreamAccumulator(model: string, provider: string, longCacheWrites: boolean): StreamAccumulator;
|
|
55
55
|
/**
|
|
56
|
-
* Map Anthropic stop reasons to
|
|
56
|
+
* Map Anthropic stop reasons to the canonical FinishReason.
|
|
57
57
|
*
|
|
58
|
-
* The
|
|
58
|
+
* The SDK's StopReason union is:
|
|
59
59
|
* 'end_turn' | 'max_tokens' | 'stop_sequence' | 'tool_use' | 'pause_turn' | 'refusal'
|
|
60
60
|
* Current Anthropic docs additionally list `model_context_window_exceeded`,
|
|
61
61
|
* but that value is not in the SDK union; accept the wire string at this boundary.
|
|
@@ -18,15 +18,15 @@ export function createStreamAccumulator(model, provider, longCacheWrites) {
|
|
|
18
18
|
}
|
|
19
19
|
// ── Finish reason mapping ─────────────────────────────────────────
|
|
20
20
|
/**
|
|
21
|
-
* Map Anthropic stop reasons to
|
|
21
|
+
* Map Anthropic stop reasons to the canonical FinishReason.
|
|
22
22
|
*
|
|
23
|
-
* The
|
|
23
|
+
* The SDK's StopReason union is:
|
|
24
24
|
* 'end_turn' | 'max_tokens' | 'stop_sequence' | 'tool_use' | 'pause_turn' | 'refusal'
|
|
25
25
|
* Current Anthropic docs additionally list `model_context_window_exceeded`,
|
|
26
26
|
* but that value is not in the SDK union; accept the wire string at this boundary.
|
|
27
27
|
*/
|
|
28
28
|
export function mapStopReason(reason) {
|
|
29
|
-
// Anthropic stop reasons mostly line up with
|
|
29
|
+
// Anthropic stop reasons mostly line up with the canonical FinishReason. `refusal`
|
|
30
30
|
// and `pause_turn` have no canonical equivalent today, so they are
|
|
31
31
|
// represented as an error finish and end_turn respectively.
|
|
32
32
|
switch (reason) {
|
|
@@ -47,7 +47,7 @@ export function mapStopReason(reason) {
|
|
|
47
47
|
}
|
|
48
48
|
// ── Usage mapping ─────────────────────────────────────────────────
|
|
49
49
|
export function mapUsage(usage, longCacheWrites, fallback) {
|
|
50
|
-
// Anthropic's input_tokens excludes cache reads and cache creation.
|
|
50
|
+
// Anthropic's input_tokens excludes cache reads and cache creation. the canonical
|
|
51
51
|
// canonical inputTokens is inclusive, so the adapter owns the additive
|
|
52
52
|
// conversion before usage reaches persistence, displays, or pricing.
|
|
53
53
|
const fallbackCacheRead = fallback?.cacheReadTokens ?? 0;
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* - Thought signatures: the model's `thoughtSignature` on a text, thought or function-call part
|
|
14
14
|
* is kept in `providerOptions.gemini.signature` and sent back on the same part, only to the
|
|
15
15
|
* provider/model pair that issued it. Gemini 3 rejects a replayed function-call step whose
|
|
16
|
-
* first call carries no signature (history from another model,
|
|
16
|
+
* first call carries no signature (history from another model, or one with its thinking removed), so that
|
|
17
17
|
* call gets Google's documented validator-bypass value.
|
|
18
18
|
* - Prompt caching is implicit: no mark and no TTL is sent.
|
|
19
19
|
*/
|
|
@@ -18,9 +18,9 @@
|
|
|
18
18
|
* guide).
|
|
19
19
|
* - Prompt caching is automatic for prompts >=1024 tokens and needs no
|
|
20
20
|
* explicit breakpoints, so this mapper only forwards `GenerateRequest.
|
|
21
|
-
* promptCacheKey` as `prompt_cache_key` (a stable per-
|
|
21
|
+
* promptCacheKey` as `prompt_cache_key` (a stable per-conversation routing hint
|
|
22
22
|
* that improves cache-affinity — OpenAI's own docs recommend one per
|
|
23
|
-
*
|
|
23
|
+
* conversation). A long retention request adds
|
|
24
24
|
* `prompt_cache_options: { ttl: "12h" }`; the short one relies on the 30m
|
|
25
25
|
* default.
|
|
26
26
|
*/
|
|
@@ -222,7 +222,7 @@ export function toOpenAIResponsesParams(request, model, call) {
|
|
|
222
222
|
...(mapTools(request.tools) ? { tools: mapTools(request.tools) } : {}),
|
|
223
223
|
reasoning: mapReasoning(call.reasoning),
|
|
224
224
|
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
|
|
225
|
-
// The 30m default needs no parameter.
|
|
225
|
+
// The 30m default needs no parameter.
|
|
226
226
|
...(cacheTtl ? { prompt_cache_options: { ttl: cacheTtl } } : {}),
|
|
227
227
|
};
|
|
228
228
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* canonical StreamEvents and a final GenerateResult. The Responses protocol
|
|
4
4
|
* names the same logical output item by different keys in different event
|
|
5
5
|
* families, so this file owns the reconciliation from provider item IDs,
|
|
6
|
-
* call IDs, and output positions into
|
|
6
|
+
* call IDs, and output positions into the canonical ContentPart,
|
|
7
7
|
* ToolCall, Usage, and FinishReason vocabulary.
|
|
8
8
|
*
|
|
9
9
|
* Key decision: provider `output_index` is treated as the source-order spine.
|
|
@@ -11,18 +11,17 @@
|
|
|
11
11
|
* position and sorted only when building the final result, so persisted content
|
|
12
12
|
* follows the provider's streamed order instead of being grouped by part type.
|
|
13
13
|
*
|
|
14
|
-
* Provider-protocol
|
|
14
|
+
* Provider-protocol facts this file relies on (checked against the `openai` SDK types and docs):
|
|
15
15
|
* - `output_index` is the position of the output item in `response.output[]`.
|
|
16
16
|
* The final array order is model-dependent and the docs do NOT explicitly
|
|
17
|
-
* guarantee non-interleaving during streaming, so
|
|
17
|
+
* guarantee non-interleaving during streaming, so this file buffers by index
|
|
18
18
|
* and sorts at assembly time rather than trusting arrival order.
|
|
19
19
|
* - `response.function_call_arguments.delta` events carry `item_id` and
|
|
20
20
|
* `output_index` but not `call_id`. This is why
|
|
21
21
|
* argument deltas are buffered in `pendingDeltas` until the durable `call_id`
|
|
22
22
|
* arrives via `response.output_item.added` or a later event.
|
|
23
|
-
* -
|
|
24
|
-
*
|
|
25
|
-
* identity so the external Meridian contract does not change.
|
|
23
|
+
* - Reasoning text events are named `response.reasoning_text.*`; the accumulator is keyed by
|
|
24
|
+
* output item identity, not by event name.
|
|
26
25
|
*/
|
|
27
26
|
import type OpenAI from "openai";
|
|
28
27
|
import type { ContentPart, FinishReason, GenerateResult, StreamEvent } from "../../domain/index.js";
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* canonical StreamEvents and a final GenerateResult. The Responses protocol
|
|
4
4
|
* names the same logical output item by different keys in different event
|
|
5
5
|
* families, so this file owns the reconciliation from provider item IDs,
|
|
6
|
-
* call IDs, and output positions into
|
|
6
|
+
* call IDs, and output positions into the canonical ContentPart,
|
|
7
7
|
* ToolCall, Usage, and FinishReason vocabulary.
|
|
8
8
|
*
|
|
9
9
|
* Key decision: provider `output_index` is treated as the source-order spine.
|
|
@@ -11,18 +11,17 @@
|
|
|
11
11
|
* position and sorted only when building the final result, so persisted content
|
|
12
12
|
* follows the provider's streamed order instead of being grouped by part type.
|
|
13
13
|
*
|
|
14
|
-
* Provider-protocol
|
|
14
|
+
* Provider-protocol facts this file relies on (checked against the `openai` SDK types and docs):
|
|
15
15
|
* - `output_index` is the position of the output item in `response.output[]`.
|
|
16
16
|
* The final array order is model-dependent and the docs do NOT explicitly
|
|
17
|
-
* guarantee non-interleaving during streaming, so
|
|
17
|
+
* guarantee non-interleaving during streaming, so this file buffers by index
|
|
18
18
|
* and sorts at assembly time rather than trusting arrival order.
|
|
19
19
|
* - `response.function_call_arguments.delta` events carry `item_id` and
|
|
20
20
|
* `output_index` but not `call_id`. This is why
|
|
21
21
|
* argument deltas are buffered in `pendingDeltas` until the durable `call_id`
|
|
22
22
|
* arrives via `response.output_item.added` or a later event.
|
|
23
|
-
* -
|
|
24
|
-
*
|
|
25
|
-
* identity so the external Meridian contract does not change.
|
|
23
|
+
* - Reasoning text events are named `response.reasoning_text.*`; the accumulator is keyed by
|
|
24
|
+
* output item identity, not by event name.
|
|
26
25
|
*/
|
|
27
26
|
import { parseToolCallArguments } from "../../helpers/parse-tool-arguments.js";
|
|
28
27
|
import { assertValidUsage } from "../../shapes/index.js";
|
|
@@ -86,7 +85,7 @@ function rememberToolCallEntry(acc, entry) {
|
|
|
86
85
|
acc.toolCallsByOutputIndex.set(entry.outputIndex, entry);
|
|
87
86
|
}
|
|
88
87
|
function findToolCallEntry(acc, params) {
|
|
89
|
-
// Lookup order follows semantic strength: callId is
|
|
88
|
+
// Lookup order follows semantic strength: callId is the gateway's durable tool
|
|
90
89
|
// identity, itemId is the provider output identity, and output_index is the
|
|
91
90
|
// ordering fallback used by deltas before either ID has appeared.
|
|
92
91
|
return ((params.callId ? acc.toolCalls.get(params.callId) : undefined) ??
|
|
@@ -99,13 +98,13 @@ export function mapUsage(usage, longCacheWrites) {
|
|
|
99
98
|
// hits and reasoning tokens live in provider-specific detail objects
|
|
100
99
|
// (`input_tokens_details.cached_tokens`, `output_tokens_details.reasoning_tokens`).
|
|
101
100
|
// Both fields are confirmed in the OpenAI reasoning guide and the installed
|
|
102
|
-
// SDK's ResponseUsage shape.
|
|
101
|
+
// SDK's ResponseUsage shape. Optional Usage fields are set only when the
|
|
103
102
|
// provider reports a positive value.
|
|
104
103
|
//
|
|
105
104
|
// GPT-5.6+ also bills cache writes: `cache_write_tokens` is their total, split by
|
|
106
105
|
// lifetime in `cache_write_tokens_details` (`ttl_30m_tokens` + `ttl_12h_tokens`), and
|
|
107
|
-
// `cached_tokens` excludes them. The installed SDK
|
|
108
|
-
//
|
|
106
|
+
// `cached_tokens` excludes them. The installed SDK types `cache_write_tokens` but not
|
|
107
|
+
// the `cache_write_tokens_details` split, so that is read through a local shape.
|
|
109
108
|
const details = usage.input_tokens_details;
|
|
110
109
|
const result = {
|
|
111
110
|
inputTokens: usage.input_tokens,
|
|
@@ -405,7 +404,7 @@ export function* eventsFromResponseStreamEvent(event, acc) {
|
|
|
405
404
|
break;
|
|
406
405
|
default:
|
|
407
406
|
// Other hosted-tool/provider events (web_search, code_interpreter, etc.)
|
|
408
|
-
// have no
|
|
407
|
+
// have no canonical ContentPart/StreamEvent mapping yet. Ignore
|
|
409
408
|
// rather than inventing a lossy custom contract at this boundary.
|
|
410
409
|
break;
|
|
411
410
|
}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* ChatCompletionChunk.Choice.Delta.ToolCall SDK shape) so interleaved argument
|
|
13
13
|
* fragments rebuild the correct ToolCall entries.
|
|
14
14
|
*
|
|
15
|
-
* Provider-protocol
|
|
15
|
+
* Provider-protocol facts this file relies on (checked against the `openai` SDK types and docs):
|
|
16
16
|
* - Chat Completions `finish_reason` on streaming chunks is `'stop' | 'length'
|
|
17
17
|
* | 'tool_calls' | 'content_filter' | 'function_call' | null` (SDK-confirmed).
|
|
18
18
|
* `null` means the stream is still in progress; only the final content chunk
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* ChatCompletionChunk.Choice.Delta.ToolCall SDK shape) so interleaved argument
|
|
13
13
|
* fragments rebuild the correct ToolCall entries.
|
|
14
14
|
*
|
|
15
|
-
* Provider-protocol
|
|
15
|
+
* Provider-protocol facts this file relies on (checked against the `openai` SDK types and docs):
|
|
16
16
|
* - Chat Completions `finish_reason` on streaming chunks is `'stop' | 'length'
|
|
17
17
|
* | 'tool_calls' | 'content_filter' | 'function_call' | null` (SDK-confirmed).
|
|
18
18
|
* `null` means the stream is still in progress; only the final content chunk
|
|
@@ -119,7 +119,7 @@ export function* eventsFromOpenAIChunk(chunk, acc) {
|
|
|
119
119
|
// OpenAI-compatible providers deliver cumulative usage on the final chunk
|
|
120
120
|
// (or a trailing usage-only chunk) when stream_options.include_usage is
|
|
121
121
|
// set. Map prompt/completion tokens and optional reasoning tokens into
|
|
122
|
-
//
|
|
122
|
+
// canonical Usage. The SDK docs warn: interrupted streams may never deliver
|
|
123
123
|
// this chunk.
|
|
124
124
|
const reasoningTokens = chunk.usage.completion_tokens_details?.reasoning_tokens;
|
|
125
125
|
const cacheReadTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What a model can take as media and how each wire format carries it.
|
|
3
|
-
*
|
|
2
|
+
* What a model can take as media and how each wire format carries it. The provider facts behind it
|
|
3
|
+
* are collected at
|
|
4
|
+
* https://github.com/haowjy/meridian-flow-docs/blob/main/kb/wiki/agents/files-the-model-sees.md
|
|
4
5
|
*
|
|
5
6
|
* Two levels, because they change independently: the model says which kinds it takes
|
|
6
7
|
* (`inputModalities`, copied from models.dev) and how large a file its provider fetches by link
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* The gateway's canonical types (the public API): the provider-neutral request,
|
|
3
3
|
* result, stream-event, message, tool, and capability shapes. The vocabulary
|
|
4
4
|
* every adapter maps its provider SDK to and from; depends on nothing provider-specific.
|
|
5
5
|
*
|
|
6
|
-
* Why this boundary exists (see
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* these types at the adapter boundary.
|
|
6
|
+
* Why this boundary exists (see the gateway's AGENTS.md): provider-specific types stay inside
|
|
7
|
+
* adapter directories so a host never imports `openai` or `@anthropic-ai/sdk` to use the gateway.
|
|
8
|
+
* Every adapter translates to and from these types at the adapter boundary.
|
|
10
9
|
*
|
|
11
10
|
* Key design invariants:
|
|
12
11
|
* - JSON-natural: every field must survive `JSON.parse(JSON.stringify(x))`.
|
|
@@ -17,7 +16,7 @@
|
|
|
17
16
|
* `tool_calls[].index`), used by adapters to reconstruct source order.
|
|
18
17
|
* - `GenerateRequest.promptCacheKey` is canonical prompt-cache *intent*, not
|
|
19
18
|
* provider syntax, so it stays top-level (never nested in `providerOptions`):
|
|
20
|
-
* the
|
|
19
|
+
* the caller knows a conversation's stable key but has no opinion on TTL or wire
|
|
21
20
|
* shape. Each adapter decides whether to forward it (a `prompt_cache_key`
|
|
22
21
|
* routing hint, or nothing). Cache breakpoints are not part of the request:
|
|
23
22
|
* the Anthropic adapter places them on the wire blocks itself.
|
|
@@ -28,20 +27,21 @@ export type ProviderOptions = Record<string, Record<string, unknown>>;
|
|
|
28
27
|
export type TokenizerFamily = "anthropic" | "o200k" | "gemini" | "deepseek";
|
|
29
28
|
/** What a model reads as input, in the models.dev vocabulary. */
|
|
30
29
|
export type InputModality = "text" | "image" | "pdf" | "audio" | "video";
|
|
31
|
-
/**
|
|
30
|
+
/** Common provider cache lifetimes in seconds (Anthropic's 5 minutes and 1 hour), for a host to put in a `PromptCacheDescriptor` or to compute a buffer from. */
|
|
32
31
|
export declare const DEFAULT_SUBAGENT_PROMPT_CACHE_TTL: number;
|
|
33
32
|
export declare const DEFAULT_PRIMARY_PROMPT_CACHE_TTL: number;
|
|
34
|
-
/**
|
|
33
|
+
/** A model's provider cache behavior and its best-known retention window. */
|
|
35
34
|
export interface PromptCacheDescriptor {
|
|
36
35
|
kind: "explicit" | "automatic";
|
|
37
|
-
/** Lifetime in seconds of a short-retention request
|
|
36
|
+
/** Lifetime in seconds of a short-retention request; null means the provider documents none. */
|
|
38
37
|
shortTtlSeconds: number | null;
|
|
39
|
-
/** Lifetime of a long-retention request
|
|
38
|
+
/** Lifetime of a long-retention request; equals `shortTtlSeconds` when the provider offers one lifetime. */
|
|
40
39
|
longTtlSeconds: number | null;
|
|
41
40
|
}
|
|
42
41
|
/**
|
|
43
|
-
* Which cache lifetime a request wants:
|
|
44
|
-
*
|
|
42
|
+
* Which cache lifetime a request wants: `long` for a conversation the caller keeps extending,
|
|
43
|
+
* `short` for a call re-read only within minutes, and `none` for a one-shot call nothing reads
|
|
44
|
+
* again (a summary): no cache mark is sent.
|
|
45
45
|
*/
|
|
46
46
|
export type PromptCacheRetention = "none" | "short" | "long";
|
|
47
47
|
/** The TTL a request with this retention gets from a model, for adapters and warm/cold prediction. */
|
|
@@ -50,7 +50,7 @@ export declare const REASONING_EFFORTS: readonly ["low", "medium", "high", "max"
|
|
|
50
50
|
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
|
|
51
51
|
/**
|
|
52
52
|
* A model's reasoning controls, copied from its models.dev `reasoning_options`
|
|
53
|
-
* (`xhigh
|
|
53
|
+
* (`xhigh` is not in `REASONING_EFFORTS`, so it is omitted from `efforts`).
|
|
54
54
|
*/
|
|
55
55
|
export interface ReasoningSupport {
|
|
56
56
|
/** Effort levels the model accepts. */
|
|
@@ -176,9 +176,8 @@ export interface Message {
|
|
|
176
176
|
content: ContentPart[];
|
|
177
177
|
}
|
|
178
178
|
/**
|
|
179
|
-
* A tool
|
|
180
|
-
*
|
|
181
|
-
* adapter translates them into the provider's tool schema format.
|
|
179
|
+
* A tool the caller executes, with a JSON Schema input signature. The request carries these as
|
|
180
|
+
* the tool definitions, and the adapter translates them into the provider's tool schema format.
|
|
182
181
|
*/
|
|
183
182
|
export interface Tool {
|
|
184
183
|
type: "function";
|
|
@@ -206,14 +205,14 @@ export interface GenerateRequest {
|
|
|
206
205
|
};
|
|
207
206
|
signal?: AbortSignal;
|
|
208
207
|
/**
|
|
209
|
-
* Stable
|
|
208
|
+
* Stable cache-routing key for one conversation (opaque, not a secret): unlike
|
|
210
209
|
* `correlation` below, adapters MAY map this into the provider request —
|
|
211
210
|
* e.g. OpenAI Responses `prompt_cache_key`, which improves cache-affinity
|
|
212
211
|
* routing for automatic caching. Anthropic has no equivalent concept and
|
|
213
212
|
* ignores it.
|
|
214
213
|
*/
|
|
215
214
|
promptCacheKey?: string;
|
|
216
|
-
/** Cache lifetime wanted;
|
|
215
|
+
/** Cache lifetime wanted; keep it the same for every call of one conversation (`none` for one-shot calls nothing re-reads) so the cached prefix never changes. */
|
|
217
216
|
promptCacheRetention: PromptCacheRetention;
|
|
218
217
|
/**
|
|
219
218
|
* Credentials for this call, keyed by provider id. The routed provider's entry replaces that
|
|
@@ -242,7 +241,7 @@ export interface GenerateRequest {
|
|
|
242
241
|
export type ErrorCode = "network_error" | "rate_limited" | "server_error" | "auth_error" | "invalid_request" | "content_filtered" | "context_overflow" | "provider_error";
|
|
243
242
|
/**
|
|
244
243
|
* Stream event — the canonical event set that every adapter emits and the
|
|
245
|
-
*
|
|
244
|
+
* caller consumes. Each event is a complete value of the gateway's own types; the
|
|
246
245
|
* generator produces them one-at-a-time so subscribers receive them as they
|
|
247
246
|
* arrive from the provider without waiting for the full response.
|
|
248
247
|
*
|
|
@@ -323,7 +322,7 @@ export interface ToolCall {
|
|
|
323
322
|
* - Anthropic StopReason → FinishReason: `end_turn`→`end_turn`, `tool_use`→`tool_use`,
|
|
324
323
|
* `max_tokens`→`max_tokens`, `stop_sequence`→`stop_sequence`, `refusal`→`error`.
|
|
325
324
|
* NOTE: Anthropic docs additionally list `model_context_window_exceeded` but the
|
|
326
|
-
*
|
|
325
|
+
* `StopReason` union in `@anthropic-ai/sdk` 0.106 does not include it; if that value arrives at runtime
|
|
327
326
|
* the adapter maps the wire string to the canonical `context_overflow` error.
|
|
328
327
|
* - OpenAI Responses status: `completed`→`end_turn`, `failed`/`cancelled`→`error`,
|
|
329
328
|
* `incomplete` maps to `max_tokens` or `error` depending on `incomplete_details.reason`.
|
|
@@ -331,8 +330,8 @@ export interface ToolCall {
|
|
|
331
330
|
* `length`→`max_tokens`, `content_filter`→`error`.
|
|
332
331
|
*
|
|
333
332
|
* Precedence: `tool_use` wins over any terminal status when tool calls were
|
|
334
|
-
* actually assembled (the
|
|
335
|
-
*
|
|
333
|
+
* actually assembled (the caller must execute the tools before treating the
|
|
334
|
+
* response as finished).
|
|
336
335
|
*/
|
|
337
336
|
export type FinishReason = "end_turn" | "tool_use" | "max_tokens" | "stop_sequence" | "error";
|
|
338
337
|
/**
|
|
@@ -343,8 +342,7 @@ export type FinishReason = "end_turn" | "tool_use" | "max_tokens" | "stop_sequen
|
|
|
343
342
|
* OpenAI Responses output_index), not grouped by part type. This preserves the
|
|
344
343
|
* model's intended content sequence: reasoning → text → tool_use interleaving.
|
|
345
344
|
*
|
|
346
|
-
* `toolCalls` is a flat list of every parsed ToolCall in the response
|
|
347
|
-
* orchestrator uses this to dispatch tool execution.
|
|
345
|
+
* `toolCalls` is a flat list of every parsed ToolCall in the response, for the caller to run.
|
|
348
346
|
*/
|
|
349
347
|
export interface GenerateResult {
|
|
350
348
|
content: ContentPart[];
|
|
@@ -447,7 +445,7 @@ export interface GatewayConfig {
|
|
|
447
445
|
initialDelayMs: number;
|
|
448
446
|
maxDelayMs: number;
|
|
449
447
|
};
|
|
450
|
-
/**
|
|
448
|
+
/** Warnings while the gateway is built (e.g. duplicate model IDs skipped). */
|
|
451
449
|
onWarning?: (span: TraceSpan) => void;
|
|
452
450
|
}
|
|
453
451
|
/** Minimal trace span passed to `GatewayConfig.onWarning`. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/** Common provider cache lifetimes in seconds (Anthropic's 5 minutes and 1 hour), for a host to put in a `PromptCacheDescriptor` or to compute a buffer from. */
|
|
2
2
|
export const DEFAULT_SUBAGENT_PROMPT_CACHE_TTL = 5 * 60;
|
|
3
3
|
export const DEFAULT_PRIMARY_PROMPT_CACHE_TTL = 60 * 60;
|
|
4
4
|
/** The TTL a request with this retention gets from a model, for adapters and warm/cold prediction. */
|
|
@@ -4,8 +4,7 @@
|
|
|
4
4
|
* domain message types.
|
|
5
5
|
*
|
|
6
6
|
* These are simple constructors that produce well-formed Message/ContentPart
|
|
7
|
-
* values without boilerplate.
|
|
8
|
-
* tests.
|
|
7
|
+
* values without boilerplate. Hosts and tests use them to build requests.
|
|
9
8
|
*/
|
|
10
9
|
import type { ContentPart, MediaPart, Message, ProviderOptions, TextPart } from "../domain/index.js";
|
|
11
10
|
export declare function user(text: string): Message;
|
|
@@ -4,8 +4,7 @@
|
|
|
4
4
|
* domain message types.
|
|
5
5
|
*
|
|
6
6
|
* These are simple constructors that produce well-formed Message/ContentPart
|
|
7
|
-
* values without boilerplate.
|
|
8
|
-
* tests.
|
|
7
|
+
* values without boilerplate. Hosts and tests use them to build requests.
|
|
9
8
|
*/
|
|
10
9
|
export function user(text) {
|
|
11
10
|
return { role: "user", content: [{ type: "text", text }] };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* Wire contract for scripting the mock model: queued replies that answer the next model calls.
|
|
3
|
+
* It is its own entry so a host's tooling that writes scripts and the mock server that reads them
|
|
4
|
+
* validate against the same schema without importing the server.
|
|
4
5
|
*/
|
|
5
6
|
import { z } from "zod";
|
|
6
7
|
/** Repeated defaults cannot execute tools or consume queue-step budgets. */
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* Wire contract for scripting the mock model: queued replies that answer the next model calls.
|
|
3
|
+
* It is its own entry so a host's tooling that writes scripts and the mock server that reads them
|
|
4
|
+
* validate against the same schema without importing the server.
|
|
4
5
|
*/
|
|
5
6
|
import { z } from "zod";
|
|
6
7
|
const toolCallSchema = z
|
|
@@ -96,7 +97,7 @@ export const mockModelStepSchema = z
|
|
|
96
97
|
export const mockModelScriptSchema = z.strictObject({
|
|
97
98
|
/**
|
|
98
99
|
* Only model calls whose latest user message contains this text consume the
|
|
99
|
-
* script, so concurrent
|
|
100
|
+
* script, so concurrent conversations do not steal each other's steps. Omitted: any call.
|
|
100
101
|
*/
|
|
101
102
|
match: z.string().min(1).optional(),
|
|
102
103
|
steps: z.array(mockModelStepSchema).min(1).max(100),
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Gateway port: the provider-neutral interface (stream/listModels)
|
|
3
|
-
*
|
|
2
|
+
* Gateway port: the provider-neutral interface (stream/listModels) a host depends on instead of
|
|
3
|
+
* any concrete provider.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* They never know which provider is behind the call.
|
|
5
|
+
* A caller runs `gateway.stream(request)` to get an AsyncIterable<StreamEvent> and never knows
|
|
6
|
+
* which provider is behind the call.
|
|
8
7
|
*
|
|
9
|
-
* Why a port: adapters are
|
|
10
|
-
*
|
|
11
|
-
* and route handlers never import concrete adapter factories.
|
|
8
|
+
* Why a port: adapters are chosen by config (see create-gateway.ts). The host builds the gateway
|
|
9
|
+
* once from GatewayConfig and never imports a concrete adapter factory.
|
|
12
10
|
*/
|
|
13
11
|
import type { GenerateRequest, ModelInfo, StreamEvent } from "../domain/index.js";
|
|
14
12
|
export interface Gateway {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* to canonical StreamEvents.
|
|
7
7
|
*
|
|
8
8
|
* Why one port: every provider has a different SDK contract, but the
|
|
9
|
-
*
|
|
9
|
+
* gateway's edge is uniform. Adding a new provider means implementing
|
|
10
10
|
* this one interface and registering it in createAdapter.
|
|
11
11
|
*/
|
|
12
12
|
import type { GenerateRequest, ModelInfo, ResolvedCall, StreamEvent } from "../domain/index.js";
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The provider's own status and message, kept as
|
|
3
|
-
*
|
|
4
|
-
* never shows the message.
|
|
2
|
+
* The provider's own status and message, kept as debug evidence for the host's operators. Never
|
|
3
|
+
* put it in event payloads or model context, and never show the message to an end user.
|
|
5
4
|
*/
|
|
6
5
|
export type ProviderErrorResponse = {
|
|
7
6
|
/** HTTP status; null when the provider reported the failure inside the stream. */
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* this plain DTO before any runtime or billing logic consumes it.
|
|
2
|
+
* The JSON-natural token usage contract the gateway returns, which a host can meter and price.
|
|
3
|
+
* Provider adapters normalize their native token accounting into this plain DTO, so a host
|
|
4
|
+
* never reads a provider's own counters.
|
|
6
5
|
*/
|
|
7
6
|
/**
|
|
8
7
|
* Normalized token usage across providers.
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* this plain DTO before any runtime or billing logic consumes it.
|
|
2
|
+
* The JSON-natural token usage contract the gateway returns, which a host can meter and price.
|
|
3
|
+
* Provider adapters normalize their native token accounting into this plain DTO, so a host
|
|
4
|
+
* never reads a provider's own counters.
|
|
6
5
|
*/
|
|
7
6
|
/** Throws when a Usage value violates the canonical inclusive-total contract. */
|
|
8
7
|
export function assertValidUsage(usage) {
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* one that has already streamed visible text or opened a tool call is not.
|
|
13
13
|
*/
|
|
14
14
|
import type { StreamEvent } from "./domain/index.js";
|
|
15
|
-
/** Content deltas
|
|
15
|
+
/** Content deltas a consumer can show to the end user. */
|
|
16
16
|
export declare function isFirstTokenEvent(event: StreamEvent): boolean;
|
|
17
17
|
/**
|
|
18
18
|
* Output the caller may have acted on: visible text, a tool call, or a terminal success. Reasoning and usage are excluded so an
|
package/package.json
CHANGED
|
@@ -1,12 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@meridian-flow/runtime",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "What an AI product needs to run models: a provider-neutral streaming model gateway (Anthropic, OpenAI, Gemini, OpenAI-compatible) with retry, deadlines and usage normalization.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"llm",
|
|
7
|
+
"ai",
|
|
8
|
+
"model-gateway",
|
|
9
|
+
"anthropic",
|
|
10
|
+
"openai",
|
|
11
|
+
"gemini",
|
|
12
|
+
"streaming",
|
|
13
|
+
"workload-identity-federation"
|
|
14
|
+
],
|
|
5
15
|
"license": "Apache-2.0",
|
|
6
16
|
"repository": {
|
|
7
17
|
"type": "git",
|
|
8
18
|
"url": "git+https://github.com/haowjy/meridian-runtime.git"
|
|
9
19
|
},
|
|
20
|
+
"homepage": "https://github.com/haowjy/meridian-runtime#readme",
|
|
21
|
+
"bugs": {
|
|
22
|
+
"url": "https://github.com/haowjy/meridian-runtime/issues"
|
|
23
|
+
},
|
|
10
24
|
"type": "module",
|
|
11
25
|
"sideEffects": false,
|
|
12
26
|
"engines": {
|