@cldmv/llm-translator 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/api/convert/anthropic/messages/to/bedrock/converse/request.mjs +2 -0
- package/dist/api/convert/anthropic/messages/to/openai/responses/request.mjs +2 -0
- package/dist/api/convert/bedrock/converse/to/anthropic/messages/response.mjs +2 -0
- package/dist/api/convert/bedrock/converse/to/anthropic/messages/stream.mjs +2 -0
- package/dist/api/convert/bedrock/converse/to/openai/chat/response.mjs +2 -0
- package/dist/api/convert/bedrock/converse/to/openai/chat/stream.mjs +2 -0
- package/dist/api/convert/openai/chat/to/bedrock/converse/request.mjs +2 -0
- package/dist/api/convert/openai/chat/to/openai/responses/request.mjs +2 -0
- package/dist/api/convert/openai/chat/to/openai/responses/response.mjs +2 -0
- package/dist/api/convert/openai/chat/to/openai/responses/stream.mjs +2 -0
- package/dist/api/convert/openai/responses/to/anthropic/messages/response.mjs +2 -0
- package/dist/api/convert/openai/responses/to/anthropic/messages/stream.mjs +2 -0
- package/dist/api/convert/openai/responses/to/openai/chat/request.mjs +6 -0
- package/dist/api/convert/openai/responses/to/openai/chat/response.mjs +2 -0
- package/dist/api/convert/openai/responses/to/openai/chat/stream.mjs +2 -0
- package/dist/api/detect.mjs +1 -1
- package/dist/api/driver/anthropic/info.mjs +1 -1
- package/dist/api/driver/azure/info.mjs +1 -1
- package/dist/api/driver/bedrock/detect.mjs +2 -0
- package/dist/api/driver/bedrock/info.mjs +2 -0
- package/dist/api/driver/bedrock/model/list.mjs +2 -0
- package/dist/api/driver/bedrock/path.mjs +2 -0
- package/dist/api/driver/bedrock/request.mjs +2 -0
- package/dist/api/driver/bedrock/sign.mjs +2 -0
- package/dist/api/driver/compatible/info.mjs +1 -1
- package/dist/api/driver/deepseek/info.mjs +1 -1
- package/dist/api/driver/llamacpp/info.mjs +1 -1
- package/dist/api/driver/lmstudio/info.mjs +1 -1
- package/dist/api/driver/localai/info.mjs +1 -1
- package/dist/api/driver/mistral/info.mjs +1 -1
- package/dist/api/driver/ollama/info.mjs +1 -1
- package/dist/api/driver/ollama/request.mjs +1 -1
- package/dist/api/driver/openai/info.mjs +1 -1
- package/dist/api/driver/openrouter/info.mjs +1 -1
- package/dist/api/driver/sglang/info.mjs +1 -1
- package/dist/api/driver/tgi/info.mjs +1 -1
- package/dist/api/driver/vertex/auth.mjs +2 -0
- package/dist/api/driver/vertex/detect.mjs +2 -0
- package/dist/api/driver/vertex/error.mjs +2 -0
- package/dist/api/driver/vertex/info.mjs +2 -0
- package/dist/api/driver/vertex/path.mjs +2 -0
- package/dist/api/driver/vertex/request.mjs +2 -0
- package/dist/api/driver/vertex/settings.mjs +2 -0
- package/dist/api/driver/vertex/token/count.mjs +2 -0
- package/dist/api/driver/vllm/info.mjs +1 -1
- package/dist/api/request.mjs +2 -2
- package/dist/api/response.mjs +1 -1
- package/dist/api/route.mjs +1 -1
- package/dist/api/send.mjs +1 -1
- package/dist/api/server/plugin.mjs +1 -1
- package/dist/api/status.mjs +1 -1
- package/dist/api/stream.mjs +1 -1
- package/dist/api/token/count.mjs +1 -1
- package/dist/index.mjs +1 -1
- package/dist/lib/bedrock.mjs +2 -0
- package/dist/lib/capture.mjs +1 -1
- package/dist/lib/config.mjs +1 -1
- package/dist/lib/converse.mjs +3 -0
- package/dist/lib/document-urls.mjs +2 -0
- package/dist/lib/documents.mjs +4 -4
- package/dist/lib/estimate.mjs +1 -1
- package/dist/lib/eventstream.mjs +2 -0
- package/dist/lib/fetch-url.mjs +2 -0
- package/dist/lib/google-auth.mjs +2 -0
- package/dist/lib/http.mjs +1 -1
- package/dist/lib/presets.mjs +1 -1
- package/dist/lib/protocol.mjs +2 -0
- package/dist/lib/responses-upstream.mjs +13 -0
- package/dist/lib/responses.mjs +2 -0
- package/dist/lib/route.mjs +1 -1
- package/dist/lib/routes.mjs +1 -1
- package/dist/lib/settings.mjs +1 -1
- package/dist/lib/sigv4.mjs +7 -0
- package/dist/lib/vertex.mjs +2 -0
- package/dist/lib/vllm.mjs +2 -2
- package/package.json +1 -1
- package/types/src/api/convert/anthropic/messages/to/bedrock/converse/request.d.mts +14 -0
- package/types/src/api/convert/anthropic/messages/to/openai/responses/request.d.mts +16 -0
- package/types/src/api/convert/bedrock/converse/to/anthropic/messages/response.d.mts +10 -0
- package/types/src/api/convert/bedrock/converse/to/anthropic/messages/stream.d.mts +14 -0
- package/types/src/api/convert/bedrock/converse/to/openai/chat/response.d.mts +10 -0
- package/types/src/api/convert/bedrock/converse/to/openai/chat/stream.d.mts +13 -0
- package/types/src/api/convert/openai/chat/to/bedrock/converse/request.d.mts +11 -0
- package/types/src/api/convert/openai/chat/to/openai/responses/request.d.mts +12 -0
- package/types/src/api/convert/openai/chat/to/openai/responses/response.d.mts +14 -0
- package/types/src/api/convert/openai/chat/to/openai/responses/stream.d.mts +18 -0
- package/types/src/api/convert/openai/responses/to/anthropic/messages/response.d.mts +8 -0
- package/types/src/api/convert/openai/responses/to/anthropic/messages/stream.d.mts +8 -0
- package/types/src/api/convert/openai/responses/to/openai/chat/request.d.mts +17 -0
- package/types/src/api/convert/openai/responses/to/openai/chat/response.d.mts +8 -0
- package/types/src/api/convert/openai/responses/to/openai/chat/stream.d.mts +10 -0
- package/types/src/api/driver/anthropic/info.d.mts +2 -1
- package/types/src/api/driver/azure/info.d.mts +2 -1
- package/types/src/api/driver/bedrock/detect.d.mts +9 -0
- package/types/src/api/driver/bedrock/info.d.mts +36 -0
- package/types/src/api/driver/bedrock/model/list.d.mts +11 -0
- package/types/src/api/driver/bedrock/path.d.mts +9 -0
- package/types/src/api/driver/bedrock/request.d.mts +9 -0
- package/types/src/api/driver/bedrock/sign.d.mts +9 -0
- package/types/src/api/driver/compatible/info.d.mts +2 -1
- package/types/src/api/driver/deepseek/info.d.mts +2 -1
- package/types/src/api/driver/fireworks/info.d.mts +2 -1
- package/types/src/api/driver/gemini/info.d.mts +2 -1
- package/types/src/api/driver/groq/info.d.mts +2 -1
- package/types/src/api/driver/llamacpp/info.d.mts +2 -1
- package/types/src/api/driver/lmstudio/info.d.mts +2 -1
- package/types/src/api/driver/localai/info.d.mts +2 -1
- package/types/src/api/driver/mistral/info.d.mts +2 -1
- package/types/src/api/driver/ollama/info.d.mts +2 -1
- package/types/src/api/driver/ollama/request.d.mts +1 -1
- package/types/src/api/driver/openai/info.d.mts +2 -1
- package/types/src/api/driver/openrouter/info.d.mts +2 -1
- package/types/src/api/driver/sglang/info.d.mts +2 -1
- package/types/src/api/driver/tgi/info.d.mts +2 -1
- package/types/src/api/driver/together/info.d.mts +2 -1
- package/types/src/api/driver/vertex/auth.d.mts +7 -0
- package/types/src/api/driver/vertex/detect.d.mts +8 -0
- package/types/src/api/driver/vertex/error.d.mts +15 -0
- package/types/src/api/driver/vertex/info.d.mts +16 -0
- package/types/src/api/driver/vertex/path.d.mts +7 -0
- package/types/src/api/driver/vertex/request.d.mts +6 -0
- package/types/src/api/driver/vertex/settings.d.mts +23 -0
- package/types/src/api/driver/vertex/token/count.d.mts +7 -0
- package/types/src/api/driver/vllm/info.d.mts +2 -1
- package/types/src/api/driver/xai/info.d.mts +2 -1
- package/types/src/api/request.d.mts +3 -2
- package/types/src/api/route.d.mts +2 -2
- package/types/src/api/send.d.mts +1 -1
- package/types/src/api/server/plugin.d.mts +1 -1
- package/types/src/api/status.d.mts +1 -1
- package/types/src/lib/bedrock.d.mts +74 -0
- package/types/src/lib/config.d.mts +35 -5
- package/types/src/lib/converse.d.mts +81 -0
- package/types/src/lib/document-urls.d.mts +55 -0
- package/types/src/lib/estimate.d.mts +1 -1
- package/types/src/lib/eventstream.d.mts +34 -0
- package/types/src/lib/fetch-url.d.mts +64 -0
- package/types/src/lib/google-auth.d.mts +58 -0
- package/types/src/lib/http.d.mts +5 -3
- package/types/src/lib/presets.d.mts +2 -1
- package/types/src/lib/protocol.d.mts +87 -0
- package/types/src/lib/responses-upstream.d.mts +58 -0
- package/types/src/lib/responses.d.mts +72 -0
- package/types/src/lib/route.d.mts +7 -3
- package/types/src/lib/routes.d.mts +2 -1
- package/types/src/lib/settings.d.mts +19 -2
- package/types/src/lib/sigv4.d.mts +41 -0
- package/types/src/lib/vertex.d.mts +55 -0
- package/types/src/lib/vllm.d.mts +4 -4
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The canonical `api` value for what a config says.
|
|
3
|
+
* @param {any} value - The configured `api`.
|
|
4
|
+
* @returns {string | undefined} `"auto"` or a canonical protocol id; undefined for anything that is not an accepted name.
|
|
5
|
+
*/
|
|
6
|
+
export function canonical(value: any): string | undefined;
|
|
7
|
+
/**
|
|
8
|
+
* The protocols a driver's server speaks: `info().protocols`, plus its own `protocol`, plus (for a driver that still declares the older flags)
|
|
9
|
+
* the Responses API (`responses: true`) and its `native` mode.
|
|
10
|
+
* @param {any} info - Driver facts.
|
|
11
|
+
* @returns {string[]} Canonical ids, without duplicates.
|
|
12
|
+
*/
|
|
13
|
+
export function supported(info: any): string[];
|
|
14
|
+
/**
|
|
15
|
+
* The protocol a client speaks to the translator.
|
|
16
|
+
* @param {"anthropic" | "openai"} front - The front.
|
|
17
|
+
* @param {string} [wire] - `"responses"` for a request that came in on the Responses API.
|
|
18
|
+
* @returns {string} Protocol id.
|
|
19
|
+
*/
|
|
20
|
+
export function frontOf(front: "anthropic" | "openai", wire?: string): string;
|
|
21
|
+
/**
|
|
22
|
+
* The protocols to try for a request, best first.
|
|
23
|
+
* With an explicit `api` that is the one protocol. Under `auto` it is the client's own protocol, then Chat Completions, then (so a driver that speaks
|
|
24
|
+
* neither, such as Anthropic's, is still reachable by conversion) the driver's own protocol, minus the ones this upstream is known not to have.
|
|
25
|
+
* @param {{api: string, front: string, info: any, missing?: Set<string>, usable?: (protocol: string) => boolean}} spec - `api` is the endpoint's canonical setting, `front` the client's protocol, `info` the driver facts, `missing` the protocols remembered missing for the upstream (ignored for an explicit `api`) and `usable` a further per-request check.
|
|
26
|
+
* @returns {string[]} Protocols in order; never empty.
|
|
27
|
+
*/
|
|
28
|
+
export function candidates({ api, front, info, missing, usable }: {
|
|
29
|
+
api: string;
|
|
30
|
+
front: string;
|
|
31
|
+
info: any;
|
|
32
|
+
missing?: Set<string>;
|
|
33
|
+
usable?: (protocol: string) => boolean;
|
|
34
|
+
}): string[];
|
|
35
|
+
/**
|
|
36
|
+
* Remember that an upstream has no route for a protocol, so later requests skip it until the next detection.
|
|
37
|
+
* @param {any} state - Translator state (`missing`).
|
|
38
|
+
* @param {string} name - Upstream name.
|
|
39
|
+
* @param {string} protocol - Protocol id.
|
|
40
|
+
* @returns {void}
|
|
41
|
+
*/
|
|
42
|
+
export function remember(state: any, name: string, protocol: string): void;
|
|
43
|
+
/**
|
|
44
|
+
*
|
|
45
|
+
* @Project: @cldmv/llm-translator
|
|
46
|
+
* @Filename: /src/lib/protocol.mjs
|
|
47
|
+
* @Date: 2026-11-18T09:00:00-08:00 (1795021200)
|
|
48
|
+
* @Author: Nate Corcoran <CLDMV>
|
|
49
|
+
* @Email: <Shinrai@users.noreply.github.com>
|
|
50
|
+
* -----
|
|
51
|
+
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
52
|
+
* @Last modified time: 2026-11-18T09:00:00-08:00 (1795021200)
|
|
53
|
+
* -----
|
|
54
|
+
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
55
|
+
*
|
|
56
|
+
*/
|
|
57
|
+
/**
|
|
58
|
+
* @fileoverview The upstream protocols, the names the `api` setting accepts for them, and how a request picks one.
|
|
59
|
+
* @module @cldmv/llm-translator/lib/protocol
|
|
60
|
+
*/
|
|
61
|
+
export const MESSAGES: "anthropic.messages";
|
|
62
|
+
export const CHAT: "openai.chat";
|
|
63
|
+
export const RESPONSES: "openai.responses";
|
|
64
|
+
export const OLLAMA: "ollama.chat";
|
|
65
|
+
export const CONVERSE: "bedrock.converse";
|
|
66
|
+
/** The canonical protocol ids, in the order the docs list them. */
|
|
67
|
+
export const IDS: string[];
|
|
68
|
+
/** Every accepted spelling of the `api` setting and the canonical id it stands for. */
|
|
69
|
+
export const ALIASES: Readonly<{
|
|
70
|
+
auto: "auto";
|
|
71
|
+
"anthropic.messages": "anthropic.messages";
|
|
72
|
+
messages: "anthropic.messages";
|
|
73
|
+
anthropic: "anthropic.messages";
|
|
74
|
+
claude: "anthropic.messages";
|
|
75
|
+
"openai.chat": "openai.chat";
|
|
76
|
+
chat: "openai.chat";
|
|
77
|
+
openai: "openai.chat";
|
|
78
|
+
chatgpt: "openai.chat";
|
|
79
|
+
"openai.responses": "openai.responses";
|
|
80
|
+
responses: "openai.responses";
|
|
81
|
+
codex: "openai.responses";
|
|
82
|
+
"ollama.chat": "ollama.chat";
|
|
83
|
+
native: "ollama.chat";
|
|
84
|
+
ollama: "ollama.chat";
|
|
85
|
+
"bedrock.converse": "bedrock.converse";
|
|
86
|
+
converse: "bedrock.converse";
|
|
87
|
+
}>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a Chat Completions request body into a Responses request body.
|
|
3
|
+
*
|
|
4
|
+
* System messages before the first turn become `instructions`; assistant turns become message and `function_call` items and their
|
|
5
|
+
* tool results `function_call_output` items. The request is always stateless (`store: false`); with `reasoningRoundTrip` the encrypted
|
|
6
|
+
* reasoning of earlier turns is asked for (`include`) and sent back as `reasoning` items. Fields the Responses API has no counterpart
|
|
7
|
+
* for (`stop`, `seed`, the penalties, `logit_bias`, `logprobs`) are dropped and recorded in `state.dropped`.
|
|
8
|
+
* @param {any} chat - Chat Completions body.
|
|
9
|
+
* @param {Partial<import("./settings.mjs").Settings>} [settings] - Translator settings.
|
|
10
|
+
* @returns {{ body: any, state: any }} Responses body and conversion state (`front`, `model`, `stream`, `includeUsage`, `dropped`).
|
|
11
|
+
* @throws {TranslatorError} For a body without `messages`, `n` above 1 (`send` fans that out instead), an unknown role, or an unknown field when configured to error.
|
|
12
|
+
*/
|
|
13
|
+
export function request(chat: any, settings?: Partial<import("./settings.mjs").Settings>): {
|
|
14
|
+
body: any;
|
|
15
|
+
state: any;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Chat Completions usage for a Responses usage object.
|
|
19
|
+
* @param {any} usage - Responses `usage`.
|
|
20
|
+
* @returns {object | undefined} Chat usage, or undefined when the reply carried none.
|
|
21
|
+
*/
|
|
22
|
+
export function usageOf(usage: any): object | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Convert a (non-streaming) Responses reply into a Chat Completions reply.
|
|
25
|
+
*
|
|
26
|
+
* Message text becomes `content` (a refusal `refusal`), `function_call` items `tool_calls`, and the summary of reasoning items the
|
|
27
|
+
* reasoning field named by `reasoningField`. With `reasoningRoundTrip` every reasoning item that carries `encrypted_content` is also
|
|
28
|
+
* returned as a `reasoning_details` item (format `openai-responses-v1`), which the Anthropic converter signs into the thinking block and the
|
|
29
|
+
* request converter turns back into a `reasoning` item.
|
|
30
|
+
* @param {any} reply - Responses object.
|
|
31
|
+
* @param {any} [state] - State from the request converter (`model`; `upstreamUsage` is written to it).
|
|
32
|
+
* @param {Partial<import("./settings.mjs").Settings>} [settings] - Translator settings.
|
|
33
|
+
* @returns {any} Chat completion.
|
|
34
|
+
* @throws {TranslatorError} When the reply is not an object with an `output` list, or did not finish.
|
|
35
|
+
*/
|
|
36
|
+
export function response(reply: any, state?: any, settings?: Partial<import("./settings.mjs").Settings>): any;
|
|
37
|
+
/**
|
|
38
|
+
* Convert a Responses event stream into Chat Completions chunks followed by `[DONE]`.
|
|
39
|
+
*
|
|
40
|
+
* `source` is SSE text or bytes, parsed events, or bare event payloads (they carry their own `type`). Text, refusal, reasoning summary and
|
|
41
|
+
* argument deltas become the matching `delta` fields; a tool call starts with its `id` and name when its item is added. With
|
|
42
|
+
* `reasoningRoundTrip` the encrypted reasoning arrives as one `reasoning_details` chunk when its item is done. The final usage goes out as a
|
|
43
|
+
* `choices: []` chunk when `state.includeUsage` is not false. A `response.failed` or `error` event, or a stream that ends before a final
|
|
44
|
+
* event, becomes an `{ error }` chunk followed by `[DONE]`.
|
|
45
|
+
* @param {ReadableStream | AsyncIterable<unknown> | Iterable<unknown> | string | Uint8Array} source - Responses stream.
|
|
46
|
+
* @param {any} [state] - State from the request converter (`model`, `includeUsage`; `upstreamUsage` is written to it).
|
|
47
|
+
* @param {Partial<import("./settings.mjs").Settings>} [settings] - Translator settings.
|
|
48
|
+
* @returns {AsyncGenerator<{ data: any }, void, undefined>} Chat chunks.
|
|
49
|
+
*/
|
|
50
|
+
export function stream(source: ReadableStream | AsyncIterable<unknown> | Iterable<unknown> | string | Uint8Array, state?: any, settings?: Partial<import("./settings.mjs").Settings>): AsyncGenerator<{
|
|
51
|
+
data: any;
|
|
52
|
+
}, void, undefined>;
|
|
53
|
+
/** Protocol name of an upstream that speaks the Responses API. */
|
|
54
|
+
export const PROTOCOL: "openai.responses";
|
|
55
|
+
/** Statuses that mean the server has no Responses API: the route is missing (404, 405) or not implemented (501). */
|
|
56
|
+
export const MISSING: number[];
|
|
57
|
+
/** `format` of the `reasoning_details` items that carry a Responses reasoning item (the encrypted content and its summary). */
|
|
58
|
+
export const FORMAT: "openai-responses-v1";
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A Responses style id: `resp_`, `msg_`, `fc_` or `rs_` and 24 hex digits.
|
|
3
|
+
* @param {"resp" | "msg" | "fc" | "rs"} prefix - Id family.
|
|
4
|
+
* @returns {string} The id.
|
|
5
|
+
*/
|
|
6
|
+
export function itemId(prefix: "resp" | "msg" | "fc" | "rs"): string;
|
|
7
|
+
/**
|
|
8
|
+
* Pack the `reasoning_details` of a reply into the `encrypted_content` of a reasoning item, so the client sends them back with the next turn.
|
|
9
|
+
* @param {unknown} details - `reasoning_details` list.
|
|
10
|
+
* @returns {string | undefined} The carrier string, or undefined when there is nothing to carry.
|
|
11
|
+
*/
|
|
12
|
+
export function pack(details: unknown): string | undefined;
|
|
13
|
+
/**
|
|
14
|
+
* Read back what {@link pack} wrote. Anything else (another producer's data, a malformed or oversized value) gives null.
|
|
15
|
+
* @param {unknown} value - `encrypted_content` of a reasoning item the client sent.
|
|
16
|
+
* @returns {object[] | null} The `reasoning_details` items, or null.
|
|
17
|
+
*/
|
|
18
|
+
export function unpack(value: unknown): object[] | null;
|
|
19
|
+
/**
|
|
20
|
+
* Responses usage for a Chat Completions usage object.
|
|
21
|
+
* @param {any} usage - Chat `usage` (may be absent).
|
|
22
|
+
* @returns {{input_tokens: number, input_tokens_details: {cached_tokens: number}, output_tokens: number, output_tokens_details: {reasoning_tokens: number}, total_tokens: number} | null} Usage, or null when the upstream sent none.
|
|
23
|
+
*/
|
|
24
|
+
export function usageOf(usage: any): {
|
|
25
|
+
input_tokens: number;
|
|
26
|
+
input_tokens_details: {
|
|
27
|
+
cached_tokens: number;
|
|
28
|
+
};
|
|
29
|
+
output_tokens: number;
|
|
30
|
+
output_tokens_details: {
|
|
31
|
+
reasoning_tokens: number;
|
|
32
|
+
};
|
|
33
|
+
total_tokens: number;
|
|
34
|
+
} | null;
|
|
35
|
+
/**
|
|
36
|
+
* Status and `incomplete_details` for a Chat Completions `finish_reason`: a length stop and a content filter end the response early.
|
|
37
|
+
* @param {string | null | undefined} finish - `finish_reason`.
|
|
38
|
+
* @returns {{status: "completed"} | {status: "incomplete", incomplete_details: {reason: string}}} Terminal status.
|
|
39
|
+
*/
|
|
40
|
+
export function endOf(finish: string | null | undefined): {
|
|
41
|
+
status: "completed";
|
|
42
|
+
} | {
|
|
43
|
+
status: "incomplete";
|
|
44
|
+
incomplete_details: {
|
|
45
|
+
reason: string;
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* The response object around a list of output items, echoing the request parameters the way the API does.
|
|
50
|
+
* @param {any} state - State from the request converter (`model`, `echo`).
|
|
51
|
+
* @param {{id: string, created: number, status: string, model?: string, output: any[], usage?: any, error?: any, incomplete_details?: any}} parts - Per-response parts.
|
|
52
|
+
* @returns {any} Response object.
|
|
53
|
+
*/
|
|
54
|
+
export function envelope(state: any, parts: {
|
|
55
|
+
id: string;
|
|
56
|
+
created: number;
|
|
57
|
+
status: string;
|
|
58
|
+
model?: string;
|
|
59
|
+
output: any[];
|
|
60
|
+
usage?: any;
|
|
61
|
+
error?: any;
|
|
62
|
+
incomplete_details?: any;
|
|
63
|
+
}): any;
|
|
64
|
+
/**
|
|
65
|
+
* A failure's `{ code, message }` for a response in the `failed` status.
|
|
66
|
+
* @param {any} error - Chat error object (`{ message, type, code }`) or anything else thrown.
|
|
67
|
+
* @returns {{code: string, message: string}} Error as a response carries it.
|
|
68
|
+
*/
|
|
69
|
+
export function failureOf(error: any): {
|
|
70
|
+
code: string;
|
|
71
|
+
message: string;
|
|
72
|
+
};
|
|
@@ -31,14 +31,16 @@ export function pick(config: any, endpoints: any[], model: string | undefined, s
|
|
|
31
31
|
* Validate a client request body and resolve where it goes. Shared by `request` and `route` so both always agree.
|
|
32
32
|
* @param {any} c - Captured translator parts (`capture`).
|
|
33
33
|
* @param {any} body - Client request body.
|
|
34
|
-
* @param {{endpoint?: string, advance?: boolean}} [options] - `endpoint` forces one pool member by name (used for failover; it must still serve the model, else `endpoint_not_in_pool`); `advance` moves the round robin cursor (`request` does, `route` does not).
|
|
35
|
-
* @returns {{endpoint: any, model: string, requested: string | undefined, unserved: boolean, pin: {effort?: string, thinking?: boolean} | undefined, convert: any, pool: any[]}} The chosen endpoint, the model name sent
|
|
36
|
-
* upstream, the model the client asked for, whether no endpoint serves it, the settings its alias pins, the converters (null for a native pass-through) and the pool in try order.
|
|
34
|
+
* @param {{endpoint?: string, advance?: boolean, wire?: string, tokens?: boolean}} [options] - `wire: "responses"` marks a request that came in on the Responses API; `endpoint` forces one pool member by name (used for failover; it must still serve the model, else `endpoint_not_in_pool`); `advance` moves the round robin cursor (`request` does, `route` does not).
|
|
35
|
+
* @returns {{endpoint: any, model: string, requested: string | undefined, unserved: boolean, pin: {effort?: string, thinking?: boolean} | undefined, convert: any, protocol: string, more: boolean, pool: any[]}} The chosen endpoint, the model name sent
|
|
36
|
+
* upstream, the model the client asked for, whether no endpoint serves it, the settings its alias pins, the protocol it goes over (`protocol`, see `lib/protocol`), whether an `auto` upstream has another to fall back to (`more`), the converters (null for a native pass-through) and the pool in try order.
|
|
37
37
|
* @throws {TranslatorError} `invalid_body` when the body is not a JSON object, `endpoint_not_in_pool` when a forced endpoint no longer serves the model.
|
|
38
38
|
*/
|
|
39
39
|
export function resolve(c: any, body: any, options?: {
|
|
40
40
|
endpoint?: string;
|
|
41
41
|
advance?: boolean;
|
|
42
|
+
wire?: string;
|
|
43
|
+
tokens?: boolean;
|
|
42
44
|
}): {
|
|
43
45
|
endpoint: any;
|
|
44
46
|
model: string;
|
|
@@ -49,5 +51,7 @@ export function resolve(c: any, body: any, options?: {
|
|
|
49
51
|
thinking?: boolean;
|
|
50
52
|
} | undefined;
|
|
51
53
|
convert: any;
|
|
54
|
+
protocol: string;
|
|
55
|
+
more: boolean;
|
|
52
56
|
pool: any[];
|
|
53
57
|
};
|
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
* @param {Promise<any>} sending - Pending `send` result.
|
|
5
5
|
* @param {any} api - Captured translator parts.
|
|
6
6
|
* @param {string} [model] - Model the client asked for (used in the early `message_start`).
|
|
7
|
+
* @param {(source: AsyncIterable<any>) => AsyncIterable<any>} [convert] - Turns the upstream's stream into the client's events (the Responses front).
|
|
7
8
|
* @returns {AsyncGenerator<any>} Client events.
|
|
8
9
|
*/
|
|
9
|
-
export function late(sending: Promise<any>, api: any, model?: string): AsyncGenerator<any>;
|
|
10
|
+
export function late(sending: Promise<any>, api: any, model?: string, convert?: (source: AsyncIterable<any>) => AsyncIterable<any>): AsyncGenerator<any>;
|
|
10
11
|
/**
|
|
11
12
|
* Register the routes for one front on a Fastify instance.
|
|
12
13
|
* @param {any} fastify - Fastify instance (usually an encapsulated plugin scope).
|
|
@@ -14,6 +14,7 @@ export function resolve(...layers: (Partial<Settings> | null | undefined)[]): Se
|
|
|
14
14
|
* @property {boolean} thinkTags - Parse a leading `<think>...</think>` in content as reasoning.
|
|
15
15
|
* @property {boolean} thinkOpen - With `thinkTags`: the output starts inside a thought because the chat template opens `<think>` in the generation prompt, so reasoning runs up to the first `</think>`. Without a `</think>` the content is plain text.
|
|
16
16
|
* @property {boolean} reasoningRoundTrip - Carry reasoning the upstream needs back across turns. OpenAI-protocol upstream: the provider's reasoning fields (`reasoning_content`, `reasoning`, `reasoning_details`) travel in the thinking block's `llmt:` signature and are restored on the next request. Anthropic-protocol upstream with an OpenAI client: thinking blocks and their signatures travel as `reasoning_details` items the client replays. Off by default; the DeepSeek driver turns it on.
|
|
17
|
+
* @property {"auto"|"concise"|"detailed"|"none"} reasoningSummary - Responses-protocol upstream: the `reasoning.summary` asked for whenever the request sets a reasoning effort, so the model's reasoning summary comes back as thinking text. `none` does not ask for one.
|
|
17
18
|
* @property {"chat_template_kwargs"|"reasoning_effort"|"none"} thinking - How Anthropic `thinking` is requested from the server.
|
|
18
19
|
* @property {boolean} thinkingBudget - Send `thinking_token_budget` for `thinking.type = "enabled"`.
|
|
19
20
|
* @property {boolean} effort - Map `output_config.effort` to `reasoning_effort`.
|
|
@@ -22,6 +23,7 @@ export function resolve(...layers: (Partial<Settings> | null | undefined)[]): Se
|
|
|
22
23
|
* @property {boolean} toolFallback - Retry a request once without tools when an OpenAI-protocol upstream answers 400 that tool parsing is not enabled (vLLM started without `--enable-auto-tool-choice` and `--tool-call-parser`). Off by default: the request then runs without tools, which silently changes what the model can do.
|
|
23
24
|
* @property {"extract"|"note"|"error"} documents - How `document` blocks are handled.
|
|
24
25
|
* @property {((block: object, ctx: object) => Promise<object[]>|object[])|null} documentConverter - Override for `document` conversion; returns Anthropic content blocks.
|
|
26
|
+
* @property {null|{ allow: string[], maxBytes: number, timeout: number, maxRedirects: number, maxDocuments: number, allowPrivate: boolean }} documentUrls - Fetch the `url` sources of `document` blocks before conversion; `null` (the default) is off, and an object turns it on (`allow` is required).
|
|
25
27
|
* @property {{ pages: boolean, maxPages: number, maxBytes: number, maxImages: number, scale: number }} pdf - PDF handling: render page images, Anthropic-like limits, image budget and render scale.
|
|
26
28
|
* @property {"pass"|"sanitize"} toolSchema - Sanitize tool JSON Schemas (drop `$schema`, remove unbounded integer limits).
|
|
27
29
|
* @property {"forward"|"drop"} deferredTools - Offer `defer_loading` tools as `toolSearch` says, or drop them (and the search tool).
|
|
@@ -47,7 +49,7 @@ export function resolve(...layers: (Partial<Settings> | null | undefined)[]): Se
|
|
|
47
49
|
* @property {"buffer"|"stream"} toolStreaming - OpenAI to Anthropic streams: `buffer` holds each tool call until its arguments are complete and emits one whole `tool_use` block (text and thinking keep streaming and appear before the block); `stream` forwards argument fragments incrementally and holds text that arrives meanwhile until the tool blocks are done.
|
|
48
50
|
* @property {boolean} effortPassthrough - vLLM driver: send `xhigh` and `max` reasoning effort unchanged instead of clamping them to `high`.
|
|
49
51
|
* @property {boolean} passthroughFilter - Restrict bodies forwarded from the OpenAI front to an OpenAI-protocol upstream to standard Chat Completions fields plus documented vLLM extensions. Off in the library; the server turns it on unless set to false.
|
|
50
|
-
* @property {boolean | "auto"} nativeMessages -
|
|
52
|
+
* @property {boolean | "auto"} nativeMessages - Deprecated: use the upstream's `api` instead. For a vLLM upstream that sets no `api`, `true` reads as `api: "anthropic.messages"`, `"auto"` as `api: "auto"` and `false` as `api: "openai.chat"`, with a deprecation warning; it is ignored (also with a warning) when `api` is set.
|
|
51
53
|
* @property {number} contextWindow - Prompt size in tokens the upstream truncates at, or 0 when unknown. Drivers whose server silently truncates (Ollama) warn once when a request is estimated to exceed it.
|
|
52
54
|
* @property {number} defaultMaxTokens - `max_tokens` used when an OpenAI request has none (Anthropic requires one).
|
|
53
55
|
*/
|
|
@@ -81,6 +83,10 @@ export type Settings = {
|
|
|
81
83
|
* - Carry reasoning the upstream needs back across turns. OpenAI-protocol upstream: the provider's reasoning fields (`reasoning_content`, `reasoning`, `reasoning_details`) travel in the thinking block's `llmt:` signature and are restored on the next request. Anthropic-protocol upstream with an OpenAI client: thinking blocks and their signatures travel as `reasoning_details` items the client replays. Off by default; the DeepSeek driver turns it on.
|
|
82
84
|
*/
|
|
83
85
|
reasoningRoundTrip: boolean;
|
|
86
|
+
/**
|
|
87
|
+
* - Responses-protocol upstream: the `reasoning.summary` asked for whenever the request sets a reasoning effort, so the model's reasoning summary comes back as thinking text. `none` does not ask for one.
|
|
88
|
+
*/
|
|
89
|
+
reasoningSummary: "auto" | "concise" | "detailed" | "none";
|
|
84
90
|
/**
|
|
85
91
|
* - How Anthropic `thinking` is requested from the server.
|
|
86
92
|
*/
|
|
@@ -113,6 +119,17 @@ export type Settings = {
|
|
|
113
119
|
* - Override for `document` conversion; returns Anthropic content blocks.
|
|
114
120
|
*/
|
|
115
121
|
documentConverter: ((block: object, ctx: object) => Promise<object[]> | object[]) | null;
|
|
122
|
+
/**
|
|
123
|
+
* - Fetch the `url` sources of `document` blocks before conversion; `null` (the default) is off, and an object turns it on (`allow` is required).
|
|
124
|
+
*/
|
|
125
|
+
documentUrls: null | {
|
|
126
|
+
allow: string[];
|
|
127
|
+
maxBytes: number;
|
|
128
|
+
timeout: number;
|
|
129
|
+
maxRedirects: number;
|
|
130
|
+
maxDocuments: number;
|
|
131
|
+
allowPrivate: boolean;
|
|
132
|
+
};
|
|
116
133
|
/**
|
|
117
134
|
* - PDF handling: render page images, Anthropic-like limits, image budget and render scale.
|
|
118
135
|
*/
|
|
@@ -225,7 +242,7 @@ export type Settings = {
|
|
|
225
242
|
*/
|
|
226
243
|
passthroughFilter: boolean;
|
|
227
244
|
/**
|
|
228
|
-
* -
|
|
245
|
+
* - Deprecated: use the upstream's `api` instead. For a vLLM upstream that sets no `api`, `true` reads as `api: "anthropic.messages"`, `"auto"` as `api: "auto"` and `false` as `api: "openai.chat"`, with a deprecation warning; it is ignored (also with a warning) when `api` is set.
|
|
229
246
|
*/
|
|
230
247
|
nativeMessages: boolean | "auto";
|
|
231
248
|
/**
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `X-Amz-Date` form of a time: `YYYYMMDD'T'HHMMSS'Z'`.
|
|
3
|
+
* @param {Date} date - The time.
|
|
4
|
+
* @returns {string} Basic ISO 8601 UTC time.
|
|
5
|
+
*/
|
|
6
|
+
export function stamp(date: Date): string;
|
|
7
|
+
/**
|
|
8
|
+
* Sign a request with Signature Version 4. Every header given is signed, together with `host`, `x-amz-date` and, when there is a session
|
|
9
|
+
* token, `x-amz-security-token`. The body is signed as the hash of the exact text or bytes passed in, so it must be what is sent.
|
|
10
|
+
* @param {object} request - The request.
|
|
11
|
+
* @param {string} [request.method] - HTTP method (default `GET`).
|
|
12
|
+
* @param {string} request.url - Absolute URL as it will be requested.
|
|
13
|
+
* @param {Record<string,string>} [request.headers] - Headers that will be sent (names in any case).
|
|
14
|
+
* @param {string | Uint8Array} [request.body] - Body that will be sent (default empty).
|
|
15
|
+
* @param {string} request.region - AWS region of the credential scope.
|
|
16
|
+
* @param {string} request.service - Service of the credential scope (`bedrock` for Amazon Bedrock).
|
|
17
|
+
* @param {string} request.accessKeyId - Access key id.
|
|
18
|
+
* @param {string} request.secretAccessKey - Secret access key.
|
|
19
|
+
* @param {string} [request.sessionToken] - Session token of temporary credentials.
|
|
20
|
+
* @param {Date} [request.date] - Signing time (default now).
|
|
21
|
+
* @returns {{headers: Record<string,string>, authorization: string, signature: string, signedHeaders: string, canonicalRequest: string, stringToSign: string}} The headers to send (the ones given plus `x-amz-date`, `x-amz-security-token` and `authorization`; `host` is left to the transport) and the intermediate values, which tests compare with published ones.
|
|
22
|
+
*/
|
|
23
|
+
export function sign({ method, url, headers, body, region, service, accessKeyId, secretAccessKey, sessionToken, date }: {
|
|
24
|
+
method?: string;
|
|
25
|
+
url: string;
|
|
26
|
+
headers?: Record<string, string>;
|
|
27
|
+
body?: string | Uint8Array;
|
|
28
|
+
region: string;
|
|
29
|
+
service: string;
|
|
30
|
+
accessKeyId: string;
|
|
31
|
+
secretAccessKey: string;
|
|
32
|
+
sessionToken?: string;
|
|
33
|
+
date?: Date;
|
|
34
|
+
}): {
|
|
35
|
+
headers: Record<string, string>;
|
|
36
|
+
authorization: string;
|
|
37
|
+
signature: string;
|
|
38
|
+
signedHeaders: string;
|
|
39
|
+
canonicalRequest: string;
|
|
40
|
+
stringToSign: string;
|
|
41
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The API host of a region: `<region>-aiplatform.googleapis.com`, `aiplatform.googleapis.com` for `global`, and `aiplatform.<us|eu>.rep.googleapis.com`
|
|
3
|
+
* for the multi-region endpoints.
|
|
4
|
+
* @param {string} region - Region such as `us-east5`, `europe-west1`, `global`, `us` or `eu`.
|
|
5
|
+
* @returns {string} Host name.
|
|
6
|
+
*/
|
|
7
|
+
export function host(region: string): string;
|
|
8
|
+
/**
|
|
9
|
+
* The region a Vertex AI host serves.
|
|
10
|
+
* @param {string} url - Upstream base URL.
|
|
11
|
+
* @returns {string | undefined} The region, `"global"` for the global host, `"us"` or `"eu"` for a multi-region host, undefined for any other host.
|
|
12
|
+
*/
|
|
13
|
+
export function regionOf(url: string): string | undefined;
|
|
14
|
+
/**
|
|
15
|
+
* The region of an upstream: the `region` setting, else the region in a Vertex AI base URL, else `CLOUD_ML_REGION`.
|
|
16
|
+
* @param {any} endpoint - Runtime endpoint (`url`, `vertex`).
|
|
17
|
+
* @param {Record<string, string | undefined>} [env] - Environment.
|
|
18
|
+
* @returns {string | undefined} Region, or undefined when none is set.
|
|
19
|
+
*/
|
|
20
|
+
export function regionFor(endpoint: any, env?: Record<string, string | undefined>): string | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* The Google Cloud project and region of an upstream. The project is the `project` setting, else `ANTHROPIC_VERTEX_PROJECT_ID`,
|
|
23
|
+
* `GOOGLE_CLOUD_PROJECT` or `GCLOUD_PROJECT`, else the `project_id` (or `quota_project_id`) of the credentials file.
|
|
24
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
25
|
+
* @param {Record<string, string | undefined>} [env] - Environment.
|
|
26
|
+
* @returns {{project: string, region: string}} Where requests go.
|
|
27
|
+
* @throws {TranslatorError} With code `INVALID_CONFIG` when either is unknown.
|
|
28
|
+
*/
|
|
29
|
+
export function locate(endpoint: any, env?: Record<string, string | undefined>): {
|
|
30
|
+
project: string;
|
|
31
|
+
region: string;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* The path of a Vertex AI call for Claude.
|
|
35
|
+
* @param {"messages" | "count"} kind - `"messages"` for a request (`rawPredict`, or `streamRawPredict` when streaming), `"count"` for token counting.
|
|
36
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
37
|
+
* @param {string} [model] - Model for a `"messages"` call, for example `claude-sonnet-4-5@20250929`.
|
|
38
|
+
* @param {{stream?: boolean, env?: Record<string, string | undefined>}} [options] - Whether the call streams, and the environment.
|
|
39
|
+
* @returns {string} Path starting with `/v1/projects/`.
|
|
40
|
+
*/
|
|
41
|
+
export function route(kind: "messages" | "count", endpoint: any, model?: string, { stream, env }?: {
|
|
42
|
+
stream?: boolean;
|
|
43
|
+
env?: Record<string, string | undefined>;
|
|
44
|
+
}): string;
|
|
45
|
+
/**
|
|
46
|
+
* The body Vertex AI takes: the Messages body with `anthropic_version` added and the model taken out (it is in the path). The model stays on the
|
|
47
|
+
* result as a property that is not enumerable, so it is not sent but the rest of the translator can still read `body.model`.
|
|
48
|
+
* @param {any} body - Anthropic Messages request body.
|
|
49
|
+
* @returns {any} Vertex AI body.
|
|
50
|
+
*/
|
|
51
|
+
export function body(body: any): any;
|
|
52
|
+
/** The `anthropic_version` Vertex AI requires in the body. */
|
|
53
|
+
export const VERSION: "vertex-2023-10-16";
|
|
54
|
+
/** The hosts that are not of the form `<region>-aiplatform.googleapis.com`: the global one and the multi-region ones. */
|
|
55
|
+
export const HOSTS: string[];
|
package/types/src/lib/vllm.d.mts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* May `/v1/messages` be tried on this upstream without being asked for? Only when the detected or configured version is known and is 0.11.1 or later:
|
|
3
|
+
* an older server has no such route, and an unknown version stays on Chat Completions (set `api` to use the route anyway).
|
|
3
4
|
* @param {any} endpoint - Runtime endpoint.
|
|
4
|
-
* @
|
|
5
|
-
* @returns {boolean} True for a vLLM upstream with `nativeMessages: true`, or `"auto"` and a known version of 0.11.1 or later (an unknown version stays on conversion).
|
|
5
|
+
* @returns {boolean} True for a vLLM upstream with a known version of 0.11.1 or later.
|
|
6
6
|
*/
|
|
7
|
-
export function
|
|
7
|
+
export function eligible(endpoint: any): boolean;
|
|
8
8
|
/**
|
|
9
9
|
* Why a request cannot go to the native endpoint, so it is converted instead.
|
|
10
10
|
*
|