@cldmv/llm-translator 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/dist/api/convert/anthropic/messages/to/openai/chat/request.mjs +8 -8
- package/dist/api/convert/openai/chat/to/anthropic/messages/response.mjs +1 -1
- package/dist/api/convert/openai/chat/to/anthropic/messages/stream.mjs +1 -1
- package/dist/api/driver/azure/detect.mjs +2 -0
- package/dist/api/driver/azure/error.mjs +2 -0
- package/dist/api/driver/azure/info.mjs +2 -0
- package/dist/api/driver/azure/model/list.mjs +2 -0
- package/dist/api/driver/azure/path.mjs +2 -0
- package/dist/api/driver/azure/request.mjs +2 -0
- package/dist/api/driver/azure/settings.mjs +2 -0
- package/dist/api/driver/lmstudio/detect.mjs +2 -0
- package/dist/api/driver/lmstudio/info.mjs +2 -0
- package/dist/api/driver/lmstudio/request.mjs +2 -0
- package/dist/api/driver/lmstudio/settings.mjs +2 -0
- package/dist/api/driver/localai/chunk.mjs +2 -0
- package/dist/api/driver/localai/detect.mjs +2 -0
- package/dist/api/driver/localai/info.mjs +2 -0
- package/dist/api/driver/localai/request.mjs +2 -0
- package/dist/api/driver/localai/response.mjs +2 -0
- package/dist/api/driver/localai/settings.mjs +2 -0
- package/dist/api/driver/mistral/chunk.mjs +2 -0
- package/dist/api/driver/mistral/detect.mjs +2 -0
- package/dist/api/driver/mistral/info.mjs +2 -0
- package/dist/api/driver/mistral/request.mjs +2 -0
- package/dist/api/driver/mistral/response.mjs +2 -0
- package/dist/api/driver/mistral/settings.mjs +2 -0
- package/dist/api/driver/openrouter/chunk.mjs +2 -0
- package/dist/api/driver/openrouter/detect.mjs +2 -0
- package/dist/api/driver/openrouter/info.mjs +2 -0
- package/dist/api/driver/openrouter/path.mjs +2 -0
- package/dist/api/driver/openrouter/request.mjs +2 -0
- package/dist/api/driver/openrouter/response.mjs +2 -0
- package/dist/api/driver/openrouter/settings.mjs +2 -0
- package/dist/api/driver/tgi/chunk.mjs +2 -0
- package/dist/api/driver/tgi/detect.mjs +2 -0
- package/dist/api/driver/tgi/info.mjs +2 -0
- package/dist/api/driver/tgi/request.mjs +2 -0
- package/dist/api/driver/tgi/response.mjs +2 -0
- package/dist/api/driver/tgi/settings.mjs +2 -0
- package/dist/api/driver/vllm/detect.mjs +1 -1
- package/dist/api/embeddings/send.mjs +1 -1
- package/dist/api/request.mjs +2 -2
- package/dist/api/send.mjs +1 -1
- package/dist/lib/azure.mjs +2 -0
- package/dist/lib/backends.mjs +1 -1
- package/dist/lib/capture.mjs +1 -1
- package/dist/lib/config.mjs +1 -1
- package/dist/lib/fanout.mjs +2 -0
- package/dist/lib/finish.mjs +1 -1
- package/dist/lib/http.mjs +1 -1
- package/dist/lib/models.mjs +1 -1
- package/dist/lib/openrouter.mjs +2 -0
- package/dist/lib/tgi.mjs +2 -0
- package/dist/lib/typed.mjs +2 -0
- package/dist/lib/usage.mjs +1 -1
- package/package.json +1 -1
- package/types/src/api/convert/openai/chat/to/anthropic/messages/request.d.mts +1 -1
- package/types/src/api/driver/azure/detect.d.mts +8 -0
- package/types/src/api/driver/azure/error.d.mts +15 -0
- package/types/src/api/driver/azure/info.d.mts +31 -0
- package/types/src/api/driver/azure/model/list.d.mts +7 -0
- package/types/src/api/driver/azure/path.d.mts +7 -0
- package/types/src/api/driver/azure/request.d.mts +8 -0
- package/types/src/api/driver/azure/settings.d.mts +23 -0
- package/types/src/api/driver/lmstudio/detect.d.mts +6 -0
- package/types/src/api/driver/lmstudio/info.d.mts +31 -0
- package/types/src/api/driver/lmstudio/request.d.mts +6 -0
- package/types/src/api/driver/lmstudio/settings.d.mts +23 -0
- package/types/src/api/driver/localai/chunk.d.mts +25 -0
- package/types/src/api/driver/localai/detect.d.mts +6 -0
- package/types/src/api/driver/localai/info.d.mts +33 -0
- package/types/src/api/driver/localai/request.d.mts +10 -0
- package/types/src/api/driver/localai/response.d.mts +26 -0
- package/types/src/api/driver/localai/settings.d.mts +25 -0
- package/types/src/api/driver/mistral/chunk.d.mts +8 -0
- package/types/src/api/driver/mistral/detect.d.mts +26 -0
- package/types/src/api/driver/mistral/info.d.mts +35 -0
- package/types/src/api/driver/mistral/request.d.mts +6 -0
- package/types/src/api/driver/mistral/response.d.mts +7 -0
- package/types/src/api/driver/mistral/settings.d.mts +24 -0
- package/types/src/api/driver/openrouter/chunk.d.mts +25 -0
- package/types/src/api/driver/openrouter/detect.d.mts +26 -0
- package/types/src/api/driver/openrouter/info.d.mts +30 -0
- package/types/src/api/driver/openrouter/path.d.mts +8 -0
- package/types/src/api/driver/openrouter/request.d.mts +10 -0
- package/types/src/api/driver/openrouter/response.d.mts +8 -0
- package/types/src/api/driver/openrouter/settings.d.mts +26 -0
- package/types/src/api/driver/tgi/chunk.d.mts +9 -0
- package/types/src/api/driver/tgi/detect.d.mts +6 -0
- package/types/src/api/driver/tgi/info.d.mts +33 -0
- package/types/src/api/driver/tgi/request.d.mts +9 -0
- package/types/src/api/driver/tgi/response.d.mts +9 -0
- package/types/src/api/driver/tgi/settings.d.mts +25 -0
- package/types/src/api/driver/vllm/detect.d.mts +1 -1
- package/types/src/api/send.d.mts +7 -0
- package/types/src/lib/azure.d.mts +17 -0
- package/types/src/lib/backends.d.mts +29 -0
- package/types/src/lib/config.d.mts +8 -0
- package/types/src/lib/fanout.d.mts +68 -0
- package/types/src/lib/finish.d.mts +14 -3
- package/types/src/lib/http.d.mts +13 -2
- package/types/src/lib/models.d.mts +2 -2
- package/types/src/lib/openrouter.d.mts +23 -0
- package/types/src/lib/tgi.d.mts +33 -0
- package/types/src/lib/typed.d.mts +10 -0
- package/types/src/lib/usage.d.mts +20 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put the calls under `/api`, so the upstream can be given as `https://openrouter.ai` or `https://openrouter.ai/api` (a trailing `/v1` is removed
|
|
3
|
+
* by the config).
|
|
4
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
5
|
+
* @param {any} context - `{ endpoint }`.
|
|
6
|
+
* @returns {string} Path starting with `/`.
|
|
7
|
+
*/
|
|
8
|
+
export function path(kind: "chat" | "embeddings" | "models", { endpoint }: any): string;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop empty tool fields and express the reasoning controls the way OpenRouter takes them: a `reasoning` object with either `effort` or
|
|
3
|
+
* `max_tokens`, never both. A `reasoning` object the client sent itself is kept as it is (its `reasoning_effort` sibling is removed). Otherwise
|
|
4
|
+
* `reasoning_effort` becomes `reasoning: { effort }`, and an Anthropic request that enabled thinking with a `budget_tokens` and no effort
|
|
5
|
+
* becomes `reasoning: { max_tokens }`, capped below the request's `max_tokens`.
|
|
6
|
+
* @param {any} body - Chat Completions request body.
|
|
7
|
+
* @param {any} [context] - `{ state }`; `state.thinkingBudget` is the thinking budget of an Anthropic request.
|
|
8
|
+
* @returns {any} Normalized body.
|
|
9
|
+
*/
|
|
10
|
+
export function request(body: any, context?: any): any;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn an error OpenRouter sent with HTTP 200 into an upstream error: `choices[i].error` on every choice, or a top-level `error` and no choices.
|
|
3
|
+
* Any partial text of the failed choice is dropped. A reply with a choice that did not fail passes unchanged.
|
|
4
|
+
* @param {any} body - Parsed `chat.completion`.
|
|
5
|
+
* @returns {any} The body.
|
|
6
|
+
* @throws {import("../../../lib/errors.mjs").TranslatorError} With the error's `message` and its `code` as the HTTP status.
|
|
7
|
+
*/
|
|
8
|
+
export function response(body: any): any;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
*
|
|
3
|
+
* @Project: @cldmv/llm-translator
|
|
4
|
+
* @Filename: /src/api/driver/openrouter/settings.mjs
|
|
5
|
+
* @Date: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
6
|
+
* @Author: Nate Corcoran <CLDMV>
|
|
7
|
+
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
|
+
* -----
|
|
9
|
+
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
+
* @Last modified time: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
11
|
+
* -----
|
|
12
|
+
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
|
+
*
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* @fileoverview OpenRouter driver: translator settings for the hosted OpenRouter API.
|
|
17
|
+
* @module @cldmv/llm-translator/driver/openrouter
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Settings for openrouter.ai: reasoning comes back in `reasoning` and `reasoning_details`, and `reasoning_details` is sent back unmodified and in
|
|
21
|
+
* order (`reasoningRoundTrip`). Thinking and effort are sent as `reasoning_effort`, which the `request` normalizer turns into OpenRouter's
|
|
22
|
+
* `reasoning` object. `max_tokens` is the output cap, `top_k` is accepted (OpenRouter drops what a model does not support), a trailing assistant
|
|
23
|
+
* message is OpenRouter's own prefill, and usage comes in a final chunk.
|
|
24
|
+
* @returns {any} Partial settings.
|
|
25
|
+
*/
|
|
26
|
+
export function settings(): any;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put a streamed TGI chunk into the OpenAI shape: `delta.tool_calls` as one object becomes a list, numeric ids become strings, object
|
|
3
|
+
* `arguments` become a JSON string, an empty id (TGI repeats the field on every fragment) is removed so the fragment continues the open
|
|
4
|
+
* call, and `eos_token` / `stop_sequence` finish reasons become `stop` (or `tool_calls` once the stream carried calls).
|
|
5
|
+
* @param {any} chunk - Parsed `chat.completion.chunk`.
|
|
6
|
+
* @param {any} [ctx] - `{ state }`; remembers whether the stream carried tool calls.
|
|
7
|
+
* @returns {any} Normalized chunk.
|
|
8
|
+
*/
|
|
9
|
+
export function chunk(chunk: any, ctx?: any): any;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe an endpoint for Text Generation Inference.
|
|
3
|
+
* @param {any} context - `{ endpoint, transport, signal, timeout, probes }`.
|
|
4
|
+
* @returns {Promise<any | null>} Facts `{ version, model, maxInputTokens, maxTotalTokens }`, or null when the endpoint is not TGI.
|
|
5
|
+
*/
|
|
6
|
+
export function detect(context: any): Promise<any | null>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
*
|
|
3
|
+
* @Project: @cldmv/llm-translator
|
|
4
|
+
* @Filename: /src/api/driver/tgi/info.mjs
|
|
5
|
+
* @Date: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
6
|
+
* @Author: Nate Corcoran <CLDMV>
|
|
7
|
+
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
|
+
* -----
|
|
9
|
+
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
+
* @Last modified time: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
11
|
+
* -----
|
|
12
|
+
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
|
+
*
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* @fileoverview Hugging Face TGI driver: static facts about Text Generation Inference.
|
|
17
|
+
* @module @cldmv/llm-translator/driver/tgi
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Describe the TGI driver. It is found through `GET /info`, TGI's router description. `grammar` is allowed through the OpenAI front's
|
|
21
|
+
* request filter because the request normalizer folds it into `response_format`, the only place the chat endpoint reads a grammar.
|
|
22
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, probes: string[], fields: string[], paths: Record<string,string>}} Driver facts.
|
|
23
|
+
*/
|
|
24
|
+
export function info(): {
|
|
25
|
+
name: string;
|
|
26
|
+
protocol: string;
|
|
27
|
+
order: number;
|
|
28
|
+
auth: string;
|
|
29
|
+
strict: boolean;
|
|
30
|
+
probes: string[];
|
|
31
|
+
fields: string[];
|
|
32
|
+
paths: Record<string, string>;
|
|
33
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools; fold a client's `grammar` into `response_format` (the chat endpoint reads a grammar only
|
|
3
|
+
* there) and rewrite `response_format` into TGI's shape; and drop `response_format` when the request also has tools, which TGI refuses
|
|
4
|
+
* (the tools win, and the drop is recorded in `state.dropped`).
|
|
5
|
+
* @param {any} body - Chat Completions request body.
|
|
6
|
+
* @param {any} [ctx] - `{ state }`.
|
|
7
|
+
* @returns {any} Normalized body.
|
|
8
|
+
*/
|
|
9
|
+
export function request(body: any, ctx?: any): any;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put TGI's historic deviations into the OpenAI shape: `tool_calls` as one object becomes a list, numeric tool-call ids become strings,
|
|
3
|
+
* object `arguments` (or `parameters`) become a JSON string, `eos_token` and `stop_sequence` finish reasons become `stop` (or `tool_calls`
|
|
4
|
+
* when the answer holds calls), and the `text_completion` object name becomes `chat.completion`. Current TGI already answers in the
|
|
5
|
+
* standard shape, which passes through unchanged.
|
|
6
|
+
* @param {any} body - Chat Completions response body.
|
|
7
|
+
* @returns {any} Normalized body.
|
|
8
|
+
*/
|
|
9
|
+
export function response(body: any): any;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
*
|
|
3
|
+
* @Project: @cldmv/llm-translator
|
|
4
|
+
* @Filename: /src/api/driver/tgi/settings.mjs
|
|
5
|
+
* @Date: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
6
|
+
* @Author: Nate Corcoran <CLDMV>
|
|
7
|
+
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
|
+
* -----
|
|
9
|
+
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
+
* @Last modified time: 2026-11-13T09:00:00-08:00 (1794589200)
|
|
11
|
+
* -----
|
|
12
|
+
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
|
+
*
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* @fileoverview Hugging Face TGI driver: translator settings for TGI servers.
|
|
17
|
+
* @module @cldmv/llm-translator/driver/tgi
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Settings for TGI: `max_tokens` as the output cap, no `top_k` (the chat request has none), no prefill continuation, no thinking switch,
|
|
21
|
+
* and `response_format` for structured output (the request normalizer rewrites it into TGI's grammar shape). Usage comes on the last
|
|
22
|
+
* content chunk when `stream_options.include_usage` is set.
|
|
23
|
+
* @returns {any} Partial settings.
|
|
24
|
+
*/
|
|
25
|
+
export function settings(): any;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Probe an endpoint for vLLM.
|
|
3
3
|
* @param {any} context - `{ endpoint, transport, signal, timeout, probes }`.
|
|
4
|
-
* @returns {Promise<any | null>} Facts `{ version, models, maxModelLen }`, or null when the endpoint is not vLLM (or is SGLang
|
|
4
|
+
* @returns {Promise<any | null>} Facts `{ version, models, maxModelLen }`, or null when the endpoint is not vLLM (or is LocalAI, SGLang, llama.cpp or TGI, which can also answer `/version`).
|
|
5
5
|
*/
|
|
6
6
|
export function detect({ endpoint, transport, signal, timeout, probes }: any): Promise<any | null>;
|
package/types/src/api/send.d.mts
CHANGED
|
@@ -17,6 +17,13 @@
|
|
|
17
17
|
* `estimated: { input_tokens, output_tokens }` (null for a side that was reported, and null altogether when nothing was estimated).
|
|
18
18
|
* On the Anthropic front a stream carries them in the final `message_delta`, because `message_start` is sent before the output is known.
|
|
19
19
|
* Replies passed through without conversion are never altered.
|
|
20
|
+
*
|
|
21
|
+
* An OpenAI-front request with `n` above 1 whose upstream speaks the Anthropic protocol (which has no `n`) is answered by `n` parallel
|
|
22
|
+
* single-choice requests, one `send` each (so pools, failover, capture and events apply per branch). The choices come back numbered 0 to
|
|
23
|
+
* n-1 and the usage is the sum of the branches; `onUsage` is called once with the summed report (`reason` and `complete` follow the
|
|
24
|
+
* branch that failed first). The first branch to fail ends the request: the others are aborted and the failure is the reply, or, once
|
|
25
|
+
* a stream has started, an error chunk and `[DONE]`. Aborting the caller's signal aborts every branch. An OpenAI-protocol upstream
|
|
26
|
+
* receives `n` unchanged and answers it itself.
|
|
20
27
|
* @param {any} body - Client request body (per `front`).
|
|
21
28
|
* @param {any} [options] - `{ signal, front, headers, clientHeaders, filter, onUsage, log }`: abort signal for the upstream call, per-call front, extra upstream headers, client headers forwarded to a native Anthropic upstream, the pass-through body filter (see `request`) and the usage callback (an exception in it is reported as a `usage` event and never breaks the reply) and the logger this request's events also go to (used by the server).
|
|
22
29
|
* @returns {Promise<{status: number, headers: Record<string,string>, body: any, usage: Promise<any>} | {status: number, headers: Record<string,string>, stream: AsyncIterable<{event?: string, data: *}>, usage: Promise<any>}>} A JSON reply, or a stream reply when the client asked for streaming and the upstream accepted.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the path (and query) of an Azure OpenAI call.
|
|
3
|
+
*
|
|
4
|
+
* The v1 API (`<endpoint>/openai/v1/...`, the model named in the body, no `api-version`) is used unless the upstream sets `apiVersion`.
|
|
5
|
+
* With `apiVersion` the classic form is used: `<endpoint>/openai/deployments/<deployment>/chat/completions?api-version=<version>`, where the
|
|
6
|
+
* deployment is the upstream's `deployment` or else the request's model. The model list has no deployment in its path (`/openai/models`).
|
|
7
|
+
* An upstream URL that already ends in `/openai` (a pasted `.../openai/v1` base has its `/v1` removed by the config) is not given a second `/openai`.
|
|
8
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
9
|
+
* @param {{url: string, apiVersion?: string, deployment?: string}} endpoint - Runtime endpoint.
|
|
10
|
+
* @param {string} [model] - The request's model, the deployment name unless the endpoint sets one.
|
|
11
|
+
* @returns {string} Path starting with `/`, with the `api-version` query for the classic form.
|
|
12
|
+
*/
|
|
13
|
+
export function route(kind: "chat" | "embeddings" | "models", endpoint: {
|
|
14
|
+
url: string;
|
|
15
|
+
apiVersion?: string;
|
|
16
|
+
deployment?: string;
|
|
17
|
+
}, model?: string): string;
|
|
@@ -19,3 +19,32 @@ export function isLlamacppProps(body: Record<string, any>): boolean;
|
|
|
19
19
|
* @returns {boolean} True when it names a model or tokenizer path.
|
|
20
20
|
*/
|
|
21
21
|
export function isSglangInfo(body: Record<string, any>): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Does a `GET /api/v0/models` reply come from LM Studio? The match is strict so no other server binds: `object: "list"`, a non-empty `data`
|
|
24
|
+
* array, and every entry an `object: "model"` with string `id`, `arch`, `compatibility_type` and `state` and a known `type`.
|
|
25
|
+
* @param {Record<string, any>} body - Reply object (see `objectOf`).
|
|
26
|
+
* @returns {boolean} True for LM Studio's model listing.
|
|
27
|
+
*/
|
|
28
|
+
export function isLmstudioModels(body: Record<string, any>): boolean;
|
|
29
|
+
/**
|
|
30
|
+
* Does a `GET /info` reply come from Hugging Face Text Generation Inference? The match is strict so no other server binds: the router
|
|
31
|
+
* names itself `text-generation-router` (Text Embeddings Inference answers `/info` too, as `text-embeddings-router`), and the reply
|
|
32
|
+
* carries a string `model_id` and `version` and a numeric `max_total_tokens` (older releases lack `max_input_tokens`, so it is not required).
|
|
33
|
+
* @param {Record<string, any>} body - Reply object (see `objectOf`).
|
|
34
|
+
* @returns {boolean} True for TGI's `/info`.
|
|
35
|
+
*/
|
|
36
|
+
export function isTgiInfo(body: Record<string, any>): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Does a `GET /version` reply come from LocalAI? It answers `{ version: "<version> (<commit>)" }` (the version, a space, and the build
|
|
39
|
+
* commit in parentheses, which is empty for a source build), where vLLM and SGLang answer a bare version, so the shape names LocalAI.
|
|
40
|
+
* @param {Record<string, any>} body - Reply object (see `objectOf`).
|
|
41
|
+
* @returns {boolean} True for LocalAI's version reply.
|
|
42
|
+
*/
|
|
43
|
+
export function isLocalaiVersion(body: Record<string, any>): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Does a `GET /.well-known/localai.json` reply come from LocalAI? It is LocalAI's discovery document: a string `version` and an
|
|
46
|
+
* `endpoints` map that names the OpenAI routes (`models`, `chat_completions`) by path.
|
|
47
|
+
* @param {Record<string, any>} body - Reply object (see `objectOf`).
|
|
48
|
+
* @returns {boolean} True for LocalAI's discovery document.
|
|
49
|
+
*/
|
|
50
|
+
export function isLocalaiDiscovery(body: Record<string, any>): boolean;
|
|
@@ -28,6 +28,14 @@ export type UpstreamConfig = {
|
|
|
28
28
|
* Extra headers sent to the upstream.
|
|
29
29
|
*/
|
|
30
30
|
headers?: Record<string, string>;
|
|
31
|
+
/**
|
|
32
|
+
* Azure OpenAI `api-version`. Set, it selects the classic per-deployment URLs; unset, the v1 API is used.
|
|
33
|
+
*/
|
|
34
|
+
apiVersion?: string;
|
|
35
|
+
/**
|
|
36
|
+
* Azure OpenAI deployment name for the classic URLs. Defaults to the request's model.
|
|
37
|
+
*/
|
|
38
|
+
deployment?: string;
|
|
31
39
|
/**
|
|
32
40
|
* Driver name, or `"auto"` (default) to detect.
|
|
33
41
|
*/
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How many branches a request needs: `n` when the OpenAI-front body asks for more than one choice and the router would send it to an
|
|
3
|
+
* Anthropic-protocol upstream, else 0 (the request goes the normal way: an OpenAI-protocol upstream handles `n` itself, and a value
|
|
4
|
+
* that is not an integer above 1 is left to the converter's own validation). The decision is made on the endpoint the router picks
|
|
5
|
+
* first; each branch is then routed and failed over like any request.
|
|
6
|
+
* @param {any} c - Captured translator parts (`capture`).
|
|
7
|
+
* @param {any} body - Client request body.
|
|
8
|
+
* @returns {number} Branch count, or 0 for no fan-out.
|
|
9
|
+
* @throws {TranslatorError} `invalid_request` when `n` is above {@link MAX}.
|
|
10
|
+
*/
|
|
11
|
+
export function wants(c: any, body: any): number;
|
|
12
|
+
/**
|
|
13
|
+
* Add two usage objects field by field (numbers add, nested objects such as `prompt_tokens_details` recurse, a field only one side has is kept).
|
|
14
|
+
* @param {any} left - Usage object.
|
|
15
|
+
* @param {any} right - Usage object.
|
|
16
|
+
* @returns {any} The sum.
|
|
17
|
+
*/
|
|
18
|
+
export function add(left: any, right: any): any;
|
|
19
|
+
/**
|
|
20
|
+
* Combine the replies of the branches of a non-streaming request: the first branch's reply with every branch's choice, numbered by
|
|
21
|
+
* branch, and the usage summed.
|
|
22
|
+
* @param {any[]} bodies - Chat completion bodies, one per branch, in order.
|
|
23
|
+
* @returns {any} The combined body.
|
|
24
|
+
*/
|
|
25
|
+
export function join(bodies: any[]): any;
|
|
26
|
+
/**
|
|
27
|
+
* Merge the streams of the branches into one: chunks go out as they arrive from any branch, each re-indexed to its branch and carrying
|
|
28
|
+
* the first chunk's id; the branches' own `[DONE]` sentinels and usage chunks are held back, and the stream ends with one usage chunk
|
|
29
|
+
* holding the sum (when the branches sent one) and a single `[DONE]`. A branch that sends an error chunk or throws ends the merged
|
|
30
|
+
* stream the way the stream converter ends a failed one: the error chunk, then `[DONE]`; the other branches are cancelled. Closing
|
|
31
|
+
* the merged stream early cancels every branch.
|
|
32
|
+
* @param {AsyncIterable<{event?: string, data: any}>[]} streams - Branch streams, in choice order.
|
|
33
|
+
* @param {() => void} abort - Aborts the branches' requests.
|
|
34
|
+
* @param {(index: number) => void} trip - Called with the index of the branch whose failure ended the stream.
|
|
35
|
+
* @returns {AsyncIterableIterator<{event?: string, data: any}>} Merged stream, closable with `return()`.
|
|
36
|
+
*/
|
|
37
|
+
export function weave(streams: AsyncIterable<{
|
|
38
|
+
event?: string;
|
|
39
|
+
data: any;
|
|
40
|
+
}>[], abort: () => void, trip: (index: number) => void): AsyncIterableIterator<{
|
|
41
|
+
event?: string;
|
|
42
|
+
data: any;
|
|
43
|
+
}>;
|
|
44
|
+
/**
|
|
45
|
+
* Answer a request with `count` parallel single-choice requests. Each branch is a full `send` of the body with `n: 1` (so it is routed,
|
|
46
|
+
* failed over, captured and metered like any request, and its own `onUsage` feeds the combined report). The first branch to fail
|
|
47
|
+
* (an error reply, or a thrown error such as an abort) ends the request: the others are aborted, the failure is the reply (or is
|
|
48
|
+
* rethrown), and the streams of branches that had already started are closed so their usage settles. The caller's signal cancels every
|
|
49
|
+
* branch. A branch that fails after the stream has started is handled by {@link weave}.
|
|
50
|
+
* @param {object} spec - `{ body, count, options, send, warn }`.
|
|
51
|
+
* @param {any} spec.body - Client request body (`n` above 1).
|
|
52
|
+
* @param {number} spec.count - Number of branches.
|
|
53
|
+
* @param {any} spec.options - The `send` options of the request.
|
|
54
|
+
* @param {(body: any, options: any) => Promise<any>} spec.send - The single-request send.
|
|
55
|
+
* @param {(message: string, details: {kind: "threw" | "rejected"}) => void} spec.warn - Receives a message when `onUsage` throws or rejects.
|
|
56
|
+
* @returns {Promise<any>} The reply, with a non-enumerable `usage` promise of the combined report.
|
|
57
|
+
*/
|
|
58
|
+
export function run({ body, count, options, send, warn }: {
|
|
59
|
+
body: any;
|
|
60
|
+
count: number;
|
|
61
|
+
options: any;
|
|
62
|
+
send: (body: any, options: any) => Promise<any>;
|
|
63
|
+
warn: (message: string, details: {
|
|
64
|
+
kind: "threw" | "rejected";
|
|
65
|
+
}) => void;
|
|
66
|
+
}): Promise<any>;
|
|
67
|
+
/** The largest `n` that is fanned out (OpenAI's own limit for the parameter). */
|
|
68
|
+
export const MAX: 128;
|
|
@@ -10,19 +10,30 @@ export function finishError(finishReason: string | null | undefined): {
|
|
|
10
10
|
code: string;
|
|
11
11
|
message: string;
|
|
12
12
|
} | null;
|
|
13
|
+
/**
|
|
14
|
+
* The Anthropic refusal category for the content filter annotations an OpenAI-style server put on a choice. Azure OpenAI (and Foundry) attach
|
|
15
|
+
* `content_filter_results` to each choice (and to streamed chunks): `{ hate: { filtered, severity }, sexual, violence, self_harm, ... }`.
|
|
16
|
+
* A harm category with `filtered: true` is a policy area outside Anthropic's four named ones (`cyber`, `bio`, `frontier_llm`,
|
|
17
|
+
* `reasoning_extraction`), which is what `general_harms` stands for. Everything else (jailbreak, protected material, blocklists, no
|
|
18
|
+
* annotations) gives no category and stays null. No other server the translator knows reports one (see D-50).
|
|
19
|
+
* @param {any} results - A choice's `content_filter_results`, if any.
|
|
20
|
+
* @returns {"general_harms"|null} The category, or null.
|
|
21
|
+
*/
|
|
22
|
+
export function filterCategory(results: any): "general_harms" | null;
|
|
13
23
|
/**
|
|
14
24
|
* Anthropic stop reason (and stop sequence) for an OpenAI choice.
|
|
15
25
|
* `model_length` is `max_tokens`; `eos`, `eos_token` and any other unknown value are `end_turn`. Fatal values are handled by {@link finishError} before this.
|
|
16
26
|
* @param {string|null|undefined} finishReason - OpenAI `finish_reason`.
|
|
17
|
-
* A `content_filter` stop also carries `stop_details` (`{ type: "refusal", category
|
|
18
|
-
* @param {{ hasToolCalls: boolean, matchedStop?: unknown, argumentsComplete?: boolean, refusal?: unknown }} info - Whether tool calls were produced, vLLM's `stop_reason` (a string for a matched stop string), whether the last tool call's arguments are complete,
|
|
27
|
+
* A `content_filter` stop also carries `stop_details` (`{ type: "refusal", category, explanation }`), the explanation being the upstream's refusal text when it sent one and the category what {@link filterCategory} found (else null). No other stop reason has the field.
|
|
28
|
+
* @param {{ hasToolCalls: boolean, matchedStop?: unknown, argumentsComplete?: boolean, refusal?: unknown, category?: string|null }} info - Whether tool calls were produced, vLLM's `stop_reason` (a string for a matched stop string), whether the last tool call's arguments are complete, the upstream's refusal text and the refusal category from {@link filterCategory}. A `length` stop with incomplete arguments is `max_tokens`, not `tool_use`.
|
|
19
29
|
* @returns {{ stop_reason: string, stop_sequence: string|null, stop_details?: { type: string, category: string|null, explanation: string|null } }} Anthropic stop fields.
|
|
20
30
|
*/
|
|
21
|
-
export function toAnthropicStop(finishReason: string | null | undefined, { hasToolCalls, matchedStop, argumentsComplete, refusal }: {
|
|
31
|
+
export function toAnthropicStop(finishReason: string | null | undefined, { hasToolCalls, matchedStop, argumentsComplete, refusal, category }: {
|
|
22
32
|
hasToolCalls: boolean;
|
|
23
33
|
matchedStop?: unknown;
|
|
24
34
|
argumentsComplete?: boolean;
|
|
25
35
|
refusal?: unknown;
|
|
36
|
+
category?: string | null;
|
|
26
37
|
}): {
|
|
27
38
|
stop_reason: string;
|
|
28
39
|
stop_sequence: string | null;
|
package/types/src/lib/http.d.mts
CHANGED
|
@@ -23,6 +23,17 @@
|
|
|
23
23
|
* @returns {string} Absolute URL.
|
|
24
24
|
*/
|
|
25
25
|
export function join(base: string, path: string): string;
|
|
26
|
+
/**
|
|
27
|
+
* The path (with any query) of a call on an upstream. A driver that has a `path(kind, { endpoint, model })` leaf builds it from the endpoint
|
|
28
|
+
* and the model (Azure OpenAI puts the deployment and `api-version` there); otherwise `info().paths[kind]` is used, with the standard
|
|
29
|
+
* OpenAI path for `embeddings` and `models` when the driver does not name one.
|
|
30
|
+
* @param {any} driver - Driver record (`info` and the optional `path` leaf).
|
|
31
|
+
* @param {"chat" | "messages" | "embeddings" | "models"} kind - Which call.
|
|
32
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
33
|
+
* @param {string} [model] - The upstream's model name for the request.
|
|
34
|
+
* @returns {Promise<string>} Path starting with `/`.
|
|
35
|
+
*/
|
|
36
|
+
export function pathOf(driver: any, kind: "chat" | "messages" | "embeddings" | "models", endpoint: any, model?: string): Promise<string>;
|
|
26
37
|
/**
|
|
27
38
|
* Whether a listen address only accepts connections from this machine.
|
|
28
39
|
* @param {string} host - Host or address.
|
|
@@ -32,7 +43,7 @@ export function isLoopback(host: string): boolean;
|
|
|
32
43
|
/**
|
|
33
44
|
* Build the request headers for an upstream call.
|
|
34
45
|
* @param {any} endpoint - Runtime endpoint.
|
|
35
|
-
* @param {string} [auth] - `"bearer"` (default)
|
|
46
|
+
* @param {string} [auth] - `"bearer"` (default), `"x-api-key"` or `"api-key"` (Azure OpenAI).
|
|
36
47
|
* @param {Record<string,string>} [extra] - Extra headers applied last.
|
|
37
48
|
* @returns {Record<string,string>} Header map.
|
|
38
49
|
*/
|
|
@@ -42,7 +53,7 @@ export function headers(endpoint: any, auth?: string, extra?: Record<string, str
|
|
|
42
53
|
* once and every driver that asks for it gets the same result, so a path fetched for one backend (for example `/v1/models`) is
|
|
43
54
|
* not fetched again for the next. Never rejects; treat the result as read-only.
|
|
44
55
|
*
|
|
45
|
-
* Without `driver` the request carries the upstream's bearer key. A driver whose `info()` says `auth: "x-api-key"` or carries
|
|
56
|
+
* Without `driver` the request carries the upstream's bearer key. A driver whose `info()` says `auth: "x-api-key"` or `"api-key"`, or carries
|
|
46
57
|
* `headers` passes its `info()` so the probe is sent the way the driver talks to its own backend; that request is a different
|
|
47
58
|
* request from the bearer one, so it is cached under its own key.
|
|
48
59
|
* @param {any} context - `{ endpoint, transport, signal, timeout, probes? }`.
|
|
@@ -14,10 +14,10 @@ export function modelsOf(result: {
|
|
|
14
14
|
}[] | null;
|
|
15
15
|
/**
|
|
16
16
|
* Fetch and normalize the models an upstream reports.
|
|
17
|
-
* @param {any} context - `{ endpoint, transport, signal, timeout, auth, extraHeaders }`.
|
|
17
|
+
* @param {any} context - `{ endpoint, transport, signal, timeout, auth, extraHeaders, path }`. `path` replaces `/v1/models`.
|
|
18
18
|
* @returns {Promise<{id: string, created?: number, owned_by?: string, max_model_len?: number}[] | null>} Models, or null when the upstream did not answer with a model list.
|
|
19
19
|
*/
|
|
20
|
-
export function fetchModels({ endpoint, transport, signal, timeout, auth, extraHeaders }: any): Promise<{
|
|
20
|
+
export function fetchModels({ endpoint, transport, signal, timeout, auth, extraHeaders, path }: any): Promise<{
|
|
21
21
|
id: string;
|
|
22
22
|
created?: number;
|
|
23
23
|
owned_by?: string;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The error a reply that answered HTTP 200 carries: `choices[i].error` on every choice (the provider failed while generating; the choice's
|
|
3
|
+
* `finish_reason` is `"error"` and any partial text is not used), or a top-level `error` with no choices.
|
|
4
|
+
* @param {any} body - Parsed reply body.
|
|
5
|
+
* @returns {{code: any, message: string, metadata?: any} | null} The error object, or null when the reply has none to report.
|
|
6
|
+
*/
|
|
7
|
+
export function found(body: any): {
|
|
8
|
+
code: any;
|
|
9
|
+
message: string;
|
|
10
|
+
metadata?: any;
|
|
11
|
+
} | null;
|
|
12
|
+
/**
|
|
13
|
+
* The `TranslatorError` for an OpenRouter error object. Its numeric `code` is the HTTP status when it is a client or server error, else 502; the
|
|
14
|
+
* `code` of the error is `metadata.error_type` when OpenRouter gives one, else `upstream_error`.
|
|
15
|
+
* @param {{code?: any, message: string, metadata?: any}} error - Error object.
|
|
16
|
+
* @returns {TranslatorError} Error to throw.
|
|
17
|
+
*/
|
|
18
|
+
export function failure(error: {
|
|
19
|
+
code?: any;
|
|
20
|
+
message: string;
|
|
21
|
+
metadata?: any;
|
|
22
|
+
}): TranslatorError;
|
|
23
|
+
import { TranslatorError } from "./errors.mjs";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One tool call in the OpenAI shape. Historic TGI sent the id as a number, the arguments as an object (under `parameters` in some
|
|
3
|
+
* releases) and omitted `type`; current TGI sends string arguments.
|
|
4
|
+
* @param {any} call - Tool call as TGI sent it.
|
|
5
|
+
* @param {number} [position] - Position in the call list, the id when TGI sent none. Left out for streamed fragments, whose missing id
|
|
6
|
+
* means the fragment continues the open call.
|
|
7
|
+
* @returns {any} Tool call with a string id, `type: "function"` and string `arguments`.
|
|
8
|
+
*/
|
|
9
|
+
export function toolCall(call: any, position?: number): any;
|
|
10
|
+
/**
|
|
11
|
+
* The tool calls of a message or delta as a list: historic TGI sent one object instead of an array.
|
|
12
|
+
* @param {any} value - `tool_calls` as TGI sent it.
|
|
13
|
+
* @param {boolean} [fill] - Give a call without an id its position as the id (whole answers; not streamed fragments).
|
|
14
|
+
* @returns {any[] | undefined} The list, or undefined when there is none.
|
|
15
|
+
*/
|
|
16
|
+
export function toolCalls(value: any, fill?: boolean): any[] | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* The OpenAI `finish_reason` for one TGI sent: `eos_token` and `stop_sequence` become `stop`, or `tool_calls` when the answer holds tool calls
|
|
19
|
+
* (historic TGI reported `eos_token` for those).
|
|
20
|
+
* @param {any} reason - `finish_reason` as TGI sent it.
|
|
21
|
+
* @param {boolean} hasToolCalls - Whether the answer holds tool calls.
|
|
22
|
+
* @returns {any} Finish reason.
|
|
23
|
+
*/
|
|
24
|
+
export function finish(reason: any, hasToolCalls: boolean): any;
|
|
25
|
+
/**
|
|
26
|
+
* TGI's grammar as `response_format` takes it: `{ type: "json" | "json_object" | "regex", value }` or `{ type: "json_schema", value: { name?, schema } }`.
|
|
27
|
+
* An OpenAI `json_schema` format (`{ type: "json_schema", json_schema: { name, schema, strict } }`) becomes TGI's `json_schema` with
|
|
28
|
+
* the name and schema; a bare `json_object` has no schema to constrain with, so it asks for any JSON object. Shapes TGI already
|
|
29
|
+
* understands pass through.
|
|
30
|
+
* @param {any} format - A `response_format` (or `grammar`) value.
|
|
31
|
+
* @returns {any} The format in TGI's shape.
|
|
32
|
+
*/
|
|
33
|
+
export function grammar(format: any): any;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Split typed content chunks into answer text and thinking. `{type:"text"}` chunks make the text, `{type:"thinking"}` chunks make the thinking
|
|
3
|
+
* (their `thinking` field is a list of text chunks, or a string); other chunk types are skipped.
|
|
4
|
+
* @param {any} content - Message or delta `content`.
|
|
5
|
+
* @returns {{ text: string, thinking: string } | null} The parts, or null when `content` is not an array.
|
|
6
|
+
*/
|
|
7
|
+
export function flatten(content: any): {
|
|
8
|
+
text: string;
|
|
9
|
+
thinking: string;
|
|
10
|
+
} | null;
|
|
@@ -6,6 +6,26 @@
|
|
|
6
6
|
* converter writes when the upstream reported nothing and no real reply has).
|
|
7
7
|
*/
|
|
8
8
|
export function normalize(protocol: "anthropic" | "openai" | "folded", usage: any): Record<string, number | null> | null;
|
|
9
|
+
/**
|
|
10
|
+
* Hand a report to the caller's `onUsage`. An exception or a rejection becomes a warning and never reaches the reply.
|
|
11
|
+
* @param {UsageReport} report - The report.
|
|
12
|
+
* @param {((report: UsageReport) => any) | undefined} onUsage - Caller's callback.
|
|
13
|
+
* @param {(message: string, details: {kind: "threw" | "rejected"}) => void} warn - Receives a message when `onUsage` throws or rejects.
|
|
14
|
+
* @returns {void}
|
|
15
|
+
*/
|
|
16
|
+
export function deliver(report: UsageReport, onUsage: ((report: UsageReport) => any) | undefined, warn: (message: string, details: {
|
|
17
|
+
kind: "threw" | "rejected";
|
|
18
|
+
}) => void): void;
|
|
19
|
+
/**
|
|
20
|
+
* Combine the reports of the branches of one fanned-out request (`n` above 1) into the single report the caller gets. Token fields and
|
|
21
|
+
* estimates are summed (null stays null when no branch knew the field), `reported` is true when any branch reported, `complete` only
|
|
22
|
+
* when every branch did, and the reason is the lead branch's (the one that failed first) when there is one, else the first branch that
|
|
23
|
+
* did not finish normally, else `"done"`. The request facts (`model`, `upstream`) come from the first branch that knew them.
|
|
24
|
+
* @param {UsageReport[]} reports - One report per branch, in choice order.
|
|
25
|
+
* @param {number} [lead] - Index of the branch whose failure ended the request, or -1 when none did.
|
|
26
|
+
* @returns {UsageReport} The combined report.
|
|
27
|
+
*/
|
|
28
|
+
export function combine(reports: UsageReport[], lead?: number): UsageReport;
|
|
9
29
|
/**
|
|
10
30
|
* Create the meter for one request.
|
|
11
31
|
* @param {{front: "anthropic" | "openai", requested: string | null, model: string | null, upstream: string | null, stream: boolean}} meta - Request facts; `model` and `upstream` may be filled in later.
|