@cldmv/llm-translator 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/api/convert/anthropic/messages/to/ollama/chat/request.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/anthropic/messages/response.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/anthropic/messages/stream.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/openai/chat/response.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/openai/chat/stream.mjs +2 -0
- package/dist/api/convert/openai/chat/to/ollama/chat/request.mjs +2 -0
- package/dist/api/detect.mjs +1 -1
- package/dist/api/driver/fireworks/detect.mjs +2 -0
- package/dist/api/driver/fireworks/info.mjs +2 -0
- package/dist/api/driver/fireworks/path.mjs +2 -0
- package/dist/api/driver/fireworks/request.mjs +2 -0
- package/dist/api/driver/fireworks/settings.mjs +2 -0
- package/dist/api/driver/gemini/detect.mjs +2 -0
- package/dist/api/driver/gemini/info.mjs +2 -0
- package/dist/api/driver/gemini/path.mjs +2 -0
- package/dist/api/driver/gemini/request.mjs +2 -0
- package/dist/api/driver/gemini/settings.mjs +2 -0
- package/dist/api/driver/groq/detect.mjs +2 -0
- package/dist/api/driver/groq/info.mjs +2 -0
- package/dist/api/driver/groq/path.mjs +2 -0
- package/dist/api/driver/groq/request.mjs +2 -0
- package/dist/api/driver/groq/settings.mjs +2 -0
- package/dist/api/driver/ollama/info.mjs +1 -1
- package/dist/api/driver/ollama/request.mjs +1 -1
- package/dist/api/driver/together/detect.mjs +2 -0
- package/dist/api/driver/together/info.mjs +2 -0
- package/dist/api/driver/together/request.mjs +2 -0
- package/dist/api/driver/together/settings.mjs +2 -0
- package/dist/api/driver/vllm/detect.mjs +1 -1
- package/dist/api/driver/vllm/info.mjs +1 -1
- package/dist/api/driver/vllm/model/list.mjs +1 -1
- package/dist/api/driver/xai/detect.mjs +2 -0
- package/dist/api/driver/xai/info.mjs +2 -0
- package/dist/api/driver/xai/request.mjs +2 -0
- package/dist/api/driver/xai/settings.mjs +2 -0
- package/dist/api/embeddings/send.mjs +1 -1
- package/dist/api/health/check.mjs +2 -0
- package/dist/api/model/list.mjs +1 -1
- package/dist/api/request.mjs +2 -2
- package/dist/api/response.mjs +1 -1
- package/dist/api/send.mjs +1 -1
- package/dist/api/status.mjs +1 -1
- package/dist/api/stream.mjs +1 -1
- package/dist/index.mjs +1 -1
- package/dist/lib/capture.mjs +1 -1
- package/dist/lib/config.mjs +1 -1
- package/dist/lib/health.mjs +2 -0
- package/dist/lib/http.mjs +1 -1
- package/dist/lib/models.mjs +1 -1
- package/dist/lib/ndjson.mjs +4 -0
- package/dist/lib/ollama.mjs +3 -0
- package/dist/lib/pool.mjs +1 -1
- package/dist/lib/presets.mjs +2 -0
- package/dist/lib/route.mjs +1 -1
- package/dist/lib/settings.mjs +1 -1
- package/dist/lib/vllm.mjs +4 -0
- package/package.json +1 -1
- package/types/src/api/convert/anthropic/messages/to/ollama/chat/request.d.mts +14 -0
- package/types/src/api/convert/ollama/chat/to/anthropic/messages/response.d.mts +10 -0
- package/types/src/api/convert/ollama/chat/to/anthropic/messages/stream.d.mts +14 -0
- package/types/src/api/convert/ollama/chat/to/openai/chat/response.d.mts +10 -0
- package/types/src/api/convert/ollama/chat/to/openai/chat/stream.d.mts +13 -0
- package/types/src/api/convert/openai/chat/to/ollama/chat/request.d.mts +12 -0
- package/types/src/api/driver/fireworks/detect.d.mts +8 -0
- package/types/src/api/driver/fireworks/info.d.mts +13 -0
- package/types/src/api/driver/fireworks/path.d.mts +7 -0
- package/types/src/api/driver/fireworks/request.d.mts +7 -0
- package/types/src/api/driver/fireworks/settings.d.mts +5 -0
- package/types/src/api/driver/gemini/detect.d.mts +8 -0
- package/types/src/api/driver/gemini/info.d.mts +13 -0
- package/types/src/api/driver/gemini/path.d.mts +7 -0
- package/types/src/api/driver/gemini/request.d.mts +7 -0
- package/types/src/api/driver/gemini/settings.d.mts +5 -0
- package/types/src/api/driver/groq/detect.d.mts +8 -0
- package/types/src/api/driver/groq/info.d.mts +13 -0
- package/types/src/api/driver/groq/path.d.mts +7 -0
- package/types/src/api/driver/groq/request.d.mts +7 -0
- package/types/src/api/driver/groq/settings.d.mts +5 -0
- package/types/src/api/driver/ollama/info.d.mts +7 -2
- package/types/src/api/driver/together/detect.d.mts +8 -0
- package/types/src/api/driver/together/info.d.mts +13 -0
- package/types/src/api/driver/together/request.d.mts +7 -0
- package/types/src/api/driver/together/settings.d.mts +5 -0
- package/types/src/api/driver/vllm/detect.d.mts +1 -1
- package/types/src/api/driver/vllm/info.d.mts +1 -1
- package/types/src/api/driver/vllm/model/list.d.mts +3 -1
- package/types/src/api/driver/xai/detect.d.mts +8 -0
- package/types/src/api/driver/xai/info.d.mts +13 -0
- package/types/src/api/driver/xai/request.d.mts +7 -0
- package/types/src/api/driver/xai/settings.d.mts +5 -0
- package/types/src/api/endpoint/status.d.mts +5 -2
- package/types/src/api/health/check.d.mts +8 -0
- package/types/src/api/status.d.mts +1 -1
- package/types/src/lib/config.d.mts +4 -0
- package/types/src/lib/events.d.mts +2 -2
- package/types/src/lib/health.d.mts +22 -0
- package/types/src/lib/models.d.mts +25 -0
- package/types/src/lib/ndjson.d.mts +16 -0
- package/types/src/lib/ollama.d.mts +86 -0
- package/types/src/lib/pool.d.mts +30 -4
- package/types/src/lib/presets.d.mts +98 -0
- package/types/src/lib/route.d.mts +1 -1
- package/types/src/lib/settings.d.mts +5 -0
- package/types/src/lib/vllm.d.mts +47 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a Chat Completions request body to an Ollama `/api/chat` body. Ollama streams unless told not to, so `stream` is always set. Tool
|
|
3
|
+
* arguments become objects, tool results name their tool, images go in `images` as base64, `max_tokens` becomes `options.num_predict`, the
|
|
4
|
+
* sampling fields go in `options`, `response_format` becomes `format` and the effort or thinking switch becomes `think`. What Ollama has no
|
|
5
|
+
* field for (`tool_choice` other than `auto`, `n`, `logit_bias`, `logprobs`, images given by URL, ...) is left out and named in `dropped`;
|
|
6
|
+
* `tool_choice: "none"` also removes the tools, so a call the client forbade cannot happen.
|
|
7
|
+
* @param {any} body - Chat Completions request body.
|
|
8
|
+
* @param {{ dropped?: string[], sampling?: Record<string, any> }} [context] - `dropped` collects the names of what was left out; `sampling` are extension sampling fields for `options`.
|
|
9
|
+
* @returns {any} Ollama request body.
|
|
10
|
+
*/
|
|
11
|
+
export function chatBody(body: any, context?: {
|
|
12
|
+
dropped?: string[];
|
|
13
|
+
sampling?: Record<string, any>;
|
|
14
|
+
}): any;
|
|
15
|
+
/**
|
|
16
|
+
* The tool calls of an Ollama reply as Chat Completions calls. Ollama sends no id (and an object for `arguments`), so each call gets one
|
|
17
|
+
* here, unless the server sent its own.
|
|
18
|
+
* @param {any[] | undefined} calls - `message.tool_calls`.
|
|
19
|
+
* @param {number} [first] - Index of the first call (a stream numbers calls across chunks).
|
|
20
|
+
* @returns {any[]} Calls with `index`, `id`, `type` and `function` (`arguments` as JSON text).
|
|
21
|
+
*/
|
|
22
|
+
export function callsOfReply(calls: any[] | undefined, first?: number): any[];
|
|
23
|
+
/**
|
|
24
|
+
* Chat Completions `finish_reason` for a finished Ollama reply: `tool_calls` when the model called tools, `length` when it ran out of tokens, else `stop`
|
|
25
|
+
* (also for the `load` and `unload` reasons of a request that generated nothing).
|
|
26
|
+
* @param {any} reply - Ollama reply or final stream line.
|
|
27
|
+
* @param {boolean} calls - Whether the reply carried tool calls.
|
|
28
|
+
* @returns {string} `finish_reason`.
|
|
29
|
+
*/
|
|
30
|
+
export function finishOf(reply: any, calls: boolean): string;
|
|
31
|
+
/**
|
|
32
|
+
* Chat Completions usage from the counters of a finished Ollama reply.
|
|
33
|
+
* @param {any} reply - Ollama reply or final stream line.
|
|
34
|
+
* @returns {{prompt_tokens: number, completion_tokens: number, total_tokens: number} | undefined} Usage, or undefined when the reply has no counters.
|
|
35
|
+
*/
|
|
36
|
+
export function usageOf(reply: any): {
|
|
37
|
+
prompt_tokens: number;
|
|
38
|
+
completion_tokens: number;
|
|
39
|
+
total_tokens: number;
|
|
40
|
+
} | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* The same counts as Anthropic usage, which is what the OpenAI front's usage report reads from a converted reply.
|
|
43
|
+
* @param {{prompt_tokens: number, completion_tokens: number} | undefined} usage - Chat Completions usage.
|
|
44
|
+
* @returns {{input_tokens: number, output_tokens: number} | undefined} Anthropic usage.
|
|
45
|
+
*/
|
|
46
|
+
export function folded(usage: {
|
|
47
|
+
prompt_tokens: number;
|
|
48
|
+
completion_tokens: number;
|
|
49
|
+
} | undefined): {
|
|
50
|
+
input_tokens: number;
|
|
51
|
+
output_tokens: number;
|
|
52
|
+
} | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* The error an Ollama reply carries in `error` (also sent with HTTP 200 in a stream).
|
|
55
|
+
* @param {any} reply - Ollama reply or stream line.
|
|
56
|
+
* @returns {string | null} The message, or null when the reply has none.
|
|
57
|
+
*/
|
|
58
|
+
export function errorOf(reply: any): string | null;
|
|
59
|
+
/**
|
|
60
|
+
* Convert a (non-streaming) Ollama `/api/chat` reply to a Chat Completions `chat.completion`: the thinking goes in the settings' reasoning
|
|
61
|
+
* field, the tool calls get ids, `done_reason` becomes `finish_reason` and the evaluation counts become `usage`.
|
|
62
|
+
* @param {any} reply - Parsed Ollama reply.
|
|
63
|
+
* @param {{ reasoningField?: string }} [options] - `reasoningField` is where the thinking goes (default `reasoning`).
|
|
64
|
+
* @returns {any} `chat.completion` object.
|
|
65
|
+
* @throws {TranslatorError} `upstream_error` (502) when the reply is an error.
|
|
66
|
+
*/
|
|
67
|
+
export function completion(reply: any, options?: {
|
|
68
|
+
reasoningField?: string;
|
|
69
|
+
}): any;
|
|
70
|
+
/**
|
|
71
|
+
* Create the converter of an Ollama stream's lines into Chat Completions chunks. One chunker is for one reply: it keeps the completion id (so every
|
|
72
|
+
* chunk carries the same one), the tool-call numbering across lines and whether any tool was called.
|
|
73
|
+
* @param {{ model?: string, reasoningField?: string, includeUsage?: boolean }} [options] - `model` overrides the model name on the chunks; `reasoningField` is where the thinking goes (default `reasoning`); `includeUsage` (default true) adds the final usage chunk.
|
|
74
|
+
* @returns {{ line: (line: any) => any[], finished: () => boolean, usage: () => any }} `line` gives the chunks one line produces (an error line gives one `{ error }` chunk), `finished` says whether the final line was seen and `usage` is the Chat Completions usage of it.
|
|
75
|
+
*/
|
|
76
|
+
export function createChunker(options?: {
|
|
77
|
+
model?: string;
|
|
78
|
+
reasoningField?: string;
|
|
79
|
+
includeUsage?: boolean;
|
|
80
|
+
}): {
|
|
81
|
+
line: (line: any) => any[];
|
|
82
|
+
finished: () => boolean;
|
|
83
|
+
usage: () => any;
|
|
84
|
+
};
|
|
85
|
+
/** Protocol name of an endpoint that talks to Ollama's native chat API. */
|
|
86
|
+
export const PROTOCOL: "ollama.chat";
|
package/types/src/lib/pool.d.mts
CHANGED
|
@@ -5,11 +5,14 @@
|
|
|
5
5
|
* @property {{base?: number, max?: number}} [backoff] Backoff in ms for a failing endpoint (defaults `1000` and `60000`).
|
|
6
6
|
* @property {number} [connectTimeout] Milliseconds to wait for the TCP connection of a request to an endpoint with pool mates (default `10000`, `0` disables). Needs the default transport.
|
|
7
7
|
* @property {number} [headersTimeout] Milliseconds to wait for the response headers of such a request (default `0`, off). Once the headers have arrived nothing caps the body.
|
|
8
|
+
* @property {number} [healthInterval] Milliseconds a readiness probe result (see `lib/health`) is trusted before the next request that has pool mates probes again (default `10000`, `0` turns probing off). Only drivers that name a health path (vLLM: `/health`) are probed.
|
|
9
|
+
* @property {number} [healthTimeout] Milliseconds a readiness probe may take before the endpoint counts as not ready (default `2000`).
|
|
8
10
|
* @property {Record<string, "failover" | "round_robin" | "least_busy">} [models] Strategy per model, keyed by the model name after aliases.
|
|
9
11
|
*/
|
|
10
12
|
/**
|
|
11
13
|
* Fresh health record for an endpoint.
|
|
12
|
-
*
|
|
14
|
+
* `ready` is the result of the last readiness probe: `true`, `false`, or `null` when none has run (or the server has no health path); `checkedAt` is when it ran, `unsupported` is set once the server answered the probe with a 4xx (no such path), and `checking` holds a probe in flight so concurrent requests share it.
|
|
15
|
+
* @returns {{failures: number, until: number, inflight: number, requests: number, errors: number, lastError: string | null, lastFailure: number | null, lastSuccess: number | null, ready: boolean | null, checkedAt: number | null, readyReason: string | null, unsupported: boolean, checking: Promise<void> | null}} Record.
|
|
13
16
|
*/
|
|
14
17
|
export function health(): {
|
|
15
18
|
failures: number;
|
|
@@ -20,11 +23,16 @@ export function health(): {
|
|
|
20
23
|
lastError: string | null;
|
|
21
24
|
lastFailure: number | null;
|
|
22
25
|
lastSuccess: number | null;
|
|
26
|
+
ready: boolean | null;
|
|
27
|
+
checkedAt: number | null;
|
|
28
|
+
readyReason: string | null;
|
|
29
|
+
unsupported: boolean;
|
|
30
|
+
checking: Promise<void> | null;
|
|
23
31
|
};
|
|
24
32
|
/**
|
|
25
33
|
* Validate and normalize the `pool` config.
|
|
26
34
|
* @param {PoolConfig | undefined} raw - Raw config.
|
|
27
|
-
* @returns {{strategy: string, backoff: {base: number, max: number}, connectTimeout: number, headersTimeout: number, models: Record<string, string>}} Normalized config.
|
|
35
|
+
* @returns {{strategy: string, backoff: {base: number, max: number}, connectTimeout: number, headersTimeout: number, healthInterval: number, healthTimeout: number, models: Record<string, string>}} Normalized config.
|
|
28
36
|
* @throws {TranslatorError} `INVALID_CONFIG` for an unknown strategy, a bad backoff or a bad timeout.
|
|
29
37
|
*/
|
|
30
38
|
export function normalize(raw?: PoolConfig | undefined): {
|
|
@@ -35,11 +43,13 @@ export function normalize(raw?: PoolConfig | undefined): {
|
|
|
35
43
|
};
|
|
36
44
|
connectTimeout: number;
|
|
37
45
|
headersTimeout: number;
|
|
46
|
+
healthInterval: number;
|
|
47
|
+
healthTimeout: number;
|
|
38
48
|
models: Record<string, string>;
|
|
39
49
|
};
|
|
40
50
|
/**
|
|
41
51
|
* Order the endpoints that serve a model into the sequence in which they are tried: healthy endpoints first, arranged by the model's
|
|
42
|
-
* strategy, then the unhealthy ones, soonest to recover first. Every endpoint is included, so an all-unhealthy pool is still tried.
|
|
52
|
+
* strategy, then the unhealthy ones (backing off after a failure, or not ready), soonest to recover first. Every endpoint is included, so an all-unhealthy pool is still tried.
|
|
43
53
|
* @param {any} config - Normalized config (`pool`).
|
|
44
54
|
* @param {any} state - Translator state (`cursors`).
|
|
45
55
|
* @param {any[]} list - Endpoints serving the model, in configuration order.
|
|
@@ -69,10 +79,13 @@ export function succeeded(endpoint: any, now?: number): void;
|
|
|
69
79
|
* Health of an endpoint as status output shows it.
|
|
70
80
|
* @param {any} endpoint - Runtime endpoint.
|
|
71
81
|
* @param {number} [now] - Current time in ms.
|
|
72
|
-
* @returns {{healthy: boolean, failures: number, retryAt: string | null, inflight: number, requests: number, errors: number, lastError: string | null, lastFailure: string | null, lastSuccess: string | null}} Health report (times are ISO strings).
|
|
82
|
+
* @returns {{healthy: boolean, ready: boolean | null, readyReason: string | null, checkedAt: string | null, failures: number, retryAt: string | null, inflight: number, requests: number, errors: number, lastError: string | null, lastFailure: string | null, lastSuccess: string | null}} Health report (times are ISO strings). `healthy` is false while the endpoint backs off after a failure or its last readiness probe said it is not ready; `ready` is that probe's verdict (null when none ran), `readyReason` why it failed.
|
|
73
83
|
*/
|
|
74
84
|
export function describe(endpoint: any, now?: number): {
|
|
75
85
|
healthy: boolean;
|
|
86
|
+
ready: boolean | null;
|
|
87
|
+
readyReason: string | null;
|
|
88
|
+
checkedAt: string | null;
|
|
76
89
|
failures: number;
|
|
77
90
|
retryAt: string | null;
|
|
78
91
|
inflight: number;
|
|
@@ -94,6 +107,11 @@ export const TIMEOUTS: Readonly<{
|
|
|
94
107
|
connect: 10000;
|
|
95
108
|
headers: 0;
|
|
96
109
|
}>;
|
|
110
|
+
/** Default readiness probing: how long a result is trusted (ms, 0 turns probing off) and how long a probe may take (ms). */
|
|
111
|
+
export const PROBES: Readonly<{
|
|
112
|
+
interval: 10000;
|
|
113
|
+
timeout: 2000;
|
|
114
|
+
}>;
|
|
97
115
|
/**
|
|
98
116
|
* Pool config as the translator accepts it.
|
|
99
117
|
*/
|
|
@@ -117,6 +135,14 @@ export type PoolConfig = {
|
|
|
117
135
|
* Milliseconds to wait for the response headers of such a request (default `0`, off). Once the headers have arrived nothing caps the body.
|
|
118
136
|
*/
|
|
119
137
|
headersTimeout?: number;
|
|
138
|
+
/**
|
|
139
|
+
* Milliseconds a readiness probe result (see `lib/health`) is trusted before the next request that has pool mates probes again (default `10000`, `0` turns probing off). Only drivers that name a health path (vLLM: `/health`) are probed.
|
|
140
|
+
*/
|
|
141
|
+
healthInterval?: number;
|
|
142
|
+
/**
|
|
143
|
+
* Milliseconds a readiness probe may take before the endpoint counts as not ready (default `2000`).
|
|
144
|
+
*/
|
|
145
|
+
healthTimeout?: number;
|
|
120
146
|
/**
|
|
121
147
|
* Strategy per model, keyed by the model name after aliases.
|
|
122
148
|
*/
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The driver facts of a preset.
|
|
3
|
+
* @param {string} name - Preset name.
|
|
4
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
5
|
+
*/
|
|
6
|
+
export function describe(name: string): {
|
|
7
|
+
name: string;
|
|
8
|
+
protocol: string;
|
|
9
|
+
order: number;
|
|
10
|
+
auth: string;
|
|
11
|
+
strict: boolean;
|
|
12
|
+
fields?: string[];
|
|
13
|
+
paths: Record<string, string>;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Match an endpoint by host name, no network access.
|
|
17
|
+
* @param {string} name - Preset name.
|
|
18
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
19
|
+
* @returns {{host: string} | null} Facts, or null when the host is not the provider's.
|
|
20
|
+
*/
|
|
21
|
+
export function match(name: string, endpoint: any): {
|
|
22
|
+
host: string;
|
|
23
|
+
} | null;
|
|
24
|
+
/**
|
|
25
|
+
* The settings of a preset (a copy).
|
|
26
|
+
* @param {string} name - Preset name.
|
|
27
|
+
* @returns {any} Partial settings.
|
|
28
|
+
*/
|
|
29
|
+
export function settingsOf(name: string): any;
|
|
30
|
+
/**
|
|
31
|
+
* Put the provider's path root in front of a call path unless the upstream URL already ends in it, so `https://api.groq.com`,
|
|
32
|
+
* `https://api.groq.com/openai` and `https://api.groq.com/openai/v1` (the config removes the `/v1`) all reach the same URL.
|
|
33
|
+
* @param {string} name - Preset name.
|
|
34
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
35
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
36
|
+
* @returns {string} Path starting with `/`.
|
|
37
|
+
*/
|
|
38
|
+
export function route(name: string, kind: "chat" | "embeddings" | "models", endpoint: any): string;
|
|
39
|
+
/**
|
|
40
|
+
* Normalize a Chat Completions body for a preset: drop empty tool fields and the fields the provider rejects (each recorded in `state.dropped`).
|
|
41
|
+
* @param {string} name - Preset name.
|
|
42
|
+
* @param {any} body - Chat Completions request body.
|
|
43
|
+
* @param {any} [context] - `{ state }`.
|
|
44
|
+
* @returns {any} Normalized body.
|
|
45
|
+
*/
|
|
46
|
+
export function trim(name: string, body: any, context?: any): any;
|
|
47
|
+
/**
|
|
48
|
+
* @typedef {object} Preset
|
|
49
|
+
* @property {number} order - Detection order (host drivers sit ahead of the probing ones).
|
|
50
|
+
* @property {string[]} hosts - Host names the provider answers on.
|
|
51
|
+
* @property {string} [root] - Path in front of the call paths (`/openai`), added unless the upstream URL already ends in it.
|
|
52
|
+
* @property {Record<string, string>} paths - Call paths under the root.
|
|
53
|
+
* @property {any} settings - Translator settings of the provider.
|
|
54
|
+
* @property {string[]} [fields] - Extra request fields the OpenAI front's `passthroughFilter` lets through (`info().fields`).
|
|
55
|
+
* @property {string[]} [drop] - Request fields the provider rejects.
|
|
56
|
+
* @property {(body: any) => string[]} [dropWhen] - Fields the provider rejects for this request (by model).
|
|
57
|
+
* @property {boolean} [dropName] - Remove `name` from messages.
|
|
58
|
+
*/
|
|
59
|
+
/** @type {Readonly<Record<string, Preset>>} */
|
|
60
|
+
export const PRESETS: Readonly<Record<string, Preset>>;
|
|
61
|
+
export type Preset = {
|
|
62
|
+
/**
|
|
63
|
+
* - Detection order (host drivers sit ahead of the probing ones).
|
|
64
|
+
*/
|
|
65
|
+
order: number;
|
|
66
|
+
/**
|
|
67
|
+
* - Host names the provider answers on.
|
|
68
|
+
*/
|
|
69
|
+
hosts: string[];
|
|
70
|
+
/**
|
|
71
|
+
* - Path in front of the call paths (`/openai`), added unless the upstream URL already ends in it.
|
|
72
|
+
*/
|
|
73
|
+
root?: string;
|
|
74
|
+
/**
|
|
75
|
+
* - Call paths under the root.
|
|
76
|
+
*/
|
|
77
|
+
paths: Record<string, string>;
|
|
78
|
+
/**
|
|
79
|
+
* - Translator settings of the provider.
|
|
80
|
+
*/
|
|
81
|
+
settings: any;
|
|
82
|
+
/**
|
|
83
|
+
* - Extra request fields the OpenAI front's `passthroughFilter` lets through (`info().fields`).
|
|
84
|
+
*/
|
|
85
|
+
fields?: string[];
|
|
86
|
+
/**
|
|
87
|
+
* - Request fields the provider rejects.
|
|
88
|
+
*/
|
|
89
|
+
drop?: string[];
|
|
90
|
+
/**
|
|
91
|
+
* - Fields the provider rejects for this request (by model).
|
|
92
|
+
*/
|
|
93
|
+
dropWhen?: (body: any) => string[];
|
|
94
|
+
/**
|
|
95
|
+
* - Remove `name` from messages.
|
|
96
|
+
*/
|
|
97
|
+
dropName?: boolean;
|
|
98
|
+
};
|
|
@@ -15,7 +15,7 @@ export function serves(endpoint: any, model: string): boolean;
|
|
|
15
15
|
* @param {boolean} [advance] - Move the round robin cursor (false to only look).
|
|
16
16
|
* @returns {{endpoint: any, model: string | undefined, unserved: boolean, pool: any[], pin: {effort?: string, thinking?: boolean} | undefined}} Chosen endpoint, the model name to send upstream, whether
|
|
17
17
|
* no endpoint serves the model and the endpoints to try in order (the chosen one first). When nothing serves it and the fallback endpoint knows
|
|
18
|
-
* exactly one model, that model is sent instead.
|
|
18
|
+
* exactly one base model (adapters aside), that model is sent instead.
|
|
19
19
|
*/
|
|
20
20
|
export function pick(config: any, endpoints: any[], model: string | undefined, state: any, advance?: boolean): {
|
|
21
21
|
endpoint: any;
|
|
@@ -44,6 +44,7 @@ export function resolve(...layers: (Partial<Settings> | null | undefined)[]): Se
|
|
|
44
44
|
* @property {"buffer"|"stream"} toolStreaming - OpenAI to Anthropic streams: `buffer` holds each tool call until its arguments are complete and emits one whole `tool_use` block (text and thinking keep streaming and appear before the block); `stream` forwards argument fragments incrementally and holds text that arrives meanwhile until the tool blocks are done.
|
|
45
45
|
* @property {boolean} effortPassthrough - vLLM driver: send `xhigh` and `max` reasoning effort unchanged instead of clamping them to `high`.
|
|
46
46
|
* @property {boolean} passthroughFilter - Restrict bodies forwarded from the OpenAI front to an OpenAI-protocol upstream to standard Chat Completions fields plus documented vLLM extensions. Off in the library; the server turns it on unless set to false.
|
|
47
|
+
* @property {boolean | "auto"} nativeMessages - vLLM driver: send an Anthropic-front request to vLLM's own `/v1/messages` (vLLM 0.11.1 and later) with only normalisation, instead of converting it to Chat Completions. `false` (default) always converts, `true` always uses the native endpoint, `"auto"` uses it when the detected or configured version is 0.11.1 or later. A request the endpoint cannot take (a trailing assistant message, `document` blocks, deferred tools unless `toolSearch` is `native`) is converted, and so is every request once the server answers 404 to `/v1/messages`. The OpenAI front is never affected.
|
|
47
48
|
* @property {number} contextWindow - Prompt size in tokens the upstream truncates at, or 0 when unknown. Drivers whose server silently truncates (Ollama) warn once when a request is estimated to exceed it.
|
|
48
49
|
* @property {number} defaultMaxTokens - `max_tokens` used when an OpenAI request has none (Anthropic requires one).
|
|
49
50
|
*/
|
|
@@ -203,6 +204,10 @@ export type Settings = {
|
|
|
203
204
|
* - Restrict bodies forwarded from the OpenAI front to an OpenAI-protocol upstream to standard Chat Completions fields plus documented vLLM extensions. Off in the library; the server turns it on unless set to false.
|
|
204
205
|
*/
|
|
205
206
|
passthroughFilter: boolean;
|
|
207
|
+
/**
|
|
208
|
+
* - vLLM driver: send an Anthropic-front request to vLLM's own `/v1/messages` (vLLM 0.11.1 and later) with only normalisation, instead of converting it to Chat Completions. `false` (default) always converts, `true` always uses the native endpoint, `"auto"` uses it when the detected or configured version is 0.11.1 or later. A request the endpoint cannot take (a trailing assistant message, `document` blocks, deferred tools unless `toolSearch` is `native`) is converted, and so is every request once the server answers 404 to `/v1/messages`. The OpenAI front is never affected.
|
|
209
|
+
*/
|
|
210
|
+
nativeMessages: boolean | "auto";
|
|
206
211
|
/**
|
|
207
212
|
* - Prompt size in tokens the upstream truncates at, or 0 when unknown. Drivers whose server silently truncates (Ollama) warn once when a request is estimated to exceed it.
|
|
208
213
|
*/
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is the native endpoint wanted for this upstream, going by the `nativeMessages` setting and the version?
|
|
3
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
4
|
+
* @param {any} settings - Resolved settings for the endpoint.
|
|
5
|
+
* @returns {boolean} True for a vLLM upstream with `nativeMessages: true`, or `"auto"` and a known version of 0.11.1 or later (an unknown version stays on conversion).
|
|
6
|
+
*/
|
|
7
|
+
export function wants(endpoint: any, settings: any): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Why a request cannot go to the native endpoint, so it is converted instead.
|
|
10
|
+
*
|
|
11
|
+
* - `prefill`: the last message is from the assistant. The endpoint turns it into a finished assistant turn and starts a new one, where conversion continues it.
|
|
12
|
+
* - `document`: the endpoint accepts no `document` blocks; conversion extracts their text and page images.
|
|
13
|
+
* - `toolsearch`: deferred tools (`defer_loading`, `tool_reference` and `tool_addition` blocks) are expanded by the translator unless `toolSearch` is `native`.
|
|
14
|
+
* @param {any} body - Anthropic request body.
|
|
15
|
+
* @param {any} settings - Resolved settings for the endpoint.
|
|
16
|
+
* @returns {"prefill" | "document" | "toolsearch" | null} The reason, or null when the endpoint can take the request.
|
|
17
|
+
*/
|
|
18
|
+
export function unsuited(body: any, settings: any): "prefill" | "document" | "toolsearch" | null;
|
|
19
|
+
/**
|
|
20
|
+
* Normalise an Anthropic request for vLLM's `/v1/messages`: the system prompt becomes one string without the billing header, and failed
|
|
21
|
+
* tool results carry the `toolErrors` marker when that setting is on. Everything else goes through unchanged (the endpoint ignores fields it does not know).
|
|
22
|
+
* @param {any} body - Anthropic request body.
|
|
23
|
+
* @param {any} settings - Resolved settings for the endpoint.
|
|
24
|
+
* @returns {any} Body to send.
|
|
25
|
+
*/
|
|
26
|
+
export function request(body: any, settings: any): any;
|
|
27
|
+
/**
|
|
28
|
+
* Give thinking blocks that have no signature the translator's own marker (vLLM before 0.17 sends none; clients expect the field).
|
|
29
|
+
* @param {any} body - Anthropic reply body.
|
|
30
|
+
* @returns {any} The reply, with every thinking block signed.
|
|
31
|
+
*/
|
|
32
|
+
export function response(body: any): any;
|
|
33
|
+
/**
|
|
34
|
+
* Normalise the event stream of the native endpoint: a `ping` follows `message_start` (Anthropic sends one, vLLM does not), and a thinking block that ends without
|
|
35
|
+
* a `signature_delta` gets one before its `content_block_stop`.
|
|
36
|
+
* @param {AsyncIterable<{event?: string, data: any}>} source - Events of the endpoint.
|
|
37
|
+
* @returns {AsyncGenerator<{event?: string, data: any}>} Events for the client.
|
|
38
|
+
*/
|
|
39
|
+
export function stream(source: AsyncIterable<{
|
|
40
|
+
event?: string;
|
|
41
|
+
data: any;
|
|
42
|
+
}>): AsyncGenerator<{
|
|
43
|
+
event?: string;
|
|
44
|
+
data: any;
|
|
45
|
+
}>;
|
|
46
|
+
/** First vLLM release that serves `/v1/messages` (vllm-project/vllm#22627). */
|
|
47
|
+
export const MIN_VERSION: "0.11.1";
|