@kreuzberg/liter-llm-node 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -1,139 +1,113 @@
|
|
|
1
1
|
# TypeScript (Node.js)
|
|
2
2
|
|
|
3
|
-
<div
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
</a>
|
|
57
|
-
<a href="https://hex.pm/packages/liter_llm">
|
|
58
|
-
<img src="https://img.shields.io/hexpm/v/liter_llm?label=Elixir&color=007ec6" alt="Elixir" />
|
|
59
|
-
</a>
|
|
60
|
-
<a href="https://github.com/kreuzberg-dev/liter-llm/pkgs/container/liter-llm">
|
|
61
|
-
<img
|
|
62
|
-
src="https://img.shields.io/badge/Docker-007ec6?logo=docker&logoColor=white"
|
|
63
|
-
alt="Docker"
|
|
64
|
-
/>
|
|
65
|
-
</a>
|
|
66
|
-
<a href="https://github.com/kreuzberg-dev/homebrew-tap/blob/main/Formula/liter-llm.rb">
|
|
67
|
-
<img
|
|
68
|
-
src="https://img.shields.io/badge/Homebrew-007ec6?logo=homebrew&logoColor=white"
|
|
69
|
-
alt="Homebrew"
|
|
70
|
-
/>
|
|
71
|
-
</a>
|
|
72
|
-
<a href="https://github.com/kreuzberg-dev/liter-llm/tree/main/crates/liter-llm-ffi">
|
|
73
|
-
<img src="https://img.shields.io/badge/C-FFI-007ec6" alt="C FFI" />
|
|
74
|
-
</a>
|
|
75
|
-
|
|
76
|
-
<!-- Project Info -->
|
|
77
|
-
<a href="https://github.com/kreuzberg-dev/liter-llm/blob/main/LICENSE">
|
|
78
|
-
<img src="https://img.shields.io/badge/License-MIT-007ec6" alt="License" />
|
|
79
|
-
</a>
|
|
80
|
-
<a href="https://docs.liter-llm.kreuzberg.dev">
|
|
81
|
-
<img src="https://img.shields.io/badge/docs-kreuzberg.dev-007ec6" alt="Docs" />
|
|
82
|
-
</a>
|
|
3
|
+
<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0">
|
|
4
|
+
<!-- Built with -->
|
|
5
|
+
<a href="https://github.com/kreuzberg-dev/alef">
|
|
6
|
+
<img src="https://img.shields.io/badge/Bindings-alef%20%D7%90-007ec6" alt="Bindings" />
|
|
7
|
+
</a>
|
|
8
|
+
<!-- Language Bindings -->
|
|
9
|
+
<a href="https://crates.io/crates/liter-llm">
|
|
10
|
+
<img src="https://img.shields.io/crates/v/liter-llm?label=Rust&color=007ec6" alt="Rust" />
|
|
11
|
+
</a>
|
|
12
|
+
<a href="https://pypi.org/project/liter-llm/">
|
|
13
|
+
<img src="https://img.shields.io/pypi/v/liter-llm?label=Python&color=007ec6" alt="Python" />
|
|
14
|
+
</a>
|
|
15
|
+
<a href="https://www.npmjs.com/package/@kreuzberg/liter-llm-node">
|
|
16
|
+
<img src="https://img.shields.io/npm/v/@kreuzberg/liter-llm-node?label=Node.js&color=007ec6" alt="Node.js" />
|
|
17
|
+
</a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@kreuzberg/liter-llm-wasm">
|
|
19
|
+
<img src="https://img.shields.io/npm/v/@kreuzberg/liter-llm-wasm?label=WASM&color=007ec6" alt="WASM" />
|
|
20
|
+
</a>
|
|
21
|
+
<a href="https://central.sonatype.com/artifact/dev.kreuzberg.literllm/liter-llm">
|
|
22
|
+
<img src="https://img.shields.io/maven-central/v/dev.kreuzberg.literllm/liter-llm?label=Java&color=007ec6" alt="Java" />
|
|
23
|
+
</a>
|
|
24
|
+
<a href="https://github.com/kreuzberg-dev/liter-llm/tree/main/packages/go">
|
|
25
|
+
<img src="https://img.shields.io/github/v/tag/kreuzberg-dev/liter-llm?label=Go&color=007ec6" alt="Go" />
|
|
26
|
+
</a>
|
|
27
|
+
<a href="https://www.nuget.org/packages/LiterLlm">
|
|
28
|
+
<img src="https://img.shields.io/nuget/v/LiterLlm?label=C%23&color=007ec6" alt="C#" />
|
|
29
|
+
</a>
|
|
30
|
+
<a href="https://packagist.org/packages/kreuzberg-dev/liter-llm">
|
|
31
|
+
<img src="https://img.shields.io/packagist/v/kreuzberg-dev/liter-llm?label=PHP&color=007ec6" alt="PHP" />
|
|
32
|
+
</a>
|
|
33
|
+
<a href="https://rubygems.org/gems/liter_llm">
|
|
34
|
+
<img src="https://img.shields.io/gem/v/liter_llm?label=Ruby&color=007ec6" alt="Ruby" />
|
|
35
|
+
</a>
|
|
36
|
+
<a href="https://hex.pm/packages/liter_llm">
|
|
37
|
+
<img src="https://img.shields.io/hexpm/v/liter_llm?label=Elixir&color=007ec6" alt="Elixir" />
|
|
38
|
+
</a>
|
|
39
|
+
<a href="https://github.com/kreuzberg-dev/liter-llm/pkgs/container/liter-llm">
|
|
40
|
+
<img src="https://img.shields.io/badge/Docker-007ec6?logo=docker&logoColor=white" alt="Docker" />
|
|
41
|
+
</a>
|
|
42
|
+
<a href="https://github.com/kreuzberg-dev/homebrew-tap/blob/main/Formula/liter-llm.rb">
|
|
43
|
+
<img src="https://img.shields.io/badge/Homebrew-007ec6?logo=homebrew&logoColor=white" alt="Homebrew" />
|
|
44
|
+
</a>
|
|
45
|
+
<a href="https://github.com/kreuzberg-dev/liter-llm/tree/main/crates/liter-llm-ffi">
|
|
46
|
+
<img src="https://img.shields.io/badge/C-FFI-007ec6" alt="C FFI" />
|
|
47
|
+
</a>
|
|
48
|
+
|
|
49
|
+
<!-- Project Info -->
|
|
50
|
+
<a href="https://github.com/kreuzberg-dev/liter-llm/blob/main/LICENSE">
|
|
51
|
+
<img src="https://img.shields.io/badge/License-MIT-007ec6" alt="License" />
|
|
52
|
+
</a>
|
|
53
|
+
<a href="https://docs.liter-llm.kreuzberg.dev">
|
|
54
|
+
<img src="https://img.shields.io/badge/Docs-liter--llm-007ec6" alt="Docs" />
|
|
55
|
+
</a>
|
|
83
56
|
</div>
|
|
84
|
-
<div align="center" style="margin:
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
</picture>
|
|
57
|
+
<div align="center" style="margin: 24px 0 0">
|
|
58
|
+
<a href="https://kreuzberg.dev">
|
|
59
|
+
<img
|
|
60
|
+
alt="kreuzberg.dev"
|
|
61
|
+
src="https://github.com/user-attachments/assets/1b6c6ad7-3b6d-4171-b1c9-f2026cc9deb8"
|
|
62
|
+
/>
|
|
63
|
+
</a>
|
|
92
64
|
</div>
|
|
93
|
-
<div align="center" style="
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
65
|
+
<div align="center" style="display: flex; flex-wrap: wrap; gap: 12px; justify-content: center; margin: 28px 0 24px">
|
|
66
|
+
<a href="https://discord.gg/xt9WY3GnKR">
|
|
67
|
+
<img
|
|
68
|
+
height="22"
|
|
69
|
+
src="https://img.shields.io/badge/Discord-Chat-007ec6?logo=discord&logoColor=white"
|
|
70
|
+
alt="Join Discord"
|
|
71
|
+
/>
|
|
72
|
+
</a>
|
|
101
73
|
</div>
|
|
102
74
|
|
|
103
|
-
Universal LLM API client for TypeScript and Node.js. Access 143
|
|
75
|
+
Universal LLM API client for TypeScript and Node.js. Access 143 LLM providers through a single interface with native NAPI-RS bindings, async/await, streaming, tool calling, and full TypeScript type definitions.
|
|
76
|
+
|
|
77
|
+
## What This Package Provides
|
|
78
|
+
|
|
79
|
+
- **One provider surface** — chat, streaming, embeddings, images, audio, search, OCR, tools, and structured output across the provider registry.
|
|
80
|
+
- **Provider/model routing** — call models with the `provider/model` convention and keep provider-specific request code out of application paths.
|
|
81
|
+
- **Production controls** — retries, fallback, rate limits, cache layers, budgets, health checks, OpenTelemetry spans, and redacted secrets.
|
|
82
|
+
- **Same core as every binding** — Rust, Python, Node.js, Go, Java, PHP, Ruby, .NET, Elixir, WASM, Kotlin Android, Swift, Dart, Zig, and C FFI use the same Rust implementation.
|
|
83
|
+
- **Node-first TypeScript API** — NAPI-RS package with typed requests/responses and async iterables for streaming.
|
|
104
84
|
|
|
105
85
|
## Installation
|
|
106
86
|
|
|
107
87
|
### Package Installation
|
|
108
88
|
|
|
109
|
-
|
|
110
89
|
Install via one of the supported package managers:
|
|
111
90
|
|
|
112
|
-
|
|
113
91
|
**npm:**
|
|
114
92
|
|
|
115
93
|
```bash
|
|
116
|
-
npm install @kreuzberg/liter-llm
|
|
94
|
+
npm install @kreuzberg/liter-llm-node
|
|
117
95
|
```
|
|
118
96
|
|
|
119
|
-
|
|
120
97
|
**pnpm:**
|
|
121
98
|
|
|
122
99
|
```bash
|
|
123
|
-
pnpm add @kreuzberg/liter-llm
|
|
100
|
+
pnpm add @kreuzberg/liter-llm-node
|
|
124
101
|
```
|
|
125
102
|
|
|
126
|
-
|
|
127
103
|
**yarn:**
|
|
128
104
|
|
|
129
105
|
```bash
|
|
130
|
-
yarn add @kreuzberg/liter-llm
|
|
106
|
+
yarn add @kreuzberg/liter-llm-node
|
|
131
107
|
```
|
|
132
108
|
|
|
133
|
-
|
|
134
109
|
### System Requirements
|
|
135
110
|
|
|
136
|
-
|
|
137
111
|
- **Node.js 22+** required (NAPI-RS native bindings)
|
|
138
112
|
- API keys via environment variables (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`)
|
|
139
113
|
|
|
@@ -145,7 +119,6 @@ Pre-built binaries available for:
|
|
|
145
119
|
- Linux (x64)
|
|
146
120
|
- Windows (x64)
|
|
147
121
|
|
|
148
|
-
|
|
149
122
|
## Quick Start
|
|
150
123
|
|
|
151
124
|
### Basic Chat
|
|
@@ -165,7 +138,6 @@ console.log(response.choices[0].message.content);
|
|
|
165
138
|
|
|
166
139
|
### Common Use Cases
|
|
167
140
|
|
|
168
|
-
|
|
169
141
|
#### Streaming Responses
|
|
170
142
|
|
|
171
143
|
Stream tokens in real time:
|
|
@@ -179,13 +151,12 @@ const chunks = await client.chatStream({
|
|
|
179
151
|
messages: [{ role: "user", content: "Tell me a story" }],
|
|
180
152
|
});
|
|
181
153
|
|
|
182
|
-
for (const chunk of chunks) {
|
|
154
|
+
for await (const chunk of chunks) {
|
|
183
155
|
process.stdout.write(chunk.choices?.[0]?.delta?.content ?? "");
|
|
184
156
|
}
|
|
185
157
|
console.log();
|
|
186
158
|
```
|
|
187
159
|
|
|
188
|
-
|
|
189
160
|
#### Tool Calling
|
|
190
161
|
|
|
191
162
|
Define and invoke tools:
|
|
@@ -219,13 +190,11 @@ for (const call of response.choices[0]?.message?.toolCalls ?? []) {
|
|
|
219
190
|
}
|
|
220
191
|
```
|
|
221
192
|
|
|
222
|
-
|
|
223
193
|
### Next Steps
|
|
224
194
|
|
|
225
195
|
- **[Provider Registry](https://github.com/kreuzberg-dev/liter-llm/blob/main/schemas/providers.json)** - Full list of supported providers
|
|
226
196
|
- **[GitHub Repository](https://github.com/kreuzberg-dev/liter-llm)** - Source, issues, and discussions
|
|
227
197
|
|
|
228
|
-
|
|
229
198
|
## NAPI-RS Implementation Details
|
|
230
199
|
|
|
231
200
|
### Native Performance
|
|
@@ -249,10 +218,9 @@ This binding uses NAPI-RS to provide native Node.js bindings with:
|
|
|
249
218
|
- Streaming buffers are released as soon as each chunk is consumed
|
|
250
219
|
- Provider registry is compiled into the binary — no runtime disk access
|
|
251
220
|
|
|
252
|
-
|
|
253
221
|
## Features
|
|
254
222
|
|
|
255
|
-
### Supported Providers (143
|
|
223
|
+
### Supported Providers (143)
|
|
256
224
|
|
|
257
225
|
Route to any provider using the `provider/model` prefix convention:
|
|
258
226
|
|
|
@@ -272,15 +240,11 @@ Route to any provider using the `provider/model` prefix convention:
|
|
|
272
240
|
|
|
273
241
|
### Key Capabilities
|
|
274
242
|
|
|
275
|
-
- **Provider Routing** -- Single client for 143
|
|
243
|
+
- **Provider Routing** -- Single client for 143 LLM providers via `provider/model` prefix
|
|
276
244
|
- **Local LLMs** — Connect to locally-hosted models via Ollama, LM Studio, vLLM, llama.cpp, and other local inference servers
|
|
277
245
|
- **Unified API** -- Consistent `chat`, `chat_stream`, `embeddings`, `list_models` interface
|
|
278
|
-
|
|
279
246
|
- **Streaming** -- Real-time token streaming via `chat_stream`
|
|
280
|
-
|
|
281
|
-
|
|
282
247
|
- **Tool Calling** -- Function calling and tool use across all supporting providers
|
|
283
|
-
|
|
284
248
|
- **Type Safe** -- Schema-driven types compiled from JSON schemas
|
|
285
249
|
- **Secure** -- API keys never logged or serialized, managed via environment variables
|
|
286
250
|
- **Observability** -- Built-in [OpenTelemetry](https://opentelemetry.io/docs/specs/semconv/gen-ai/) with GenAI semantic conventions
|
|
@@ -295,10 +259,9 @@ Built on a compiled Rust core for speed and safety:
|
|
|
295
259
|
- **Zero-copy streaming** with SSE and AWS EventStream support
|
|
296
260
|
- **API keys** wrapped in secure memory, zeroed on drop
|
|
297
261
|
|
|
298
|
-
|
|
299
262
|
## Provider Routing
|
|
300
263
|
|
|
301
|
-
Route to 143
|
|
264
|
+
Route to 143 providers using the `provider/model` prefix convention:
|
|
302
265
|
|
|
303
266
|
```text
|
|
304
267
|
openai/gpt-4o
|
|
@@ -309,7 +272,6 @@ mistral/mistral-large-latest
|
|
|
309
272
|
|
|
310
273
|
See the [provider registry](https://github.com/kreuzberg-dev/liter-llm/blob/main/schemas/providers.json) for the full list.
|
|
311
274
|
|
|
312
|
-
|
|
313
275
|
## Proxy Server
|
|
314
276
|
|
|
315
277
|
liter-llm also ships as an OpenAI-compatible proxy server with Docker support:
|
|
@@ -318,7 +280,7 @@ liter-llm also ships as an OpenAI-compatible proxy server with Docker support:
|
|
|
318
280
|
docker run -p 4000:4000 -e LITER_LLM_MASTER_KEY=sk-your-key ghcr.io/kreuzberg-dev/liter-llm
|
|
319
281
|
```
|
|
320
282
|
|
|
321
|
-
See the [proxy server documentation](https://docs.liter-llm.kreuzberg.dev/server/proxy/) for configuration, CLI usage, and MCP integration.
|
|
283
|
+
See the [proxy server documentation](https://docs.liter-llm.kreuzberg.dev/server/proxy-server/) for configuration, CLI usage, and MCP integration.
|
|
322
284
|
|
|
323
285
|
## Documentation
|
|
324
286
|
|
|
@@ -332,8 +294,9 @@ See the [proxy server documentation](https://docs.liter-llm.kreuzberg.dev/server
|
|
|
332
294
|
- [Kreuzberg Cloud](https://github.com/kreuzberg-dev/kreuzberg-cloud) — managed extraction API with SDKs, dashboards, and observability.
|
|
333
295
|
- [kreuzcrawl](https://github.com/kreuzberg-dev/kreuzcrawl) — web crawling and scraping with HTML→Markdown and headless-Chrome fallback.
|
|
334
296
|
- [html-to-markdown](https://github.com/kreuzberg-dev/html-to-markdown) — fast, lossless HTML→Markdown engine.
|
|
297
|
+
- [liter-llm](https://github.com/kreuzberg-dev/liter-llm) — universal LLM API client with native bindings for 14 languages and 143 providers.
|
|
335
298
|
- [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — tree-sitter grammars and code-intelligence primitives.
|
|
336
|
-
- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces
|
|
299
|
+
- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces every per-language binding across the 5 polyglot repos.
|
|
337
300
|
- [Discord](https://discord.gg/xt9WY3GnKR) — community, roadmap, announcements.
|
|
338
301
|
|
|
339
302
|
## Contributing
|
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:9ebda2bedb27bc07d8a4cfcbfadd55fd159de7f875fcb63308eaee6e1c3a895d
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify --exit-code
|
|
5
5
|
/* eslint-disable */
|
|
@@ -10,9 +10,40 @@ export type JsonValue = string | number | boolean | null | JsonValue[] | { [key:
|
|
|
10
10
|
* Return all provider configs from the registry.
|
|
11
11
|
*
|
|
12
12
|
* Useful for tooling, documentation generation, or runtime enumeration.
|
|
13
|
+
* Returns the public [`ProviderConfig`] slice (without capability flags).
|
|
14
|
+
* To query capability flags for a specific provider use [`capabilities`].
|
|
13
15
|
*/
|
|
14
16
|
export declare function allProviders(): Array<ProviderConfig>;
|
|
15
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Return the capability flags for a named provider.
|
|
20
|
+
*
|
|
21
|
+
* Performs an O(n) linear scan over the embedded registry (143 entries).
|
|
22
|
+
* Returns an owned value so that bindings can box/copy it across the FFI
|
|
23
|
+
* boundary without dealing with lifetimes. `ProviderCapabilities` is `Copy`,
|
|
24
|
+
* so this is a cheap memcpy of seven `bool` fields.
|
|
25
|
+
*
|
|
26
|
+
* For unknown `provider_name` values the function returns an all-`false`
|
|
27
|
+
* sentinel so callers never need to handle `Option`.
|
|
28
|
+
*/
|
|
29
|
+
export declare function capabilities(providerName: string): ProviderCapabilities;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Assert that `current_len + incoming` does not exceed `limit`.
|
|
33
|
+
*
|
|
34
|
+
* Call this before appending `incoming` bytes to any buffer that must
|
|
35
|
+
* stay below `limit`. Returns `Err(LiterLlmError::Streaming)` on overflow
|
|
36
|
+
* and emits a `tracing::warn!` with context.
|
|
37
|
+
*/
|
|
38
|
+
export declare function checkBound(context: string, currentLen: number, incoming: number, limit: number): void;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Remove all guardrails from the global registry.
|
|
42
|
+
*
|
|
43
|
+
* Primarily useful in tests to reset state between test cases.
|
|
44
|
+
*/
|
|
45
|
+
export declare function clear(): void;
|
|
46
|
+
|
|
16
47
|
/**
|
|
17
48
|
* Calculate the estimated cost of a completion given a model name and token
|
|
18
49
|
* counts.
|
|
@@ -400,6 +431,37 @@ export interface Choice {
|
|
|
400
431
|
readonly finishReason?: FinishReason
|
|
401
432
|
}
|
|
402
433
|
|
|
434
|
+
/**
|
|
435
|
+
* A per-chunk transformation in the [`StreamPipeline`].
|
|
436
|
+
*
|
|
437
|
+
* Each middleware receives a typed chunk and returns `Ok(Some(chunk))`
|
|
438
|
+
* to pass it through (optionally modified), `Ok(None)` to drop the chunk,
|
|
439
|
+
* or `Err(e)` to propagate a stream error.
|
|
440
|
+
*
|
|
441
|
+
* The trait is object-safe so implementations can be stored in a
|
|
442
|
+
* `Vec<Box<dyn ChunkMiddleware>>` inside [`StreamPipeline`].
|
|
443
|
+
*/
|
|
444
|
+
export interface ChunkMiddleware {
|
|
445
|
+
/**
|
|
446
|
+
* Process a single chunk.
|
|
447
|
+
*
|
|
448
|
+
* - `Ok(Some(chunk))` — emit (possibly transformed) chunk.
|
|
449
|
+
* - `Ok(None)` — drop this chunk silently.
|
|
450
|
+
* - `Err(e)` — propagate as a stream error.
|
|
451
|
+
*/
|
|
452
|
+
process(chunk?: ChatCompletionChunk | undefined | null): string
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Observable state of a circuit breaker. */
|
|
456
|
+
export declare enum CircuitState {
|
|
457
|
+
/** Requests flow through normally. */
|
|
458
|
+
Closed = "Closed",
|
|
459
|
+
/** All requests are rejected; the circuit is waiting for the backoff to elapse. */
|
|
460
|
+
Open = "Open",
|
|
461
|
+
/** One probe request is allowed through to test service health. */
|
|
462
|
+
HalfOpen = "HalfOpen",
|
|
463
|
+
}
|
|
464
|
+
|
|
403
465
|
/** A single content part in a user message — text, image, document, or audio. */
|
|
404
466
|
export type ContentPart =
|
|
405
467
|
| { type: 'text'; text: string }
|
|
@@ -512,7 +574,7 @@ export interface CustomProviderConfig {
|
|
|
512
574
|
/**
|
|
513
575
|
* Default client implementation backed by `reqwest`.
|
|
514
576
|
*
|
|
515
|
-
* Sends requests to
|
|
577
|
+
* Sends requests to 143 LLM providers with automatic provider detection
|
|
516
578
|
* and per-request routing. The provider is resolved at construction time
|
|
517
579
|
* from `model_hint` (or defaults to OpenAI), but individual requests can
|
|
518
580
|
* override the provider via model name prefix (e.g. `"anthropic/claude-3-5-sonnet"`
|
|
@@ -547,6 +609,17 @@ export declare class DefaultClient {
|
|
|
547
609
|
retrieveBatch(batchId: string): Promise<BatchObject>
|
|
548
610
|
listBatches(query?: BatchListQuery | undefined | null): Promise<BatchListResponse>
|
|
549
611
|
cancelBatch(batchId: string): Promise<BatchObject>
|
|
612
|
+
fetchBatchForPolling(batchId: string): Promise<BatchObject>
|
|
613
|
+
/**
|
|
614
|
+
* Poll a batch until it reaches a terminal status (Completed, Failed, Expired, Cancelled).
|
|
615
|
+
*
|
|
616
|
+
* Uses exponential backoff with configurable initial interval, maximum interval, and backoff multiplier.
|
|
617
|
+
* Optionally supports a timeout that aborts polling if exceeded.
|
|
618
|
+
* @throws Returns `BatchWaitError::Failed` if the batch reaches a failure terminal status.
|
|
619
|
+
* Returns `BatchWaitError::Timeout` if the configured timeout is exceeded.
|
|
620
|
+
* Returns `BatchWaitError::Client` for underlying client errors.
|
|
621
|
+
*/
|
|
622
|
+
waitForBatch(batchId: string, config?: WaitForBatchConfig | undefined | null): Promise<BatchObject>
|
|
550
623
|
createResponse(req?: CreateResponseRequest | undefined | null): Promise<ResponseObject>
|
|
551
624
|
retrieveResponse(responseId: string): Promise<ResponseObject>
|
|
552
625
|
cancelResponse(responseId: string): Promise<ResponseObject>
|
|
@@ -746,6 +819,31 @@ export interface FunctionMessage {
|
|
|
746
819
|
readonly name?: string
|
|
747
820
|
}
|
|
748
821
|
|
|
822
|
+
/**
|
|
823
|
+
* Abstraction over a health probe strategy.
|
|
824
|
+
*
|
|
825
|
+
* Implementors issue a lightweight probe against `upstream` (typically a
|
|
826
|
+
* provider base URL or named identifier) and report [`HealthStatus`].
|
|
827
|
+
*/
|
|
828
|
+
export interface HealthChecker {
|
|
829
|
+
/**
|
|
830
|
+
* Probe `upstream` and return its current [`HealthStatus`].
|
|
831
|
+
*
|
|
832
|
+
* The parameter is taken by value (`String`) so that implementations can
|
|
833
|
+
* move it into the returned future without a clone, making the
|
|
834
|
+
* `'static + Send` bound on the future trivially satisfiable.
|
|
835
|
+
*/
|
|
836
|
+
check(upstream: string): Promise<string>
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** The result of a single health probe. */
|
|
840
|
+
export declare enum HealthStatus {
|
|
841
|
+
/** The probe succeeded; the upstream is reachable. */
|
|
842
|
+
Healthy = "Healthy",
|
|
843
|
+
/** The probe failed; the upstream may be down. */
|
|
844
|
+
Unhealthy = "Unhealthy",
|
|
845
|
+
}
|
|
846
|
+
|
|
749
847
|
/** A single generated image, returned as either a URL or base64 data. */
|
|
750
848
|
export interface Image {
|
|
751
849
|
/** Image URL (if response_format was "url"). */
|
|
@@ -782,6 +880,16 @@ export interface ImageUrl {
|
|
|
782
880
|
readonly detail?: ImageDetail
|
|
783
881
|
}
|
|
784
882
|
|
|
883
|
+
/** An intent prototype: `(intent_name, prototype_embedding, target_model_id)`. */
|
|
884
|
+
export interface IntentPrototype {
|
|
885
|
+
/** Human-readable name for the intent (used in logs/metrics). */
|
|
886
|
+
readonly name: string
|
|
887
|
+
/** Pre-computed embedding vector for this intent. */
|
|
888
|
+
readonly embedding: Array<number>
|
|
889
|
+
/** Model to route to when this intent is detected. */
|
|
890
|
+
readonly model: string
|
|
891
|
+
}
|
|
892
|
+
|
|
785
893
|
/** JSON Schema specification for constrained output. */
|
|
786
894
|
export interface JsonSchemaFormat {
|
|
787
895
|
/** Name of the schema (must be unique in the request). */
|
|
@@ -987,7 +1095,56 @@ export interface PromptTokensDetails {
|
|
|
987
1095
|
readonly audioTokens?: number
|
|
988
1096
|
}
|
|
989
1097
|
|
|
990
|
-
/**
|
|
1098
|
+
/**
|
|
1099
|
+
* Static capability flags for a provider.
|
|
1100
|
+
*
|
|
1101
|
+
* Each flag indicates whether the provider's models *generally* support that
|
|
1102
|
+
* feature. For providers that aggregate many underlying models (e.g. Bedrock,
|
|
1103
|
+
* OpenRouter, vLLM) the flags reflect the superset of available model
|
|
1104
|
+
* capabilities — a flag being `true` means at least one model supports the
|
|
1105
|
+
* feature, not every model.
|
|
1106
|
+
*
|
|
1107
|
+
* All flags default to `false` so that newly added providers are safe.
|
|
1108
|
+
*
|
|
1109
|
+
* Access via the crate-level [`capabilities`] function:
|
|
1110
|
+
*
|
|
1111
|
+
* ```rust
|
|
1112
|
+
* use liter_llm::capabilities;
|
|
1113
|
+
*
|
|
1114
|
+
* let caps = capabilities("openai");
|
|
1115
|
+
* assert!(caps.function_calling);
|
|
1116
|
+
* assert!(caps.vision);
|
|
1117
|
+
*
|
|
1118
|
+
* // Unknown providers return a default-all-false reference.
|
|
1119
|
+
* let unknown = capabilities("my-private-model");
|
|
1120
|
+
* assert!(!unknown.function_calling);
|
|
1121
|
+
* ```
|
|
1122
|
+
*/
|
|
1123
|
+
export interface ProviderCapabilities {
|
|
1124
|
+
/** The provider accepts image input in chat messages. */
|
|
1125
|
+
readonly vision?: boolean
|
|
1126
|
+
/** The provider supports extended-thinking / reasoning tokens. */
|
|
1127
|
+
readonly reasoning?: boolean
|
|
1128
|
+
/** The provider supports JSON-mode or `response_format` structured output. */
|
|
1129
|
+
readonly structuredOutput?: boolean
|
|
1130
|
+
/** The provider supports tool / function calling. */
|
|
1131
|
+
readonly functionCalling?: boolean
|
|
1132
|
+
/** The provider accepts audio as input. */
|
|
1133
|
+
readonly audioIn?: boolean
|
|
1134
|
+
/** The provider can generate audio / TTS output. */
|
|
1135
|
+
readonly audioOut?: boolean
|
|
1136
|
+
/** The provider accepts video as input. */
|
|
1137
|
+
readonly videoIn?: boolean
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* Static configuration for a single provider entry in providers.json.
|
|
1142
|
+
*
|
|
1143
|
+
* This struct deliberately does not include capability flags or streaming
|
|
1144
|
+
* format, which are accessed via the [`capabilities`] function. Keeping
|
|
1145
|
+
* these fields separate preserves backward compatibility with all generated
|
|
1146
|
+
* binding code that constructs `ProviderConfig` using struct literal syntax.
|
|
1147
|
+
*/
|
|
991
1148
|
export interface ProviderConfig {
|
|
992
1149
|
/** Provider identifier (matches the entry key in providers.json). */
|
|
993
1150
|
readonly name: string
|
|
@@ -1162,6 +1319,16 @@ export interface SearchResult {
|
|
|
1162
1319
|
readonly date?: string
|
|
1163
1320
|
}
|
|
1164
1321
|
|
|
1322
|
+
/**
|
|
1323
|
+
* The value broadcast from a singleflight leader to all followers.
|
|
1324
|
+
*
|
|
1325
|
+
* `Arc<LiterLlmError>` is used because `LiterLlmError` is not `Clone` and
|
|
1326
|
+
* broadcast channels require `T: Clone`. The `Arc` adds only a reference-count
|
|
1327
|
+
* bump per follower, which is negligible under the burst loads this layer targets.
|
|
1328
|
+
*/
|
|
1329
|
+
export declare class SingleflightResult {
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1165
1332
|
/** Name of the specific function to invoke. */
|
|
1166
1333
|
export interface SpecificFunction {
|
|
1167
1334
|
/** Function name. */
|
|
@@ -1208,6 +1375,21 @@ export interface StreamDelta {
|
|
|
1208
1375
|
readonly refusal?: string
|
|
1209
1376
|
}
|
|
1210
1377
|
|
|
1378
|
+
/**
|
|
1379
|
+
* The streaming wire format a provider uses for its response stream.
|
|
1380
|
+
*
|
|
1381
|
+
* Most providers use standard Server-Sent Events (SSE). AWS Bedrock uses
|
|
1382
|
+
* a proprietary binary EventStream framing.
|
|
1383
|
+
*
|
|
1384
|
+
* Deserialized from the `streaming_format` JSON field via [`serde`].
|
|
1385
|
+
*/
|
|
1386
|
+
export declare enum StreamFormat {
|
|
1387
|
+
/** Standard Server-Sent Events (text/event-stream). */
|
|
1388
|
+
Sse = "sse",
|
|
1389
|
+
/** AWS EventStream binary framing (application/vnd.amazon.eventstream). */
|
|
1390
|
+
AwsEventStream = "aws_event_stream",
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1211
1393
|
/** Partial function call details in a stream. */
|
|
1212
1394
|
export interface StreamFunctionCall {
|
|
1213
1395
|
/** Function name (typically in the first chunk). */
|
|
@@ -1347,6 +1529,23 @@ export interface UserMessage {
|
|
|
1347
1529
|
readonly name?: string
|
|
1348
1530
|
}
|
|
1349
1531
|
|
|
1532
|
+
/**
|
|
1533
|
+
* Configuration for polling a batch until terminal status.
|
|
1534
|
+
*
|
|
1535
|
+
* All time values are in seconds as `f64` so the struct bridges across FFI
|
|
1536
|
+
* boundaries without requiring a `Duration` shim.
|
|
1537
|
+
*/
|
|
1538
|
+
export interface WaitForBatchConfig {
|
|
1539
|
+
/** Initial interval between polls, in seconds. */
|
|
1540
|
+
readonly initialIntervalSecs?: number
|
|
1541
|
+
/** Maximum interval between polls (backoff plateau), in seconds. */
|
|
1542
|
+
readonly maxIntervalSecs?: number
|
|
1543
|
+
/** Exponential backoff multiplier (e.g., 1.5 increases delay by 50% each poll). */
|
|
1544
|
+
readonly backoffMultiplier?: number
|
|
1545
|
+
/** Optional timeout in seconds — polling fails if this duration is exceeded. */
|
|
1546
|
+
readonly timeoutSecs?: number
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1350
1549
|
/**
|
|
1351
1550
|
* Register a custom provider in the global runtime registry.
|
|
1352
1551
|
*
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kreuzberg/liter-llm-node",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.1",
|
|
4
4
|
"description": "Universal LLM API client with Rust-powered polyglot bindings.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
},
|
|
19
19
|
"files": ["index.js", "index.d.ts", "*.node"],
|
|
20
20
|
"optionalDependencies": {
|
|
21
|
-
"@kreuzberg/liter-llm-node-linux-x64-gnu": "1.
|
|
22
|
-
"@kreuzberg/liter-llm-node-linux-arm64-gnu": "1.
|
|
23
|
-
"@kreuzberg/liter-llm-node-darwin-x64": "1.
|
|
24
|
-
"@kreuzberg/liter-llm-node-darwin-arm64": "1.
|
|
25
|
-
"@kreuzberg/liter-llm-node-win32-x64-msvc": "1.
|
|
26
|
-
"@kreuzberg/liter-llm-node-win32-arm64-msvc": "1.
|
|
21
|
+
"@kreuzberg/liter-llm-node-linux-x64-gnu": "1.6.1",
|
|
22
|
+
"@kreuzberg/liter-llm-node-linux-arm64-gnu": "1.6.1",
|
|
23
|
+
"@kreuzberg/liter-llm-node-darwin-x64": "1.6.1",
|
|
24
|
+
"@kreuzberg/liter-llm-node-darwin-arm64": "1.6.1",
|
|
25
|
+
"@kreuzberg/liter-llm-node-win32-x64-msvc": "1.6.1",
|
|
26
|
+
"@kreuzberg/liter-llm-node-win32-arm64-msvc": "1.6.1"
|
|
27
27
|
},
|
|
28
28
|
"napi": {
|
|
29
29
|
"packageName": "@kreuzberg/liter-llm-node",
|