@ruvector/ruvllm 2.6.2 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +144 -38
  2. package/bin/cli.js +38 -18
  3. package/dist/cjs/engine.d.ts +79 -12
  4. package/dist/cjs/engine.d.ts.map +1 -1
  5. package/dist/cjs/engine.js +264 -55
  6. package/dist/cjs/native.d.ts +54 -6
  7. package/dist/cjs/native.d.ts.map +1 -1
  8. package/dist/cjs/native.js +30 -9
  9. package/dist/cjs/types.d.ts +95 -7
  10. package/dist/cjs/types.d.ts.map +1 -1
  11. package/dist/cjs/types.js +1 -1
  12. package/dist/esm/engine.d.ts +79 -12
  13. package/dist/esm/engine.d.ts.map +1 -1
  14. package/dist/esm/engine.js +263 -54
  15. package/dist/esm/native.d.ts +54 -6
  16. package/dist/esm/native.d.ts.map +1 -1
  17. package/dist/esm/native.js +31 -10
  18. package/dist/esm/types.d.ts +95 -7
  19. package/dist/esm/types.d.ts.map +1 -1
  20. package/dist/esm/types.js +1 -1
  21. package/package.json +4 -3
  22. package/dist/cjs/rlm/controller.d.ts +0 -245
  23. package/dist/cjs/rlm/controller.d.ts.map +0 -1
  24. package/dist/cjs/rlm/controller.js +0 -554
  25. package/dist/cjs/rlm/index.d.ts +0 -62
  26. package/dist/cjs/rlm/index.d.ts.map +0 -1
  27. package/dist/cjs/rlm/index.js +0 -97
  28. package/dist/cjs/rlm/training.d.ts +0 -333
  29. package/dist/cjs/rlm/training.d.ts.map +0 -1
  30. package/dist/cjs/rlm/training.js +0 -680
  31. package/dist/cjs/rlm/types.d.ts +0 -189
  32. package/dist/cjs/rlm/types.d.ts.map +0 -1
  33. package/dist/cjs/rlm/types.js +0 -10
  34. package/dist/esm/rlm/controller.d.ts +0 -245
  35. package/dist/esm/rlm/controller.d.ts.map +0 -1
  36. package/dist/esm/rlm/controller.js +0 -550
  37. package/dist/esm/rlm/index.d.ts +0 -62
  38. package/dist/esm/rlm/index.d.ts.map +0 -1
  39. package/dist/esm/rlm/index.js +0 -71
  40. package/dist/esm/rlm/training.d.ts +0 -333
  41. package/dist/esm/rlm/training.d.ts.map +0 -1
  42. package/dist/esm/rlm/training.js +0 -672
  43. package/dist/esm/rlm/types.d.ts +0 -189
  44. package/dist/esm/rlm/types.d.ts.map +0 -1
  45. package/dist/esm/rlm/types.js +0 -9
package/README.md CHANGED
@@ -24,23 +24,94 @@ import { RuvLLM, RuvLLMConfig } from '@ruvector/ruvllm';
24
24
  const llm = new RuvLLM();
25
25
 
26
26
  // Or with custom configuration
27
- const llm = new RuvLLM({
28
- modelPath: './models/ruvltra-small-q4km.gguf',
29
- sonaEnabled: true,
30
- flashAttention: true,
31
- maxTokens: 256,
27
+ const llm = new RuvLLM({ embeddingDim: 384, learningEnabled: true });
28
+
29
+ // Routing, memory and embeddings (no model needed)
30
+ const decision = llm.route('Explain quantum computing');
31
+ llm.addMemory('RuvLLM routes queries with FastGRNN', { source: 'docs' });
32
+ const hits = llm.searchMemory('how are queries routed?', 3);
33
+ const vec = llm.embed('Explain quantum computing');
34
+ ```
35
+
36
+ ## Text Generation (GGUF)
37
+
38
+ Text comes only from a model you load. `loadModel()` runs a local GGUF file
39
+ on the native candle backend (CPU; Metal on macOS builds with the `metal`
40
+ feature):
41
+
42
+ ```typescript
43
+ const llm = new RuvLLM();
44
+ llm.loadModel('./qwen2.5-0.5b-instruct-q4_k_m.gguf', { maxContext: 4096 });
45
+
46
+ const out = llm.chat([{ role: 'user', content: 'What is the capital of France?' }], {
47
+ maxTokens: 64,
48
+ temperature: 0,
32
49
  });
50
+ // { text: 'The capital of France is Paris.', promptTokens: 15,
51
+ // completionTokens: 8, finishReason: 'stop' }
52
+ ```
33
53
 
34
- // Generate text
35
- const response = await llm.query('Explain quantum computing');
36
- console.log(response.text);
54
+ - **Architectures**: GGUF `llama`, `mistral`, `qwen2` (Qwen2.5) and `qwen3`.
55
+ Any other architecture (phi, gemma, qwen3.5, ...) fails to load with an error
56
+ naming it.
57
+ - **Tokenizer**: a `tokenizer.json` next to the GGUF, else the tokenizer
58
+ embedded in the GGUF. The embedded one works for byte-level BPE (`gpt2`)
59
+ vocabularies such as Qwen and Llama 3; GGUFs with a SentencePiece vocabulary
60
+ (Llama 2, Mistral, TinyLlama) need the `tokenizer.json`, otherwise loading
61
+ fails with `RUVLLM_TOKENIZER_MISSING`.
62
+ - **`chat()`** applies the model's chat template, read from the GGUF's
63
+ embedded `tokenizer.chat_template` (ChatML for Qwen). A `qwen2`/`qwen3` GGUF
64
+ whose template is not ChatML (e.g. DeepSeek-R1-Distill-Qwen) fails to load
65
+ with `RUVLLM_MODEL_LOAD_FAILED` rather than run under the wrong format.
66
+ - **`generate()`** continues the prompt verbatim; **`query()`** routes, then
67
+ answers as a single chat message.
68
+ - **Usage**: `promptTokens`/`completionTokens` are counted with the model's
69
+ tokenizer; `finishReason` is `stop` (end-of-sequence token or a stop
70
+ sequence) or `length` (`maxTokens` reached or the context is full).
71
+ - **Blocking**: loading and generation are synchronous and block the event
72
+ loop; run them in a worker thread if that matters.
73
+
74
+ Without a loaded model, `generate()`, `query()`, `chat()` and
75
+ `generateDetailed()` throw `RUVLLM_NO_LANGUAGE_MODEL`; they never return text
76
+ that is not model output. For demos and tests, `new RuvLLM({ allowPlaceholder:
77
+ true })` (or `RUVLLM_ALLOW_MOCK=1`) makes `generate()`/`query()` return the
78
+ labelled `PLACEHOLDER_TEXT` instead, with a one-time warning.
79
+
80
+ ### Native binary requirements
81
+
82
+ Model loading needs a native binary built with the `candle` feature;
83
+ `supportsModelLoading()` reports whether the installed one was.
84
+
85
+ | Native binary | `loadModel()` |
86
+ |---------------|---------------|
87
+ | Built from this source (`npm run build:native`) | Works |
88
+ | `@ruvector/ruvllm-<platform>` 3.x (built by the release workflow with `candle`) | Works |
89
+ | `@ruvector/ruvllm-<platform>` 2.0.x | Throws `RUVLLM_NATIVE_TOO_OLD`: they predate model loading |
90
+ | Built with `--features napi` only (no `candle`) | Throws `RUVLLM_NO_INFERENCE_BACKEND` |
91
+ | None for your platform | Throws `RUVLLM_NATIVE_UNAVAILABLE` |
92
+
93
+ To use a binary you built, point `RUVLLM_NATIVE_PATH` at it (it takes
94
+ precedence over the installed platform package):
37
95
 
38
- // Stream generation
39
- for await (const token of llm.stream('Write a haiku about Rust')) {
40
- process.stdout.write(token);
41
- }
96
+ ```bash
97
+ npm run build:native # writes ruvllm.<platform>.node (needs a Rust toolchain)
98
+ RUVLLM_NATIVE_PATH=$PWD/ruvllm.linux-x64-gnu.node node app.js
42
99
  ```
43
100
 
101
+ Until the 3.x platform packages are on npm, build the binary from source as
102
+ above, or use the ruvllm CLI (`ruvllm serve <model>`, an OpenAI-compatible
103
+ server that exits if the model fails to load).
104
+
105
+ | Error code | Meaning |
106
+ |------------|---------|
107
+ | `RUVLLM_NO_LANGUAGE_MODEL` | Generation was requested with no model loaded |
108
+ | `RUVLLM_NATIVE_UNAVAILABLE` | No native binary could be loaded |
109
+ | `RUVLLM_NATIVE_TOO_OLD` | The native binary predates model loading (2.0.x platform packages) |
110
+ | `RUVLLM_NO_INFERENCE_BACKEND` | The native binary was built without the `candle` backend |
111
+ | `RUVLLM_MODEL_NOT_FOUND` | The model path does not exist |
112
+ | `RUVLLM_MODEL_LOAD_FAILED` | The file could not be loaded (unsupported architecture, not GGUF, ...) |
113
+ | `RUVLLM_TOKENIZER_MISSING` | Weights loaded but no usable tokenizer was found |
114
+
44
115
  ## What's New in v2.5
45
116
 
46
117
  | Feature | Description |
@@ -82,9 +153,10 @@ import { simd } from '@ruvector/ruvllm/simd';
82
153
  ## CLI Usage
83
154
 
84
155
  ```bash
85
- # Query a model
86
- ruvllm query "What is machine learning?"
156
+ # Query a model (this package's `ruvllm` bin; needs --model, see above)
157
+ ruvllm query "What is machine learning?" --model ./qwen2.5-0.5b-instruct-q4_k_m.gguf
87
158
 
159
+ # The commands below are the Rust ruvllm CLI (crate ruvllm-cli)
88
160
  # Stream output
89
161
  ruvllm query --stream "Write a poem"
90
162
 
@@ -106,14 +178,27 @@ ruvllm eval --model ./models/model.gguf --subset lite --max-tasks 50
106
178
  class RuvLLM {
107
179
  constructor(config?: RuvLLMConfig);
108
180
 
109
- // Generate text
110
- query(prompt: string, params?: GenerateParams): Promise<Response>;
181
+ // Language model (see "Text Generation" above)
182
+ loadModel(path: string, options?: LoadModelOptions): LoadedModelInfo;
183
+ unloadModel(): void;
184
+ isModelLoaded(): boolean;
185
+ modelInfo(): LoadedModelInfo | null;
186
+ supportsModelLoading(): boolean; // native binary can load models
111
187
 
112
- // Stream generation
113
- stream(prompt: string, params?: GenerateParams): AsyncIterable<string>;
188
+ // Generation; all throw RUVLLM_NO_LANGUAGE_MODEL without a loaded model
189
+ chat(messages: ChatMessage[], config?: GenerationConfig): GenerationResult;
190
+ generateDetailed(prompt: string, config?: GenerationConfig): GenerationResult;
191
+ generate(prompt: string, config?: GenerationConfig): string;
192
+ query(text: string, config?: GenerationConfig): QueryResponse; // routed, + token usage
114
193
 
115
- // Load a model
116
- loadModel(path: string): Promise<void>;
194
+ // Routing, memory and embeddings
195
+ route(text: string): RoutingDecision;
196
+ addMemory(content: string, metadata?: Record<string, unknown>): MemoryId;
197
+ searchMemory(text: string, k?: number): MemoryResult[];
198
+ embed(text: string): Embedding;
199
+
200
+ // Not in this package: token streaming (StreamingGenerator chunks a
201
+ // finished generate() result), mistral-rs backends.
117
202
 
118
203
  // Get SONA learning stats
119
204
  sonaStats(): SonaStats | null;
@@ -127,25 +212,33 @@ class RuvLLM {
127
212
 
128
213
  ```typescript
129
214
  interface RuvLLMConfig {
130
- modelPath?: string; // Path to GGUF model
131
- sonaEnabled?: boolean; // Enable SONA learning (default: true)
132
- flashAttention?: boolean; // Use Flash Attention 2 (default: true)
133
- maxTokens?: number; // Max generation tokens (default: 256)
134
- temperature?: number; // Sampling temperature (default: 0.7)
135
- topP?: number; // Top-p sampling (default: 0.9)
215
+ modelPath?: string; // GGUF to load in the constructor (throws if it cannot be loaded)
216
+ modelOptions?: LoadModelOptions; // { maxContext?: number } for modelPath
217
+ allowPlaceholder?: boolean; // generate()/query() return PLACEHOLDER_TEXT instead of
218
+ // throwing when no model is loaded (default: false)
219
+ strict?: boolean; // Throw instead of warning for unsupported options (`backend`)
220
+ // ...plus routing/memory options (embeddingDim, hnswM, ...), see types.ts
136
221
  }
137
222
  ```
138
223
 
139
- ### Generate Parameters
224
+ ### Generation Config and Result
140
225
 
141
226
  ```typescript
142
- interface GenerateParams {
143
- maxTokens?: number;
144
- temperature?: number;
145
- topP?: number;
146
- topK?: number;
147
- repetitionPenalty?: number;
148
- stopSequences?: string[];
227
+ interface GenerationConfig {
228
+ maxTokens?: number; // default 256
229
+ temperature?: number; // default 0.7; 0 = greedy
230
+ topP?: number; // default 0.9, in (0, 1]
231
+ topK?: number; // default 50; 0 disables
232
+ repetitionPenalty?: number; // default 1.1
233
+ stopSequences?: string[]; // stop text is not included in the output
234
+ seed?: number; // reproducible sampling at temperature > 0
235
+ }
236
+
237
+ interface GenerationResult {
238
+ text: string;
239
+ promptTokens: number; // after the chat template, for chat()
240
+ completionTokens: number; // includes an end-of-sequence token that ended generation
241
+ finishReason: 'stop' | 'length' | 'cancelled';
149
242
  }
150
243
  ```
151
244
 
@@ -181,6 +274,9 @@ simd.rmsNorm(hidden, weights, epsilon);
181
274
 
182
275
  ## Evaluation Harness
183
276
 
277
+ > **Not in this npm package**: `EvaluationHarness` and `AblationMode` are
278
+ > not exported by `@ruvector/ruvllm`. The sketch below shows the intended API.
279
+
184
280
  Run model evaluations with SWE-Bench integration:
185
281
 
186
282
  ```typescript
@@ -243,20 +339,30 @@ await llm.loadModel('mistralai/Mistral-7B-Instruct-v0.2');
243
339
  const response = await llm.query('Write production code');
244
340
  ```
245
341
 
246
- > **Note**: mistral-rs features require the Rust backend with `mistral-rs` feature enabled. Native bindings will use mistral-rs when available.
342
+ > **Not in this npm package**: `MistralBackend` is not exported, the
343
+ > `backend` option is ignored (with a `RUVLLM_UNSUPPORTED_OPTION` warning), and
344
+ > `loadModel()` takes a local GGUF path, not a Hub id. mistral-rs serving is a
345
+ > feature of the Rust crate (`mistral-rs` feature).
247
346
 
248
347
  ## Supported Models
249
348
 
349
+ GGUF files loadable with `loadModel()` (architectures llama, mistral, qwen2, qwen3):
350
+
250
351
  - **RuvLTRA-Small** (494M) - Q4K, Q5K, Q8
251
352
  - **RuvLTRA-Medium** (3B) - Q4K, Q5K, Q8
252
- - **Qwen 2.5** (0.5B-72B)
353
+ - **Qwen 2.5 / Qwen 3** (0.5B-72B)
253
354
  - **Llama 3.x** (8B-70B)
254
355
  - **Mistral** (7B-22B)
255
- - **Phi-3** (3.8B-14B)
256
- - **Gemma-2** (2B-27B)
356
+
357
+ Phi-3, Gemma-2 and Qwen3.5 GGUFs are not supported: loading fails with an
358
+ error naming the architecture.
257
359
 
258
360
  ## Platform Support
259
361
 
362
+ The table covers routing, memory and embeddings. Model loading additionally
363
+ needs a native binary built with the `candle` feature (see "Native binary
364
+ requirements").
365
+
260
366
  | Platform | Architecture | Status |
261
367
  |----------|--------------|--------|
262
368
  | macOS | arm64 (M1-M4) | ✅ Full support |
package/bin/cli.js CHANGED
@@ -3,8 +3,8 @@
3
3
  * RuvLLM CLI - Self-learning LLM orchestration
4
4
  *
5
5
  * Usage:
6
- * ruvllm query "What is machine learning?"
7
- * ruvllm generate "Write a haiku about AI"
6
+ * ruvllm query "What is machine learning?" --model ./model.gguf
7
+ * ruvllm generate "Write a haiku about AI" --model ./model.gguf
8
8
  * ruvllm memory add "Important context"
9
9
  * ruvllm memory search "context"
10
10
  * ruvllm models list
@@ -54,33 +54,46 @@ function formatTable(data) {
54
54
  }
55
55
 
56
56
  // Commands
57
- async function runQuery(llm, text, flags) {
57
+ function generationConfig(flags) {
58
58
  const config = {};
59
59
  if (flags.temperature) config.temperature = parseFloat(flags.temperature);
60
60
  if (flags['max-tokens']) config.maxTokens = parseInt(flags['max-tokens']);
61
61
  if (flags['top-p']) config.topP = parseFloat(flags['top-p']);
62
62
  if (flags['top-k']) config.topK = parseInt(flags['top-k']);
63
+ return config;
64
+ }
63
65
 
64
- const response = llm.query(text, config);
66
+ // query/generate need a model: without --model they fail with
67
+ // RUVLLM_NO_LANGUAGE_MODEL (and the CLI exits 1).
68
+ function loadModelFromFlags(llm, flags) {
69
+ if (!flags.model || flags.model === true) return;
70
+ const options = {};
71
+ if (flags['max-context']) options.maxContext = parseInt(flags['max-context']);
72
+ const info = llm.loadModel(flags.model, options);
73
+ if (!flags.json) console.error(`Loaded ${info.name} (${info.architecture}, ${info.quantization ?? 'quantization unknown'})`);
74
+ }
75
+
76
+ async function runQuery(llm, text, flags) {
77
+ loadModelFromFlags(llm, flags);
78
+ const response = llm.query(text, generationConfig(flags));
65
79
 
66
80
  if (flags.json) {
67
81
  console.log(formatJson(response));
68
82
  } else {
69
83
  console.log('\n' + response.text);
70
- console.log(`\n--- Model: ${response.model} | Confidence: ${(response.confidence * 100).toFixed(1)}% | Latency: ${response.latencyMs.toFixed(2)}ms ---`);
84
+ console.log(`\n--- Route: ${response.model} | Confidence: ${(response.confidence * 100).toFixed(1)}% | Tokens: ${response.promptTokens}+${response.completionTokens} (${response.finishReason}) | Latency: ${response.latencyMs.toFixed(2)}ms ---`);
71
85
  }
72
86
  }
73
87
 
74
88
  async function runGenerate(llm, prompt, flags) {
75
- console.error('Warning: Built-in SIMD inference is experimental. For production use, configure an external LLM provider (Ollama, OpenAI, etc.).');
76
-
77
- const config = {};
78
- if (flags.temperature) config.temperature = parseFloat(flags.temperature);
79
- if (flags['max-tokens']) config.maxTokens = parseInt(flags['max-tokens']);
80
- if (flags['top-p']) config.topP = parseFloat(flags['top-p']);
89
+ loadModelFromFlags(llm, flags);
90
+ const config = generationConfig(flags);
81
91
 
82
- const text = llm.generate(prompt, config);
83
- console.log(text);
92
+ if (flags.json) {
93
+ console.log(formatJson(llm.generateDetailed(prompt, config)));
94
+ } else {
95
+ console.log(llm.generate(prompt, config));
96
+ }
84
97
  }
85
98
 
86
99
  async function runMemoryAdd(llm, content, flags) {
@@ -780,8 +793,9 @@ RuvLLM - Self-learning LLM Orchestration
780
793
  Usage: ruvllm <command> [options]
781
794
 
782
795
  Commands:
783
- query <text> Query the LLM with automatic routing
784
- generate <prompt> Generate text with SIMD inference
796
+ query <text> --model <gguf> Route, then answer with the model (chat template)
797
+ generate <prompt> --model <gguf>
798
+ Continue a raw prompt with the model
785
799
  route <text> Get routing decision for query
786
800
  memory add <content> Add content to memory
787
801
  memory search <query> Search memory for similar content
@@ -813,7 +827,10 @@ Training & Fine-tuning:
813
827
 
814
828
  Options:
815
829
  --json Output as JSON
816
- --temperature <float> Sampling temperature (0.0-2.0)
830
+ --model <path> GGUF file to load for query/generate (required for them;
831
+ needs a native binary with model support, see README)
832
+ --max-context <int> Context window for --model (default: 4096)
833
+ --temperature <float> Sampling temperature (0 = greedy)
817
834
  --max-tokens <int> Maximum tokens to generate
818
835
  --top-p <float> Nucleus sampling (0.0-1.0)
819
836
  --top-k <int> Top-k sampling
@@ -835,8 +852,8 @@ Available Models (from https://huggingface.co/ruv/ruvltra):
835
852
  medium RuvLTRA Medium (669MB) - General purpose
836
853
 
837
854
  Examples:
838
- ruvllm query "What is machine learning?"
839
- ruvllm generate "Write a poem about AI" --temperature 0.9
855
+ ruvllm query "What is machine learning?" --model ./qwen2.5-0.5b-instruct-q4_k_m.gguf
856
+ ruvllm generate "Once upon a time" --model ./model.gguf --max-tokens 64 --json
840
857
  ruvllm memory add "Important context" --metadata '{"type":"note"}'
841
858
  ruvllm memory search "context" --k 5
842
859
  ruvllm similarity "hello world" "hi there"
@@ -1007,6 +1024,9 @@ async function main() {
1007
1024
  }
1008
1025
  } catch (error) {
1009
1026
  console.error('Error:', error.message);
1027
+ if (error.code === 'RUVLLM_NO_LANGUAGE_MODEL') {
1028
+ console.error('Hint: pass --model <path to a GGUF file>');
1029
+ }
1010
1030
  if (flags.verbose) {
1011
1031
  console.error(error.stack);
1012
1032
  }
@@ -1,46 +1,113 @@
1
1
  /**
2
2
  * RuvLLM Engine - Main orchestrator for self-learning LLM
3
3
  */
4
- import { RuvLLMConfig, GenerationConfig, QueryResponse, RoutingDecision, MemoryResult, MemoryId, RuvLLMStats, Feedback, Embedding, BatchQueryRequest, BatchQueryResponse } from './types';
4
+ import { RuvLLMConfig, GenerationConfig, QueryResponse, RoutingDecision, MemoryResult, MemoryId, RuvLLMStats, Feedback, Embedding, BatchQueryRequest, BatchQueryResponse, ChatMessage, GenerationResult, LoadModelOptions, LoadedModelInfo } from './types';
5
+ /** Returned by generate()/query() only with `allowPlaceholder` / `RUVLLM_ALLOW_MOCK=1`. */
6
+ export declare const PLACEHOLDER_TEXT = "[RuvLLM placeholder: no language model is loaded; this text is not model output]";
5
7
  /**
6
8
  * RuvLLM - Self-learning LLM orchestrator
7
9
  *
8
- * Combines SONA adaptive learning with HNSW memory,
9
- * FastGRNN routing, and SIMD-optimized inference.
10
+ * Combines SONA adaptive learning with HNSW memory, FastGRNN routing, and
11
+ * GGUF inference through the native candle backend.
12
+ *
13
+ * Text comes only from a model loaded with `loadModel()` (or the `modelPath`
14
+ * option). Without one, `generate()`, `query()`, `chat()` and
15
+ * `generateDetailed()` throw `RUVLLM_NO_LANGUAGE_MODEL`.
10
16
  *
11
17
  * @example
12
18
  * ```typescript
13
19
  * import { RuvLLM } from '@ruvector/ruvllm';
14
20
  *
15
- * const llm = new RuvLLM({ embeddingDim: 768 });
21
+ * const llm = new RuvLLM();
22
+ * llm.loadModel('./qwen2.5-0.5b-instruct-q4_k_m.gguf');
16
23
  *
17
- * // Query with automatic routing
18
- * const response = await llm.query('What is machine learning?');
19
- * console.log(response.text);
24
+ * const out = llm.chat([{ role: 'user', content: 'What is machine learning?' }], {
25
+ * maxTokens: 128,
26
+ * });
27
+ * console.log(out.text, out.finishReason, out.completionTokens);
20
28
  *
21
- * // Provide feedback for learning
29
+ * // Routed query, answered by the loaded model
30
+ * const response = llm.query('What is machine learning?');
22
31
  * llm.feedback({ requestId: response.requestId, rating: 5 });
23
32
  * ```
24
33
  */
25
34
  export declare class RuvLLM {
26
35
  private native;
36
+ private nativeVersion;
27
37
  private config;
38
+ private allowPlaceholder;
28
39
  private fallbackState;
29
40
  /**
30
41
  * Create a new RuvLLM instance
31
42
  */
32
43
  constructor(config?: RuvLLMConfig);
33
44
  /**
34
- * Query the LLM with automatic routing
45
+ * Whether the native binary can load and run a language model. False
46
+ * without a native binary, for binaries that predate `loadModel` (the 2.0.x
47
+ * `@ruvector/ruvllm-<platform>` packages), and for binaries built without
48
+ * the `candle` inference backend.
49
+ */
50
+ supportsModelLoading(): boolean;
51
+ /** Error for a binary that has `loadModel` but no inference backend. */
52
+ private noBackendError;
53
+ /**
54
+ * Load a GGUF model (or a directory holding one) for `generate()`,
55
+ * `chat()` and `query()`. Supported GGUF architectures: llama, mistral,
56
+ * qwen2, qwen3; others fail with an error naming the architecture. The
57
+ * tokenizer comes from a `tokenizer.json` beside the file, else from the
58
+ * GGUF itself. Replaces any loaded model; if loading fails, no model is
59
+ * loaded. Synchronous: blocks the event loop while the weights load.
60
+ *
61
+ * @throws `RUVLLM_NATIVE_UNAVAILABLE` without a native binary,
62
+ * `RUVLLM_NATIVE_TOO_OLD` when the binary predates model loading,
63
+ * `RUVLLM_NO_INFERENCE_BACKEND` when it was built without `candle`,
64
+ * `RUVLLM_MODEL_NOT_FOUND` / `RUVLLM_MODEL_LOAD_FAILED` from the backend.
65
+ */
66
+ loadModel(path: string, options?: LoadModelOptions): LoadedModelInfo;
67
+ /**
68
+ * Unload the language model and free its memory
69
+ */
70
+ unloadModel(): void;
71
+ /**
72
+ * Whether a language model is loaded
73
+ */
74
+ isModelLoaded(): boolean;
75
+ /**
76
+ * The loaded language model, or null
77
+ */
78
+ modelInfo(): LoadedModelInfo | null;
79
+ /**
80
+ * Route `text`, then answer it with the loaded model (as one user message
81
+ * under the model's chat template).
82
+ *
83
+ * @throws `RUVLLM_NO_LANGUAGE_MODEL` when no model is loaded, unless
84
+ * `allowPlaceholder` is set (then `text` is `PLACEHOLDER_TEXT`).
35
85
  */
36
86
  query(text: string, config?: GenerationConfig): QueryResponse;
37
87
  /**
38
- * Generate text with SIMD-optimized inference
88
+ * Continue `prompt` verbatim with the loaded model (no chat template; use
89
+ * `chat()` for instruction-tuned models). Returns the generated text.
39
90
  *
40
- * Note: If no trained model is loaded (demo mode), returns an informational
41
- * message instead of garbled output.
91
+ * @throws `RUVLLM_NO_LANGUAGE_MODEL` when no model is loaded, unless
92
+ * `allowPlaceholder` is set (then it returns `PLACEHOLDER_TEXT`).
42
93
  */
43
94
  generate(prompt: string, config?: GenerationConfig): string;
95
+ /**
96
+ * Like `generate()`, with token counts (from the model's tokenizer) and
97
+ * the finish reason. Always throws `RUVLLM_NO_LANGUAGE_MODEL` without a
98
+ * loaded model.
99
+ */
100
+ generateDetailed(prompt: string, config?: GenerationConfig): GenerationResult;
101
+ /**
102
+ * Run the loaded model on a conversation, formatted with the model's chat
103
+ * template (ChatML for Qwen, Llama 3 / Mistral formats for those models).
104
+ * Always throws `RUVLLM_NO_LANGUAGE_MODEL` without a loaded model.
105
+ */
106
+ chat(messages: ChatMessage[], config?: GenerationConfig): GenerationResult;
107
+ private requireModel;
108
+ private noModelError;
109
+ /** Throw `RUVLLM_NO_LANGUAGE_MODEL`, or return the labelled placeholder when opted in. */
110
+ private placeholderOrThrow;
44
111
  /**
45
112
  * Get routing decision for a query
46
113
  */
@@ -1 +1 @@
1
- {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AA0CjB;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,MAAM,CAAe;IAG7B,OAAO,CAAC,aAAa,CAInB;IAEF;;OAEG;gBACS,MAAM,CAAC,EAAE,YAAY;IAajC;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,aAAa;IA4B7D;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,MAAM;IAyB3D;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAsBpC;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,YAAY,EAAE;IAmClD;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ;IAexE;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO;IAO/B;;OAEG;IACH,KAAK,IAAI,WAAW;IA8BpB;;OAEG;IACH,UAAU,IAAI,MAAM;IAOpB;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAmB9B;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IA6BhD;;OAEG;IACH,OAAO,IAAI,OAAO;IAOlB;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAO5B;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,kBAAkB;IAS1D;;OAEG;IACH,cAAc,IAAI,OAAO;CAG1B"}
1
+ {"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EAChB,MAAM,SAAS,CAAC;AAsHjB,2FAA2F;AAC3F,eAAO,MAAM,gBAAgB,qFACuD,CAAC;AAErF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,gBAAgB,CAAU;IAGlC,OAAO,CAAC,aAAa,CAInB;IAEF;;OAEG;gBACS,MAAM,CAAC,EAAE,YAAY;IA4BjC;;;;;OAKG;IACH,oBAAoB,IAAI,OAAO;IAQ/B,wEAAwE;IACxE,OAAO,CAAC,cAAc;IAUtB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,eAAe;IA2BpE;;OAEG;IACH,WAAW,IAAI,IAAI;IAInB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,SAAS,IAAI,eAAe,GAAG,IAAI;IAKnC;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,aAAa;IA8B7D;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,MAAM;IAK3D;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,gBAAgB;IAO7E;;;;OAIG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,gBAAgB;IAK1E,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,YAAY;IA0BpB,0FAA0F;IAC1F,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAsBpC;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,YAAY,EAAE;IAmClD;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ;IAexE;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO;IAO/B;;OAEG;IACH,KAAK,IAAI,WAAW;IA8BpB;;OAEG;IACH,UAAU,IAAI,MAAM;IAOpB;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAmB9B;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IA6BhD;;OAEG;IACH,OAAO,IAAI,OAAO;IAOlB;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAO5B;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,kBAAkB;IAS1D;;OAEG;IACH,cAAc,IAAI,OAAO;CAG1B"}