@ruvector/ruvllm 2.6.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -38
- package/bin/cli.js +38 -18
- package/dist/cjs/engine.d.ts +79 -12
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +264 -55
- package/dist/cjs/native.d.ts +54 -6
- package/dist/cjs/native.d.ts.map +1 -1
- package/dist/cjs/native.js +30 -9
- package/dist/cjs/types.d.ts +95 -7
- package/dist/cjs/types.d.ts.map +1 -1
- package/dist/cjs/types.js +1 -1
- package/dist/esm/engine.d.ts +79 -12
- package/dist/esm/engine.d.ts.map +1 -1
- package/dist/esm/engine.js +263 -54
- package/dist/esm/native.d.ts +54 -6
- package/dist/esm/native.d.ts.map +1 -1
- package/dist/esm/native.js +31 -10
- package/dist/esm/types.d.ts +95 -7
- package/dist/esm/types.d.ts.map +1 -1
- package/dist/esm/types.js +1 -1
- package/package.json +4 -3
- package/dist/cjs/rlm/controller.d.ts +0 -245
- package/dist/cjs/rlm/controller.d.ts.map +0 -1
- package/dist/cjs/rlm/controller.js +0 -554
- package/dist/cjs/rlm/index.d.ts +0 -62
- package/dist/cjs/rlm/index.d.ts.map +0 -1
- package/dist/cjs/rlm/index.js +0 -97
- package/dist/cjs/rlm/training.d.ts +0 -333
- package/dist/cjs/rlm/training.d.ts.map +0 -1
- package/dist/cjs/rlm/training.js +0 -680
- package/dist/cjs/rlm/types.d.ts +0 -189
- package/dist/cjs/rlm/types.d.ts.map +0 -1
- package/dist/cjs/rlm/types.js +0 -10
- package/dist/esm/rlm/controller.d.ts +0 -245
- package/dist/esm/rlm/controller.d.ts.map +0 -1
- package/dist/esm/rlm/controller.js +0 -550
- package/dist/esm/rlm/index.d.ts +0 -62
- package/dist/esm/rlm/index.d.ts.map +0 -1
- package/dist/esm/rlm/index.js +0 -71
- package/dist/esm/rlm/training.d.ts +0 -333
- package/dist/esm/rlm/training.d.ts.map +0 -1
- package/dist/esm/rlm/training.js +0 -672
- package/dist/esm/rlm/types.d.ts +0 -189
- package/dist/esm/rlm/types.d.ts.map +0 -1
- package/dist/esm/rlm/types.js +0 -9
package/README.md
CHANGED
|
@@ -24,23 +24,94 @@ import { RuvLLM, RuvLLMConfig } from '@ruvector/ruvllm';
|
|
|
24
24
|
const llm = new RuvLLM();
|
|
25
25
|
|
|
26
26
|
// Or with custom configuration
|
|
27
|
-
const llm = new RuvLLM({
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
const llm = new RuvLLM({ embeddingDim: 384, learningEnabled: true });
|
|
28
|
+
|
|
29
|
+
// Routing, memory and embeddings (no model needed)
|
|
30
|
+
const decision = llm.route('Explain quantum computing');
|
|
31
|
+
llm.addMemory('RuvLLM routes queries with FastGRNN', { source: 'docs' });
|
|
32
|
+
const hits = llm.searchMemory('how are queries routed?', 3);
|
|
33
|
+
const vec = llm.embed('Explain quantum computing');
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Text Generation (GGUF)
|
|
37
|
+
|
|
38
|
+
Text comes only from a model you load. `loadModel()` runs a local GGUF file
|
|
39
|
+
on the native candle backend (CPU; Metal on macOS builds with the `metal`
|
|
40
|
+
feature):
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
const llm = new RuvLLM();
|
|
44
|
+
llm.loadModel('./qwen2.5-0.5b-instruct-q4_k_m.gguf', { maxContext: 4096 });
|
|
45
|
+
|
|
46
|
+
const out = llm.chat([{ role: 'user', content: 'What is the capital of France?' }], {
|
|
47
|
+
maxTokens: 64,
|
|
48
|
+
temperature: 0,
|
|
32
49
|
});
|
|
50
|
+
// { text: 'The capital of France is Paris.', promptTokens: 15,
|
|
51
|
+
// completionTokens: 8, finishReason: 'stop' }
|
|
52
|
+
```
|
|
33
53
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
54
|
+
- **Architectures**: GGUF `llama`, `mistral`, `qwen2` (Qwen2.5) and `qwen3`.
|
|
55
|
+
Any other architecture (phi, gemma, qwen3.5, ...) fails to load with an error
|
|
56
|
+
naming it.
|
|
57
|
+
- **Tokenizer**: a `tokenizer.json` next to the GGUF, else the tokenizer
|
|
58
|
+
embedded in the GGUF. The embedded one works for byte-level BPE (`gpt2`)
|
|
59
|
+
vocabularies such as Qwen and Llama 3; GGUFs with a SentencePiece vocabulary
|
|
60
|
+
(Llama 2, Mistral, TinyLlama) need the `tokenizer.json`, otherwise loading
|
|
61
|
+
fails with `RUVLLM_TOKENIZER_MISSING`.
|
|
62
|
+
- **`chat()`** applies the model's chat template, read from the GGUF's
|
|
63
|
+
embedded `tokenizer.chat_template` (ChatML for Qwen). A `qwen2`/`qwen3` GGUF
|
|
64
|
+
whose template is not ChatML (e.g. DeepSeek-R1-Distill-Qwen) fails to load
|
|
65
|
+
with `RUVLLM_MODEL_LOAD_FAILED` rather than run under the wrong format.
|
|
66
|
+
- **`generate()`** continues the prompt verbatim; **`query()`** routes, then
|
|
67
|
+
answers as a single chat message.
|
|
68
|
+
- **Usage**: `promptTokens`/`completionTokens` are counted with the model's
|
|
69
|
+
tokenizer; `finishReason` is `stop` (end-of-sequence token or a stop
|
|
70
|
+
sequence) or `length` (`maxTokens` reached or the context is full).
|
|
71
|
+
- **Blocking**: loading and generation are synchronous and block the event
|
|
72
|
+
loop; run them in a worker thread if that matters.
|
|
73
|
+
|
|
74
|
+
Without a loaded model, `generate()`, `query()`, `chat()` and
|
|
75
|
+
`generateDetailed()` throw `RUVLLM_NO_LANGUAGE_MODEL`; they never return text
|
|
76
|
+
that is not model output. For demos and tests, `new RuvLLM({ allowPlaceholder:
|
|
77
|
+
true })` (or `RUVLLM_ALLOW_MOCK=1`) makes `generate()`/`query()` return the
|
|
78
|
+
labelled `PLACEHOLDER_TEXT` instead, with a one-time warning.
|
|
79
|
+
|
|
80
|
+
### Native binary requirements
|
|
81
|
+
|
|
82
|
+
Model loading needs a native binary built with the `candle` feature;
|
|
83
|
+
`supportsModelLoading()` reports whether the installed one was.
|
|
84
|
+
|
|
85
|
+
| Native binary | `loadModel()` |
|
|
86
|
+
|---------------|---------------|
|
|
87
|
+
| Built from this source (`npm run build:native`) | Works |
|
|
88
|
+
| `@ruvector/ruvllm-<platform>` 3.x (built by the release workflow with `candle`) | Works |
|
|
89
|
+
| `@ruvector/ruvllm-<platform>` 2.0.x | Throws `RUVLLM_NATIVE_TOO_OLD`: they predate model loading |
|
|
90
|
+
| Built with `--features napi` only (no `candle`) | Throws `RUVLLM_NO_INFERENCE_BACKEND` |
|
|
91
|
+
| None for your platform | Throws `RUVLLM_NATIVE_UNAVAILABLE` |
|
|
92
|
+
|
|
93
|
+
To use a binary you built, point `RUVLLM_NATIVE_PATH` at it (it takes
|
|
94
|
+
precedence over the installed platform package):
|
|
37
95
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}
|
|
96
|
+
```bash
|
|
97
|
+
npm run build:native # writes ruvllm.<platform>.node (needs a Rust toolchain)
|
|
98
|
+
RUVLLM_NATIVE_PATH=$PWD/ruvllm.linux-x64-gnu.node node app.js
|
|
42
99
|
```
|
|
43
100
|
|
|
101
|
+
Until the 3.x platform packages are on npm, build the binary from source as
|
|
102
|
+
above, or use the ruvllm CLI (`ruvllm serve <model>`, an OpenAI-compatible
|
|
103
|
+
server that exits if the model fails to load).
|
|
104
|
+
|
|
105
|
+
| Error code | Meaning |
|
|
106
|
+
|------------|---------|
|
|
107
|
+
| `RUVLLM_NO_LANGUAGE_MODEL` | Generation was requested with no model loaded |
|
|
108
|
+
| `RUVLLM_NATIVE_UNAVAILABLE` | No native binary could be loaded |
|
|
109
|
+
| `RUVLLM_NATIVE_TOO_OLD` | The native binary predates model loading (2.0.x platform packages) |
|
|
110
|
+
| `RUVLLM_NO_INFERENCE_BACKEND` | The native binary was built without the `candle` backend |
|
|
111
|
+
| `RUVLLM_MODEL_NOT_FOUND` | The model path does not exist |
|
|
112
|
+
| `RUVLLM_MODEL_LOAD_FAILED` | The file could not be loaded (unsupported architecture, not GGUF, ...) |
|
|
113
|
+
| `RUVLLM_TOKENIZER_MISSING` | Weights loaded but no usable tokenizer was found |
|
|
114
|
+
|
|
44
115
|
## What's New in v2.5
|
|
45
116
|
|
|
46
117
|
| Feature | Description |
|
|
@@ -82,9 +153,10 @@ import { simd } from '@ruvector/ruvllm/simd';
|
|
|
82
153
|
## CLI Usage
|
|
83
154
|
|
|
84
155
|
```bash
|
|
85
|
-
# Query a model
|
|
86
|
-
ruvllm query "What is machine learning?"
|
|
156
|
+
# Query a model (this package's `ruvllm` bin; needs --model, see above)
|
|
157
|
+
ruvllm query "What is machine learning?" --model ./qwen2.5-0.5b-instruct-q4_k_m.gguf
|
|
87
158
|
|
|
159
|
+
# The commands below are the Rust ruvllm CLI (crate ruvllm-cli)
|
|
88
160
|
# Stream output
|
|
89
161
|
ruvllm query --stream "Write a poem"
|
|
90
162
|
|
|
@@ -106,14 +178,27 @@ ruvllm eval --model ./models/model.gguf --subset lite --max-tasks 50
|
|
|
106
178
|
class RuvLLM {
|
|
107
179
|
constructor(config?: RuvLLMConfig);
|
|
108
180
|
|
|
109
|
-
//
|
|
110
|
-
|
|
181
|
+
// Language model (see "Text Generation" above)
|
|
182
|
+
loadModel(path: string, options?: LoadModelOptions): LoadedModelInfo;
|
|
183
|
+
unloadModel(): void;
|
|
184
|
+
isModelLoaded(): boolean;
|
|
185
|
+
modelInfo(): LoadedModelInfo | null;
|
|
186
|
+
supportsModelLoading(): boolean; // native binary can load models
|
|
111
187
|
|
|
112
|
-
//
|
|
113
|
-
|
|
188
|
+
// Generation; all throw RUVLLM_NO_LANGUAGE_MODEL without a loaded model
|
|
189
|
+
chat(messages: ChatMessage[], config?: GenerationConfig): GenerationResult;
|
|
190
|
+
generateDetailed(prompt: string, config?: GenerationConfig): GenerationResult;
|
|
191
|
+
generate(prompt: string, config?: GenerationConfig): string;
|
|
192
|
+
query(text: string, config?: GenerationConfig): QueryResponse; // routed, + token usage
|
|
114
193
|
|
|
115
|
-
//
|
|
116
|
-
|
|
194
|
+
// Routing, memory and embeddings
|
|
195
|
+
route(text: string): RoutingDecision;
|
|
196
|
+
addMemory(content: string, metadata?: Record<string, unknown>): MemoryId;
|
|
197
|
+
searchMemory(text: string, k?: number): MemoryResult[];
|
|
198
|
+
embed(text: string): Embedding;
|
|
199
|
+
|
|
200
|
+
// Not in this package: token streaming (StreamingGenerator chunks a
|
|
201
|
+
// finished generate() result), mistral-rs backends.
|
|
117
202
|
|
|
118
203
|
// Get SONA learning stats
|
|
119
204
|
sonaStats(): SonaStats | null;
|
|
@@ -127,25 +212,33 @@ class RuvLLM {
|
|
|
127
212
|
|
|
128
213
|
```typescript
|
|
129
214
|
interface RuvLLMConfig {
|
|
130
|
-
modelPath?: string;
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
215
|
+
modelPath?: string; // GGUF to load in the constructor (throws if it cannot be loaded)
|
|
216
|
+
modelOptions?: LoadModelOptions; // { maxContext?: number } for modelPath
|
|
217
|
+
allowPlaceholder?: boolean; // generate()/query() return PLACEHOLDER_TEXT instead of
|
|
218
|
+
// throwing when no model is loaded (default: false)
|
|
219
|
+
strict?: boolean; // Throw instead of warning for unsupported options (`backend`)
|
|
220
|
+
// ...plus routing/memory options (embeddingDim, hnswM, ...), see types.ts
|
|
136
221
|
}
|
|
137
222
|
```
|
|
138
223
|
|
|
139
|
-
###
|
|
224
|
+
### Generation Config and Result
|
|
140
225
|
|
|
141
226
|
```typescript
|
|
142
|
-
interface
|
|
143
|
-
maxTokens?: number;
|
|
144
|
-
temperature?: number;
|
|
145
|
-
topP?: number;
|
|
146
|
-
topK?: number;
|
|
147
|
-
repetitionPenalty?: number;
|
|
148
|
-
stopSequences?: string[];
|
|
227
|
+
interface GenerationConfig {
|
|
228
|
+
maxTokens?: number; // default 256
|
|
229
|
+
temperature?: number; // default 0.7; 0 = greedy
|
|
230
|
+
topP?: number; // default 0.9, in (0, 1]
|
|
231
|
+
topK?: number; // default 50; 0 disables
|
|
232
|
+
repetitionPenalty?: number; // default 1.1
|
|
233
|
+
stopSequences?: string[]; // stop text is not included in the output
|
|
234
|
+
seed?: number; // reproducible sampling at temperature > 0
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
interface GenerationResult {
|
|
238
|
+
text: string;
|
|
239
|
+
promptTokens: number; // after the chat template, for chat()
|
|
240
|
+
completionTokens: number; // includes an end-of-sequence token that ended generation
|
|
241
|
+
finishReason: 'stop' | 'length' | 'cancelled';
|
|
149
242
|
}
|
|
150
243
|
```
|
|
151
244
|
|
|
@@ -181,6 +274,9 @@ simd.rmsNorm(hidden, weights, epsilon);
|
|
|
181
274
|
|
|
182
275
|
## Evaluation Harness
|
|
183
276
|
|
|
277
|
+
> **Not in this npm package**: `EvaluationHarness` and `AblationMode` are
|
|
278
|
+
> not exported by `@ruvector/ruvllm`. The sketch below shows the intended API.
|
|
279
|
+
|
|
184
280
|
Run model evaluations with SWE-Bench integration:
|
|
185
281
|
|
|
186
282
|
```typescript
|
|
@@ -243,20 +339,30 @@ await llm.loadModel('mistralai/Mistral-7B-Instruct-v0.2');
|
|
|
243
339
|
const response = await llm.query('Write production code');
|
|
244
340
|
```
|
|
245
341
|
|
|
246
|
-
> **
|
|
342
|
+
> **Not in this npm package**: `MistralBackend` is not exported, the
|
|
343
|
+
> `backend` option is ignored (with a `RUVLLM_UNSUPPORTED_OPTION` warning), and
|
|
344
|
+
> `loadModel()` takes a local GGUF path, not a Hub id. mistral-rs serving is a
|
|
345
|
+
> feature of the Rust crate (`mistral-rs` feature).
|
|
247
346
|
|
|
248
347
|
## Supported Models
|
|
249
348
|
|
|
349
|
+
GGUF files loadable with `loadModel()` (architectures llama, mistral, qwen2, qwen3):
|
|
350
|
+
|
|
250
351
|
- **RuvLTRA-Small** (494M) - Q4K, Q5K, Q8
|
|
251
352
|
- **RuvLTRA-Medium** (3B) - Q4K, Q5K, Q8
|
|
252
|
-
- **Qwen 2.5** (0.5B-72B)
|
|
353
|
+
- **Qwen 2.5 / Qwen 3** (0.5B-72B)
|
|
253
354
|
- **Llama 3.x** (8B-70B)
|
|
254
355
|
- **Mistral** (7B-22B)
|
|
255
|
-
|
|
256
|
-
-
|
|
356
|
+
|
|
357
|
+
Phi-3, Gemma-2 and Qwen3.5 GGUFs are not supported: loading fails with an
|
|
358
|
+
error naming the architecture.
|
|
257
359
|
|
|
258
360
|
## Platform Support
|
|
259
361
|
|
|
362
|
+
The table covers routing, memory and embeddings. Model loading additionally
|
|
363
|
+
needs a native binary built with the `candle` feature (see "Native binary
|
|
364
|
+
requirements").
|
|
365
|
+
|
|
260
366
|
| Platform | Architecture | Status |
|
|
261
367
|
|----------|--------------|--------|
|
|
262
368
|
| macOS | arm64 (M1-M4) | ✅ Full support |
|
package/bin/cli.js
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* RuvLLM CLI - Self-learning LLM orchestration
|
|
4
4
|
*
|
|
5
5
|
* Usage:
|
|
6
|
-
* ruvllm query "What is machine learning?"
|
|
7
|
-
* ruvllm generate "Write a haiku about AI"
|
|
6
|
+
* ruvllm query "What is machine learning?" --model ./model.gguf
|
|
7
|
+
* ruvllm generate "Write a haiku about AI" --model ./model.gguf
|
|
8
8
|
* ruvllm memory add "Important context"
|
|
9
9
|
* ruvllm memory search "context"
|
|
10
10
|
* ruvllm models list
|
|
@@ -54,33 +54,46 @@ function formatTable(data) {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
// Commands
|
|
57
|
-
|
|
57
|
+
function generationConfig(flags) {
|
|
58
58
|
const config = {};
|
|
59
59
|
if (flags.temperature) config.temperature = parseFloat(flags.temperature);
|
|
60
60
|
if (flags['max-tokens']) config.maxTokens = parseInt(flags['max-tokens']);
|
|
61
61
|
if (flags['top-p']) config.topP = parseFloat(flags['top-p']);
|
|
62
62
|
if (flags['top-k']) config.topK = parseInt(flags['top-k']);
|
|
63
|
+
return config;
|
|
64
|
+
}
|
|
63
65
|
|
|
64
|
-
|
|
66
|
+
// query/generate need a model: without --model they fail with
|
|
67
|
+
// RUVLLM_NO_LANGUAGE_MODEL (and the CLI exits 1).
|
|
68
|
+
function loadModelFromFlags(llm, flags) {
|
|
69
|
+
if (!flags.model || flags.model === true) return;
|
|
70
|
+
const options = {};
|
|
71
|
+
if (flags['max-context']) options.maxContext = parseInt(flags['max-context']);
|
|
72
|
+
const info = llm.loadModel(flags.model, options);
|
|
73
|
+
if (!flags.json) console.error(`Loaded ${info.name} (${info.architecture}, ${info.quantization ?? 'quantization unknown'})`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function runQuery(llm, text, flags) {
|
|
77
|
+
loadModelFromFlags(llm, flags);
|
|
78
|
+
const response = llm.query(text, generationConfig(flags));
|
|
65
79
|
|
|
66
80
|
if (flags.json) {
|
|
67
81
|
console.log(formatJson(response));
|
|
68
82
|
} else {
|
|
69
83
|
console.log('\n' + response.text);
|
|
70
|
-
console.log(`\n---
|
|
84
|
+
console.log(`\n--- Route: ${response.model} | Confidence: ${(response.confidence * 100).toFixed(1)}% | Tokens: ${response.promptTokens}+${response.completionTokens} (${response.finishReason}) | Latency: ${response.latencyMs.toFixed(2)}ms ---`);
|
|
71
85
|
}
|
|
72
86
|
}
|
|
73
87
|
|
|
74
88
|
async function runGenerate(llm, prompt, flags) {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const config = {};
|
|
78
|
-
if (flags.temperature) config.temperature = parseFloat(flags.temperature);
|
|
79
|
-
if (flags['max-tokens']) config.maxTokens = parseInt(flags['max-tokens']);
|
|
80
|
-
if (flags['top-p']) config.topP = parseFloat(flags['top-p']);
|
|
89
|
+
loadModelFromFlags(llm, flags);
|
|
90
|
+
const config = generationConfig(flags);
|
|
81
91
|
|
|
82
|
-
|
|
83
|
-
|
|
92
|
+
if (flags.json) {
|
|
93
|
+
console.log(formatJson(llm.generateDetailed(prompt, config)));
|
|
94
|
+
} else {
|
|
95
|
+
console.log(llm.generate(prompt, config));
|
|
96
|
+
}
|
|
84
97
|
}
|
|
85
98
|
|
|
86
99
|
async function runMemoryAdd(llm, content, flags) {
|
|
@@ -780,8 +793,9 @@ RuvLLM - Self-learning LLM Orchestration
|
|
|
780
793
|
Usage: ruvllm <command> [options]
|
|
781
794
|
|
|
782
795
|
Commands:
|
|
783
|
-
query <text>
|
|
784
|
-
generate <prompt>
|
|
796
|
+
query <text> --model <gguf> Route, then answer with the model (chat template)
|
|
797
|
+
generate <prompt> --model <gguf>
|
|
798
|
+
Continue a raw prompt with the model
|
|
785
799
|
route <text> Get routing decision for query
|
|
786
800
|
memory add <content> Add content to memory
|
|
787
801
|
memory search <query> Search memory for similar content
|
|
@@ -813,7 +827,10 @@ Training & Fine-tuning:
|
|
|
813
827
|
|
|
814
828
|
Options:
|
|
815
829
|
--json Output as JSON
|
|
816
|
-
--
|
|
830
|
+
--model <path> GGUF file to load for query/generate (required for them;
|
|
831
|
+
needs a native binary with model support, see README)
|
|
832
|
+
--max-context <int> Context window for --model (default: 4096)
|
|
833
|
+
--temperature <float> Sampling temperature (0 = greedy)
|
|
817
834
|
--max-tokens <int> Maximum tokens to generate
|
|
818
835
|
--top-p <float> Nucleus sampling (0.0-1.0)
|
|
819
836
|
--top-k <int> Top-k sampling
|
|
@@ -835,8 +852,8 @@ Available Models (from https://huggingface.co/ruv/ruvltra):
|
|
|
835
852
|
medium RuvLTRA Medium (669MB) - General purpose
|
|
836
853
|
|
|
837
854
|
Examples:
|
|
838
|
-
ruvllm query "What is machine learning?"
|
|
839
|
-
ruvllm generate "
|
|
855
|
+
ruvllm query "What is machine learning?" --model ./qwen2.5-0.5b-instruct-q4_k_m.gguf
|
|
856
|
+
ruvllm generate "Once upon a time" --model ./model.gguf --max-tokens 64 --json
|
|
840
857
|
ruvllm memory add "Important context" --metadata '{"type":"note"}'
|
|
841
858
|
ruvllm memory search "context" --k 5
|
|
842
859
|
ruvllm similarity "hello world" "hi there"
|
|
@@ -1007,6 +1024,9 @@ async function main() {
|
|
|
1007
1024
|
}
|
|
1008
1025
|
} catch (error) {
|
|
1009
1026
|
console.error('Error:', error.message);
|
|
1027
|
+
if (error.code === 'RUVLLM_NO_LANGUAGE_MODEL') {
|
|
1028
|
+
console.error('Hint: pass --model <path to a GGUF file>');
|
|
1029
|
+
}
|
|
1010
1030
|
if (flags.verbose) {
|
|
1011
1031
|
console.error(error.stack);
|
|
1012
1032
|
}
|
package/dist/cjs/engine.d.ts
CHANGED
|
@@ -1,46 +1,113 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* RuvLLM Engine - Main orchestrator for self-learning LLM
|
|
3
3
|
*/
|
|
4
|
-
import { RuvLLMConfig, GenerationConfig, QueryResponse, RoutingDecision, MemoryResult, MemoryId, RuvLLMStats, Feedback, Embedding, BatchQueryRequest, BatchQueryResponse } from './types';
|
|
4
|
+
import { RuvLLMConfig, GenerationConfig, QueryResponse, RoutingDecision, MemoryResult, MemoryId, RuvLLMStats, Feedback, Embedding, BatchQueryRequest, BatchQueryResponse, ChatMessage, GenerationResult, LoadModelOptions, LoadedModelInfo } from './types';
|
|
5
|
+
/** Returned by generate()/query() only with `allowPlaceholder` / `RUVLLM_ALLOW_MOCK=1`. */
|
|
6
|
+
export declare const PLACEHOLDER_TEXT = "[RuvLLM placeholder: no language model is loaded; this text is not model output]";
|
|
5
7
|
/**
|
|
6
8
|
* RuvLLM - Self-learning LLM orchestrator
|
|
7
9
|
*
|
|
8
|
-
* Combines SONA adaptive learning with HNSW memory,
|
|
9
|
-
*
|
|
10
|
+
* Combines SONA adaptive learning with HNSW memory, FastGRNN routing, and
|
|
11
|
+
* GGUF inference through the native candle backend.
|
|
12
|
+
*
|
|
13
|
+
* Text comes only from a model loaded with `loadModel()` (or the `modelPath`
|
|
14
|
+
* option). Without one, `generate()`, `query()`, `chat()` and
|
|
15
|
+
* `generateDetailed()` throw `RUVLLM_NO_LANGUAGE_MODEL`.
|
|
10
16
|
*
|
|
11
17
|
* @example
|
|
12
18
|
* ```typescript
|
|
13
19
|
* import { RuvLLM } from '@ruvector/ruvllm';
|
|
14
20
|
*
|
|
15
|
-
* const llm = new RuvLLM(
|
|
21
|
+
* const llm = new RuvLLM();
|
|
22
|
+
* llm.loadModel('./qwen2.5-0.5b-instruct-q4_k_m.gguf');
|
|
16
23
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
24
|
+
* const out = llm.chat([{ role: 'user', content: 'What is machine learning?' }], {
|
|
25
|
+
* maxTokens: 128,
|
|
26
|
+
* });
|
|
27
|
+
* console.log(out.text, out.finishReason, out.completionTokens);
|
|
20
28
|
*
|
|
21
|
-
* //
|
|
29
|
+
* // Routed query, answered by the loaded model
|
|
30
|
+
* const response = llm.query('What is machine learning?');
|
|
22
31
|
* llm.feedback({ requestId: response.requestId, rating: 5 });
|
|
23
32
|
* ```
|
|
24
33
|
*/
|
|
25
34
|
export declare class RuvLLM {
|
|
26
35
|
private native;
|
|
36
|
+
private nativeVersion;
|
|
27
37
|
private config;
|
|
38
|
+
private allowPlaceholder;
|
|
28
39
|
private fallbackState;
|
|
29
40
|
/**
|
|
30
41
|
* Create a new RuvLLM instance
|
|
31
42
|
*/
|
|
32
43
|
constructor(config?: RuvLLMConfig);
|
|
33
44
|
/**
|
|
34
|
-
*
|
|
45
|
+
* Whether the native binary can load and run a language model. False
|
|
46
|
+
* without a native binary, for binaries that predate `loadModel` (the 2.0.x
|
|
47
|
+
* `@ruvector/ruvllm-<platform>` packages), and for binaries built without
|
|
48
|
+
* the `candle` inference backend.
|
|
49
|
+
*/
|
|
50
|
+
supportsModelLoading(): boolean;
|
|
51
|
+
/** Error for a binary that has `loadModel` but no inference backend. */
|
|
52
|
+
private noBackendError;
|
|
53
|
+
/**
|
|
54
|
+
* Load a GGUF model (or a directory holding one) for `generate()`,
|
|
55
|
+
* `chat()` and `query()`. Supported GGUF architectures: llama, mistral,
|
|
56
|
+
* qwen2, qwen3; others fail with an error naming the architecture. The
|
|
57
|
+
* tokenizer comes from a `tokenizer.json` beside the file, else from the
|
|
58
|
+
* GGUF itself. Replaces any loaded model; if loading fails, no model is
|
|
59
|
+
* loaded. Synchronous: blocks the event loop while the weights load.
|
|
60
|
+
*
|
|
61
|
+
* @throws `RUVLLM_NATIVE_UNAVAILABLE` without a native binary,
|
|
62
|
+
* `RUVLLM_NATIVE_TOO_OLD` when the binary predates model loading,
|
|
63
|
+
* `RUVLLM_NO_INFERENCE_BACKEND` when it was built without `candle`,
|
|
64
|
+
* `RUVLLM_MODEL_NOT_FOUND` / `RUVLLM_MODEL_LOAD_FAILED` from the backend.
|
|
65
|
+
*/
|
|
66
|
+
loadModel(path: string, options?: LoadModelOptions): LoadedModelInfo;
|
|
67
|
+
/**
|
|
68
|
+
* Unload the language model and free its memory
|
|
69
|
+
*/
|
|
70
|
+
unloadModel(): void;
|
|
71
|
+
/**
|
|
72
|
+
* Whether a language model is loaded
|
|
73
|
+
*/
|
|
74
|
+
isModelLoaded(): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* The loaded language model, or null
|
|
77
|
+
*/
|
|
78
|
+
modelInfo(): LoadedModelInfo | null;
|
|
79
|
+
/**
|
|
80
|
+
* Route `text`, then answer it with the loaded model (as one user message
|
|
81
|
+
* under the model's chat template).
|
|
82
|
+
*
|
|
83
|
+
* @throws `RUVLLM_NO_LANGUAGE_MODEL` when no model is loaded, unless
|
|
84
|
+
* `allowPlaceholder` is set (then `text` is `PLACEHOLDER_TEXT`).
|
|
35
85
|
*/
|
|
36
86
|
query(text: string, config?: GenerationConfig): QueryResponse;
|
|
37
87
|
/**
|
|
38
|
-
*
|
|
88
|
+
* Continue `prompt` verbatim with the loaded model (no chat template; use
|
|
89
|
+
* `chat()` for instruction-tuned models). Returns the generated text.
|
|
39
90
|
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
91
|
+
* @throws `RUVLLM_NO_LANGUAGE_MODEL` when no model is loaded, unless
|
|
92
|
+
* `allowPlaceholder` is set (then it returns `PLACEHOLDER_TEXT`).
|
|
42
93
|
*/
|
|
43
94
|
generate(prompt: string, config?: GenerationConfig): string;
|
|
95
|
+
/**
|
|
96
|
+
* Like `generate()`, with token counts (from the model's tokenizer) and
|
|
97
|
+
* the finish reason. Always throws `RUVLLM_NO_LANGUAGE_MODEL` without a
|
|
98
|
+
* loaded model.
|
|
99
|
+
*/
|
|
100
|
+
generateDetailed(prompt: string, config?: GenerationConfig): GenerationResult;
|
|
101
|
+
/**
|
|
102
|
+
* Run the loaded model on a conversation, formatted with the model's chat
|
|
103
|
+
* template (ChatML for Qwen, Llama 3 / Mistral formats for those models).
|
|
104
|
+
* Always throws `RUVLLM_NO_LANGUAGE_MODEL` without a loaded model.
|
|
105
|
+
*/
|
|
106
|
+
chat(messages: ChatMessage[], config?: GenerationConfig): GenerationResult;
|
|
107
|
+
private requireModel;
|
|
108
|
+
private noModelError;
|
|
109
|
+
/** Throw `RUVLLM_NO_LANGUAGE_MODEL`, or return the labelled placeholder when opted in. */
|
|
110
|
+
private placeholderOrThrow;
|
|
44
111
|
/**
|
|
45
112
|
* Get routing decision for a query
|
|
46
113
|
*/
|
package/dist/cjs/engine.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,kBAAkB,
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EAChB,MAAM,SAAS,CAAC;AAsHjB,2FAA2F;AAC3F,eAAO,MAAM,gBAAgB,qFACuD,CAAC;AAErF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,gBAAgB,CAAU;IAGlC,OAAO,CAAC,aAAa,CAInB;IAEF;;OAEG;gBACS,MAAM,CAAC,EAAE,YAAY;IA4BjC;;;;;OAKG;IACH,oBAAoB,IAAI,OAAO;IAQ/B,wEAAwE;IACxE,OAAO,CAAC,cAAc;IAUtB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,eAAe;IA2BpE;;OAEG;IACH,WAAW,IAAI,IAAI;IAInB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,SAAS,IAAI,eAAe,GAAG,IAAI;IAKnC;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,aAAa;IA8B7D;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,MAAM;IAK3D;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,gBAAgB;IAO7E;;;;OAIG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,gBAAgB;IAK1E,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,YAAY;IA0BpB,0FAA0F;IAC1F,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAsBpC;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,YAAY,EAAE;IAmClD;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ;IAexE;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO;IAO/B;;OAEG;IACH,KAAK,IAAI,WAAW;IA8BpB;;OAEG;IACH,UAAU,IAAI,MAAM;IAOpB;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAmB9B;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IA6BhD;;OAEG;IACH,OAAO,IAAI,OAAO;IAOlB;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAO5B;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,kBAAkB;IAS1D;;OAEG;IACH,cAAc,IAAI,OAAO;CAG1B"}
|