@fugood/buttress-server 2.25.0-beta.70 → 2.25.0-beta.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -9
- package/config/sample.toml +9 -0
- package/lib/index.mjs +67 -67
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -163,7 +163,7 @@ Most ggml-llm `[generators.model]` keys can also live in `[runtime]` as defaults
|
|
|
163
163
|
| `no_extra_bufts` | boolean | Disable extra compute buffer types |
|
|
164
164
|
| `cpu_mask`, `cpu_strict` | string / boolean | CPU affinity (advanced) |
|
|
165
165
|
| `devices` | string[] | Restrict to specific GGML devices |
|
|
166
|
-
| Speculative keys | various | `speculative`, `spec_type`, `spec_draft_n_max/n_min/p_min/p_split
|
|
166
|
+
| Speculative keys | various | `speculative`, `spec_type`, `spec_draft_n_max/n_min/p_min/p_split`, plus draft-model GPU/cache settings |
|
|
167
167
|
|
|
168
168
|
### `[autodiscover]`
|
|
169
169
|
|
|
@@ -257,14 +257,23 @@ Loads a GGUF LLM. Runtime keys above can be overridden per-generator under `[gen
|
|
|
257
257
|
|
|
258
258
|
**Speculative decoding**
|
|
259
259
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
|
266
|
-
|
|
|
267
|
-
| `
|
|
260
|
+
`model_draft` may be a direct URL/path string or a table with the same
|
|
261
|
+
`repo_id`/`filename`/`url`/`local_path` model keys described above. Local draft paths require
|
|
262
|
+
`allow_local_file = true`. When `download = true`, Buttress pre-downloads both target and draft
|
|
263
|
+
models and includes both in its memory plan.
|
|
264
|
+
|
|
265
|
+
| Key | Type | Notes |
|
|
266
|
+
| -------------------------- | --------------- | ------------------------------------------------------------ |
|
|
267
|
+
| `model_draft` | string \| table | Optional separate GGUF draft model |
|
|
268
|
+
| `speculative` | bool \| string \| table | Enable speculative decoding and optionally select a strategy |
|
|
269
|
+
| `spec_type` | string | Strategy, such as `"draft-mtp"` |
|
|
270
|
+
| `spec_draft_n_max` | int | Max drafted tokens per step |
|
|
271
|
+
| `spec_draft_n_min` | int | Min drafted tokens |
|
|
272
|
+
| `spec_draft_p_min` | number | Min acceptance probability |
|
|
273
|
+
| `spec_draft_p_split` | number | Split threshold |
|
|
274
|
+
| `spec_draft_n_gpu_layers` | int | Draft-model layers offloaded to GPU (`-1` auto, `-2` all) |
|
|
275
|
+
| `spec_draft_cache_type_k` | string | Draft-model K-cache dtype |
|
|
276
|
+
| `spec_draft_cache_type_v` | string | Draft-model V-cache dtype |
|
|
268
277
|
|
|
269
278
|
**Example**
|
|
270
279
|
|
|
@@ -281,6 +290,19 @@ n_ctx = 12800
|
|
|
281
290
|
download = true
|
|
282
291
|
```
|
|
283
292
|
|
|
293
|
+
A separate draft model can be configured inline:
|
|
294
|
+
|
|
295
|
+
```toml
|
|
296
|
+
[generators.model]
|
|
297
|
+
repo_id = "org/target-model-GGUF"
|
|
298
|
+
model_draft = { repo_id = "org/draft-model-GGUF", filename = "draft-q8_0.gguf" }
|
|
299
|
+
speculative = { type = "draft-mtp" }
|
|
300
|
+
spec_draft_n_max = 4
|
|
301
|
+
spec_draft_n_gpu_layers = -1
|
|
302
|
+
spec_draft_cache_type_k = "f16"
|
|
303
|
+
spec_draft_cache_type_v = "f16"
|
|
304
|
+
```
|
|
305
|
+
|
|
284
306
|
---
|
|
285
307
|
|
|
286
308
|
### `ggml-stt` (whisper.cpp via `@fugood/whisper.node`)
|
package/config/sample.toml
CHANGED
|
@@ -73,6 +73,15 @@ quantization = "mxfp4"
|
|
|
73
73
|
download = true
|
|
74
74
|
n_ctx = 12800 # Max: 131072
|
|
75
75
|
|
|
76
|
+
# Optional separate draft model for speculative decoding. Buttress includes the
|
|
77
|
+
# draft model in pre-downloads and memory planning.
|
|
78
|
+
# model_draft = { repo_id = "org/draft-model-GGUF", filename = "draft-q8_0.gguf" }
|
|
79
|
+
# speculative = { type = "draft-mtp" }
|
|
80
|
+
# spec_draft_n_max = 4
|
|
81
|
+
# spec_draft_n_gpu_layers = -1
|
|
82
|
+
# spec_draft_cache_type_k = "f16"
|
|
83
|
+
# spec_draft_cache_type_v = "f16"
|
|
84
|
+
|
|
76
85
|
[[generators]]
|
|
77
86
|
type = "ggml-llm"
|
|
78
87
|
[generators.backend]
|