@fugood/buttress-server 2.26.0-beta.13 → 2.26.0-beta.15

package/README.md

@@ -216,6 +216,8 @@ Most ggml-llm `[generators.model]` keys can also live in `[runtime]` as defaults
 | `no_extra_bufts`             | boolean                       | Disable extra compute buffer types                                     |
 | `cpu_mask`, `cpu_strict`     | string / boolean              | CPU affinity (advanced)                                                |
 | `devices`                    | string[]                      | Restrict to specific GGML devices                                      |
+| `split_mode`                 | `"none"` / `"layer"` / `"row"` / `"tensor"` | ggml-llm GPU splitting: single GPU, layers + KV (default), weight rows, or weights + KV (experimental tensor parallelism) |
+| `tensor_split`               | number[]\|string              | ggml-llm per-GPU proportions, e.g. `[3, 1]` or `"3,1"`; unset uses the native automatic split |
 | Speculative keys             | various                       | `speculative`, `spec_type`, `spec_draft_n_max/n_min/p_min/p_split`, plus draft-model GPU/cache settings |
 
 ### `[autodiscover]`
@@ -277,6 +279,10 @@ Additional keys honored by **ggml-llm**, **ggml-stt** and **ggml-tts** (mlx-llm
 
 Loads a GGUF LLM. Runtime keys above can be overridden per-generator under `[generators.model]`; `[generators.backend]` only controls backend selection and resource planning.
 
+GPU splitting (`split_mode`, `tensor_split`) is currently configurable only in Buttress.
+TODO: Expose these options in the LLM (GGML) generator. The experimental `tensor` mode
+requires a compatible GPU backend and model architecture.
+
 **`[generators.backend]`**
 
 | Key                   | Type     | Default                                       | Notes                                                            |
@@ -292,6 +298,7 @@ Loads a GGUF LLM. Runtime keys above can be overridden per-generator under `[gen
 | ----------------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------- |
 | `n_ctx`                                                                       | number           | Context window. Auto-capped at the model's training context.         |
 | `n_gpu_layers`                                                                | number\|`"auto"` | Layers offloaded to GPU (default `"auto"`)                           |
+| `split_mode`, `tensor_split`                                                   | string / number[]\|string | Per-model overrides for the GPU split defaults in `[runtime]`       |
 | `n_batch`                                                                     | number           | Prompt batch size (default `512`)                                    |
 | `embedding`                                                                   | boolean          | Enable llama.cpp embedding mode for a dedicated embedding model (default `false`) |
 | `pooling_type`                                                                | string           | Optional embedding pooling override such as `mean`                   |
@@ -364,12 +371,21 @@ spec_draft_cache_type_v = "f16"
 
 Loads a Whisper GGML model for speech-to-text.
 
+With `@fugood/whisper.node` 1.2.0-rc.0, `snapdragon` enables experimental
+Whisper Hexagon NPU acceleration on Linux arm64 with a working Qualcomm
+HTP/FastRPC runtime. This variant does not include OpenCL. Install it with
+`--ggml-variant=snapdragon` (the default CPU package remains available for fallback).
+The platform package includes the HTP libraries; the loader defaults
+`ADSP_LIBRARY_PATH` to that package directory and `GGML_HEXAGON_NDEV` to `16`,
+without overriding explicit environment values. Hexagon always enables flash
+attention, even when `use_flash_attn` is off.
+
 **`[generators.backend]`**
 
 | Key                   | Type     | Default                       | Notes                              |
 | --------------------- | -------- | ----------------------------- | ---------------------------------- |
-| `variant`             | string   | auto                          | `cuda` / `vulkan` / `default`      |
-| `variant_preference`  | string[] | `["cuda","vulkan","default"]` | Probe order                        |
+| `variant`             | string   | auto                          | `cuda` / `vulkan` / `snapdragon` / `default` |
+| `variant_preference`  | string[] | `["cuda","vulkan","snapdragon","default"]` | Probe order                  |
 | `gpu_memory_fraction` | number   | `0.85`                        |                                    |
 | `cpu_memory_fraction` | number   | `0.5`                         |                                    |
 
@@ -394,7 +410,7 @@ Loads a Whisper GGML model for speech-to-text.
 [[generators]]
 type = "ggml-stt"
 [generators.backend]
-variant_preference = ["cuda", "vulkan", "default"]
+variant_preference = ["cuda", "vulkan", "snapdragon", "default"]
 [generators.model]
 repo_id = "BricksDisplay/whisper-ggml"
 filename = "ggml-large-v3-turbo-q8_0.bin"

package/config/sample.toml

@@ -78,6 +78,12 @@ cache_type_k = "q8_0"
 cache_type_v = "q8_0"
 n_threads = 6
 
+# Optional ggml-llm multi-GPU defaults (also supported under [generators.model]).
+# split_mode = "layer"   # none / layer (default) / row / tensor (experimental)
+# tensor_split = [3, 1]  # per-GPU proportions; also accepts "3,1"
+# Tensor mode requires a compatible GPU backend and model architecture.
+# TODO: Expose these Buttress-only options in the LLM (GGML) generator.
+
 # Delay before releasing context when refCount reaches 0 (default: 10000ms)
 # Set to 0 for immediate release
 context_release_delay_ms = 30000