@fugood/buttress-server-poc 2.23.0-beta.3 → 2.23.0-beta.30

package/README.md

@@ -40,6 +40,13 @@ log_level = "info"
 [runtime]
 cache_dir = "~/.buttress/models"
 
+# Session state cache for ggml-llm (saves KV cache to disk for prompt reuse)
+[runtime.session_cache]
+enabled = true
+max_size_bytes = "10GB"  # Supports string (e.g., "10GB", "500MB") or number
+max_entries = 1000
+
+# GGML LLM generator
 [[generators]]
 type = "ggml-llm"
 [generators.backend]
@@ -48,6 +55,15 @@ variant_preference = ["cuda", "vulkan", "default"]
 repo_id = "ggml-org/gpt-oss-20b-GGUF"
 quantization = "mxfp4"
 n_ctx = 12800
+
+# GGML STT (Speech-to-Text) generator
+[[generators]]
+type = "ggml-stt"
+[generators.backend]
+variant_preference = ["coreml", "default"]
+[generators.model]
+repo_id = "BricksDisplay/whisper-ggml"
+filename = "ggml-small.bin"
 ```
 
 ### Programmatic Usage
@@ -103,4 +119,75 @@ Port can be configured via multiple sources (highest priority first):
 
 1. **Command-line flag**: `--port 3000`
 2. **Config file**: `[server] port = 2080`
-4. **Default**: `2080`
+3. **Default**: `2080`
+
+## CLI Reference
+
+```
+bricks-buttress-poc v2.23.0-beta.22
+
+Buttress server for remote inference with GGML backends.
+
+Usage:
+  bricks-buttress-poc [options]
+
+Options:
+  -h, --help                    Show this help message
+  -v, --version                 Show version number
+  -p, --port <port>             Port to listen on (default: 2080)
+  -c, --config <path|toml>      Path to TOML config file or inline TOML string
+
+Testing Options:
+  --test-caps <backend>         Test model capabilities (ggml-llm or ggml-stt)
+  --test-caps-model-id <id>     Model ID to test (used with --test-caps)
+  --test-models <ids>           Comma-separated list of model IDs to test
+  --test-models-default         Test default set of models
+
+  Note: --test-models and --test-models-default output a markdown report
+        file (e.g., ggml-llm-model-capabilities-YYYY-MM-DD.md)
+
+Environment Variables:
+  BUTTRESS_PORT                 Port to listen on (overridden by --port)
+  NODE_ENV                      Set to 'development' for dev mode
+
+Examples:
+  bricks-buttress-poc
+  bricks-buttress-poc --port 3000
+  bricks-buttress-poc --config ./config.toml
+  bricks-buttress-poc --test-caps ggml-llm --test-models-default
+  bricks-buttress-poc --test-caps ggml-stt --test-caps-model-id BricksDisplay/whisper-ggml:ggml-small.bin
+```
+
+## Session State Cache
+
+The server supports session state caching for ggml-llm generators, which saves KV cache state to disk after completions. This enables:
+
+- **Prompt reuse**: Same or similar prompts can reuse cached state, skipping prompt processing
+- **Multi-turn conversations**: Conversation history state is preserved across requests
+
+### Configuration
+
+```toml
+[runtime.session_cache]
+enabled = true                  # Enable/disable session caching (default: true)
+max_size_bytes = "10GB"         # Supports string (e.g., "10GB", "500MB") or number (default: 10GB)
+max_entries = 1000              # Max number of cached entries (default: 1000)
+```
+
+### How it works
+
+1. After a successful completion, the KV cache state is saved to disk
+2. On new completions, the server checks if any cached state matches the prompt prefix
+3. If a match is found, the cached state is loaded, skipping redundant prompt processing
+4. LRU eviction removes oldest entries when limits are exceeded
+
+### Cache location
+
+Cache files are stored in `{cache_dir}/.session-state-cache/`:
+- `cache-map.json` - Index of cached entries
+- `states/` - Binary state files
+- `temp/` - Temporary files (auto-cleaned after 1 hour)
+
+## Tips
+
+- macOS: Use `sudo sysctl iogpu.wired_limit_mb=<number>` to increase GPU memory allocation. The default available memory of GPU is about ~70%. For example, if the hardware have 128GB memory, you can use `sudo sysctl iogpu.wired_limit_mb=137438` to increase to 128GB. Run `sudo sysctl iogpu.wired_limit_mb=0` if you want to back to default.

package/bin/start.mjs

@@ -3,12 +3,61 @@ import fs from 'node:fs'
 import path from 'node:path'
 import os from 'node:os'
 import TOML from '@iarna/toml'
+import bytes from 'bytes'
+import { createRequire } from 'node:module'
 
 const serverModule =
   process.env.NODE_ENV === 'development'
     ? await import('../src/index.js')
     : await import('../lib/index.js')
-const { startServer, checkAndNotifyUpdates } = serverModule
+
+const require = createRequire(import.meta.url)
+const pkg = require('../package.json')
+
+// Handle --version/-v flag
+if (process.argv.includes('--version') || process.argv.includes('-v')) {
+  console.log(pkg.version)
+  process.exit(0)
+}
+
+// Handle --help/-h flag
+if (process.argv.includes('--help') || process.argv.includes('-h')) {
+  console.log(`
+bricks-buttress-poc v${pkg.version}
+
+Buttress server for remote inference with GGML backends.
+
+Usage:
+  bricks-buttress-poc [options]
+
+Options:
+  -h, --help                    Show this help message
+  -v, --version                 Show version number
+  -p, --port <port>             Port to listen on (default: 2080)
+  -c, --config <path|toml>      Path to TOML config file or inline TOML string
+
+Testing Options:
+  --test-caps <backend>         Test model capabilities (ggml-llm or ggml-stt)
+  --test-caps-model-id <id>     Model ID to test (used with --test-caps)
+  --test-models <ids>           Comma-separated list of model IDs to test
+  --test-models-default         Test default set of models
+
+  Note: --test-models and --test-models-default output a markdown report
+        file (e.g., ggml-llm-model-capabilities-YYYY-MM-DD.md)
+
+Environment Variables:
+  BUTTRESS_PORT                 Port to listen on (overridden by --port)
+  NODE_ENV                      Set to 'development' for dev mode
+
+Examples:
+  bricks-buttress-poc
+  bricks-buttress-poc --port 3000
+  bricks-buttress-poc --config ./config.toml
+  bricks-buttress-poc --test-caps ggml-llm --test-models-default
+  bricks-buttress-poc --test-caps ggml-stt --test-caps-model-id BricksDisplay/whisper-ggml:ggml-small.bin
+`)
+  process.exit(0)
+}
 
 const portArgIndex = process.argv.findIndex((arg) => arg === '--port' || arg === '-p')
 const portValue = portArgIndex >= 0 ? Number(process.argv[portArgIndex + 1]) : undefined
@@ -53,6 +102,117 @@ if (configInput) {
   }
 }
 
+const {
+  testGgmlLlmCapabilities,
+  testGgmlSttCapabilities,
+  showModelsTable,
+  showSttModelsTable,
+  startServer,
+  checkAndNotifyUpdates,
+} = serverModule
+
+// Default models for --test-models-default
+const DEFAULT_TEST_GGML_LLM_MODELS = [
+  'ggml-org/gpt-oss-20b-GGUF',
+  'ggml-org/gpt-oss-120b-GGUF',
+  'unsloth/Nemotron-3-Nano-30B-A3B-GGUF',
+  'unsloth/Qwen3-VL-30B-A3B-Instruct-GGUF',
+  'bartowski/Mistral-Nemo-Instruct-2407-GGUF',
+  'mistralai/Magistral-Small-2509-GGUF',
+  'mistralai/Ministral-3-14B-Reasoning-2512-GGUF',
+  'bartowski/mistralai_Devstral-Small-2-24B-Instruct-2512-GGUF',
+  'bartowski/mistralai_Devstral-2-123B-Instruct-2512-GGUF',
+  'ggml-org/gemma-3-12b-it-qat-GGUF',
+  'ggml-org/gemma-3-27b-it-qat-GGUF',
+  'unsloth/phi-4-GGUF',
+]
+
+const DEFAULT_TEST_GGML_STT_MODELS = [
+  'BricksDisplay/whisper-ggml:ggml-small.bin',
+  'BricksDisplay/whisper-ggml:ggml-small-q8_0.bin',
+  'BricksDisplay/whisper-ggml:ggml-medium.bin',
+  'BricksDisplay/whisper-ggml:ggml-medium-q8_0.bin',
+  'BricksDisplay/whisper-ggml:ggml-large-v3-turbo.bin',
+  'BricksDisplay/whisper-ggml:ggml-large-v3-turbo-q8_0.bin',
+  'BricksDisplay/whisper-ggml:ggml-large-v3.bin',
+]
+
+// Handle --test-caps flag
+const testCapsArgIndex = process.argv.findIndex((arg) => arg === '--test-caps')
+if (testCapsArgIndex >= 0) {
+  const backendType = process.argv[testCapsArgIndex + 1] || 'ggml-llm'
+  if (backendType !== 'ggml-llm' && backendType !== 'ggml-stt') {
+    console.error('Only ggml-llm and ggml-stt backends are supported for testing capabilities')
+    process.exit(1)
+  }
+
+  // Check for --test-models or --test-models-default
+  const testModelsArgIndex = process.argv.findIndex((arg) => arg === '--test-models')
+  const hasTestModelsDefault = process.argv.includes('--test-models-default')
+
+  if (backendType === 'ggml-stt') {
+    // STT backend
+    if (testModelsArgIndex >= 0) {
+      const modelIdsInput = process.argv[testModelsArgIndex + 1]
+      if (!modelIdsInput) {
+        console.error('Error: --test-models requires a comma-separated list of model IDs')
+        process.exit(1)
+      }
+      const modelIds = modelIdsInput.split(',').map((id) => id.trim())
+
+      await showSttModelsTable({
+        modelIds,
+        defaultConfig,
+      })
+    } else if (hasTestModelsDefault) {
+      await showSttModelsTable({
+        modelIds: DEFAULT_TEST_GGML_STT_MODELS,
+        defaultConfig,
+      })
+    } else {
+      // Single model test
+      const testCapsModelIdArgIndex = process.argv.findIndex(
+        (arg) => arg === '--test-caps-model-id',
+      )
+      const modelId =
+        testCapsModelIdArgIndex >= 0 ? process.argv[testCapsModelIdArgIndex + 1] : null
+
+      await testGgmlSttCapabilities({
+        modelId,
+        defaultConfig,
+      })
+    }
+  } else if (testModelsArgIndex >= 0) {
+    // LLM backend with custom models
+    const modelIdsInput = process.argv[testModelsArgIndex + 1]
+    if (!modelIdsInput) {
+      console.error('Error: --test-models requires a comma-separated list of model IDs')
+      process.exit(1)
+    }
+    const modelIds = modelIdsInput.split(',').map((id) => id.trim())
+
+    await showModelsTable({
+      modelIds,
+      defaultConfig,
+    })
+  } else if (hasTestModelsDefault) {
+    // LLM backend with default models
+    await showModelsTable({
+      modelIds: DEFAULT_TEST_GGML_LLM_MODELS,
+      defaultConfig,
+    })
+  } else {
+    // LLM backend single model test
+    const testCapsModelIdArgIndex = process.argv.findIndex((arg) => arg === '--test-caps-model-id')
+    const modelId = testCapsModelIdArgIndex >= 0 ? process.argv[testCapsModelIdArgIndex + 1] : null
+
+    await testGgmlLlmCapabilities({
+      modelId,
+      defaultConfig,
+    })
+  }
+}
+
 const getLocalIpAddress = () => {
   const networkInterfaces = os.networkInterfaces()
   const localIp = Object.values(networkInterfaces)
@@ -62,11 +222,17 @@ const getLocalIpAddress = () => {
 }
 
 const configPort = defaultConfig?.server?.port
+const configMaxBodySizeRaw = defaultConfig?.server?.max_body_size
+const configMaxBodySize =
+  typeof configMaxBodySizeRaw === 'string'
+    ? bytes.parse(configMaxBodySizeRaw)
+    : configMaxBodySizeRaw
 const finalPort = portValue || configPort || Number(process.env.BUTTRESS_PORT) || 2080
 
 startServer({
   port: finalPort,
   defaultConfig,
+  maxBodySize: configMaxBodySize,
 })
   .then(async ({ port }) => {
     const ip = getLocalIpAddress()
@@ -74,10 +240,18 @@ startServer({
     console.log('--------------------------------')
     await checkAndNotifyUpdates()
 
+    console.log()
+    console.log('Current supported Generators:')
+    console.log('- LLM (GGML)')
+    console.log('- STT (GGML)')
+    console.log()
     console.log(
-      'Please configure `Buttress (Remote Inference)` in the Generator like LLM (GGML) to connect to this server.',
+      'Please configure `Buttress (Remote Inference)` in the Generator to connect to this server.',
     )
-    console.log(`Use http://${ip}:${port} (tRPC path: /trpc) to connect to this server via LAN.`)
+    console.log()
+    console.log(`- Use http://${ip}:${port}/trpc to connect to this server via LAN.`)
+    console.log(`- Visit http://${ip}:${port}/status to see status via LAN.`)
+    console.log()
   })
   .catch((error) => {
     console.error('Failed to start Buttress server POC:', error)

package/config/sample.toml

@@ -8,18 +8,52 @@
 [server]
 port = 2080
 log_level = "info"
+# max_body_size = "100MB"  # Supports string (e.g., "100MB", "1GB") or number in bytes
 
 [runtime]
 cache_dir = "./.buttress-cache"
+# huggingface_token = "hf_xx"
+
+# Global model params
+flash_attn_type = "on"
+cache_type_k = "q8_0"
+cache_type_v = "q8_0"
+
+# Session state cache for ggml-llm (saves KV cache to disk for prompt reuse)
+[runtime.session_cache]
+enabled = true
+max_size_bytes = "10GB"  # Supports string (e.g., "10GB", "500MB") or number
+max_entries = 1000
 
 [[generators]]
 type = "ggml-llm"
 [generators.backend]
-variant_preference = ["cuda", "vulkan", "default"]
+variant_preference = ["cuda", "vulkan", "snapdragon", "default"]
+gpu_memory_fraction = 0.95
+cpu_memory_fraction = 0.95
 [generators.model]
 repo_id = "ggml-org/gpt-oss-20b-GGUF"
 quantization = "mxfp4"
-n_ctx = 12800
+# n_ctx = 12800 # Max: 131072
+
+[[generators]]
+type = "ggml-llm"
+[generators.backend]
+variant_preference = ["cuda", "vulkan", "snapdragon", "default"]
+gpu_memory_fraction = 0.95
+[generators.model]
+repo_id = "ggml-org/gpt-oss-120b-GGUF"
+quantization = "mxfp4"
+
+[[generators]]
+type = "ggml-llm"
+[generators.backend]
+variant_preference = ["default"]
+gpu_memory_fraction = 0.95
+[generators.model]
+repo_id = "bartowski/mistralai_Devstral-2-123B-Instruct-2512-GGUF"
+quantization = "q4_0"
+# n_ctx = 128000 # Max: 262144
 
 [[generators]]
 type = "ggml-llm"
@@ -28,3 +62,30 @@ variant_preference = ["default"]
 [generators.model]
 repo_id = "ggml-org/gemma-3-270m-qat-GGUF"
 quantization = "q4_0"
+
+[[generators]]
+type = "ggml-llm"
+[generators.backend]
+variant_preference = ["default"]
+[generators.model]
+repo_id = "ggml-org/gemma-3-12b-it-qat-GGUF"
+
+# Speech-to-Text (STT) generators
+[[generators]]
+type = "ggml-stt"
+[generators.backend]
+variant_preference = ["cuda", "vulkan", "default"]
+[generators.model]
+repo_id = "BricksDisplay/whisper-ggml"
+filename = "ggml-small-q8_0.bin"
+use_gpu = true
+
+[[generators]]
+type = "ggml-stt"
+[generators.backend]
+variant_preference = ["cuda", "vulkan", "default"]
+[generators.model]
+repo_id = "BricksDisplay/whisper-ggml"
+filename = "ggml-large-v3-turbo-q8_0.bin"
+use_gpu = true
+use_flash_attn = true