@atomic-chat/core 0.5.1

package/src/settings/schema/llamacpp-upstream.json

@@ -0,0 +1,489 @@
+[
+  {
+    "key": "version_backend",
+    "title": "Version & Backend",
+    "description": "Version and Backend for llama.cpp",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "none",
+      "options": [],
+      "recommended": ""
+    }
+  },
+  {
+    "key": "mtp",
+    "title": "Enable MTP (Multi-Token Prediction)",
+    "description": "Speed up generation with a compatible MTP head. Atomic Chat uses a built-in or matching downloaded head when available. Requires an MTP-capable llama.cpp backend and is mutually exclusive with DFlash.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "dflash",
+    "title": "Enable DFlash (multi-layer draft speculative decoding)",
+    "description": "Speed up generation with a compatible DFlash draft model. Atomic Chat offers a matching draft download when available. Requires a DFlash-capable llama.cpp backend and is mutually exclusive with MTP. [Browse Atomic Chat DFlash models](https://huggingface.co/collections/AtomicChat/dflash-speculative-drafts).",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "dflash_block_size",
+    "title": "DFlash Block Size",
+    "description": "Draft block size for DFlash. The number of speculative tokens per step (--spec-draft-n-max) is derived as block_size - 1. Default 16.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 16,
+      "placeholder": "16",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "concurrent_mode",
+    "title": "Concurrent Mode",
+    "description": "Run the model as a multi-slot server: serves multiple chat requests in parallel on one loaded model. Overrides Parallel Sequences and Continuous Batching and exposes Prometheus /metrics.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "concurrent_slots",
+    "title": "Concurrent Slots",
+    "description": "Number of parallel slots when Concurrent Mode is enabled. Per-slot context = ctx_size / slots. Higher values serve more concurrent requests but give each less context.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 8,
+      "placeholder": "8",
+      "type": "number",
+      "textAlign": "right",
+      "min": 2,
+      "max": 16
+    }
+  },
+  {
+    "key": "expose_metrics",
+    "title": "Expose Prometheus /metrics",
+    "description": "Start llama-server with --metrics so the local API server can proxy /v1/metrics?model=<id>. Automatically enabled by Concurrent Mode.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "parallel",
+    "title": "Parallel Sequences",
+    "description": "Number of parallel sequences to decode (0 = llama.cpp default of 1). Set to 1 to also enable -kvu for optimized single-request serving. Ignored when Concurrent Mode is on.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1,
+      "placeholder": "1",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0
+    }
+  },
+  {
+    "key": "cont_batching",
+    "title": "Continuous Batching",
+    "description": "Enable continuous batching (a.k.a dynamic batching) for concurrent requests.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "llamacpp_env",
+    "title": "Environmental variables",
+    "description": "Environmental variables for llama.cpp(KEY=VALUE), separated by ';'",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Eg. GGML_VK_VISIBLE_DEVICES=0,1",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "reasoning_preserve",
+    "title": "Preserve Reasoning",
+    "description": "Preserve supported reasoning traces across conversation turns. Requires a compatible chat template and llama.cpp b9837 or newer.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "extra_args",
+    "title": "Additional llama-server Arguments",
+    "description": "Advanced: append arbitrary llama-server arguments. Separate arguments with spaces and quote values containing spaces. Arguments are passed directly without shell expansion.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "--reasoning-format deepseek --reasoning-budget 4096",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "timeout",
+    "title": "Timeout for llamacpp",
+    "description": "Llamacpp connection and load timeout in seconds. Model loading and chat streaming both use at least 1800s regardless of a lower value here.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "1800",
+      "placeholder": "e.g 1800",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "fit",
+    "title": "Fit context to device memory",
+    "description": "Let llama.cpp pick the largest context window that fits this device's memory (--fit). While on, the context-size slider is set by the engine; turn off to set it by hand. (env: LLAMA_ARG_FIT)",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": true
+    }
+  },
+  {
+    "key": "fit_target",
+    "title": "Fit Target per Device (MiB)",
+    "description": "Target margin per device for fit, comma-separated list of MiB values. Single value is broadcast across all devices. (env: LLAMA_ARG_FIT_TARGET)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "1024",
+      "placeholder": "1024 or MiB0,MiB1,...",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "fit_ctx",
+    "title": "Fit Minimum Context Size",
+    "description": "Minimum context size (tokens) that can be set by --fit option. (env: LLAMA_ARG_FIT_CTX)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 4096,
+      "placeholder": "4096",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "threads",
+    "title": "Threads",
+    "description": "Number of threads to use during generation (-1 for logical cores).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "threads_batch",
+    "title": "Threads (Batch)",
+    "description": "Number of threads for batch and prompt processing (default: same as Threads).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1 (same as Threads)",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "ctx_shift",
+    "title": "Context Shift",
+    "description": "Allow model to cut text in the beginning to accommodate new text in its memory",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "n_predict",
+    "title": "Max Tokens to Predict",
+    "description": "Maximum number of tokens to generate (-1 = infinity).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "ubatch_size",
+    "title": "uBatch Size",
+    "description": "Physical maximum batch size for processing prompts.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 512,
+      "placeholder": "512",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "device",
+    "title": "Devices for Offload",
+    "description": "Comma-separated list of devices to use for offloading (e.g., 'CUDA0', 'CUDA0,CUDA1'). Leave empty to use default/CPU only.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "CUDA0",
+      "type": "text"
+    }
+  },
+  {
+    "key": "split_mode",
+    "title": "GPU Split Mode",
+    "description": "How to split the model across multiple GPUs.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "layer",
+      "options": [
+        {
+          "value": "none",
+          "name": "None"
+        },
+        {
+          "value": "layer",
+          "name": "Layer"
+        },
+        {
+          "value": "row",
+          "name": "Row"
+        }
+      ]
+    }
+  },
+  {
+    "key": "main_gpu",
+    "title": "Main GPU Index",
+    "description": "The GPU to use for the model (split-mode=none) or intermediate results (split-mode=row).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0,
+      "placeholder": "0",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "flash_attn",
+    "title": "Flash Attention",
+    "description": "Enable Flash Attention for optimized performance.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "auto",
+      "options": [
+        {
+          "value": "auto",
+          "name": "Auto"
+        },
+        {
+          "value": "on",
+          "name": "On"
+        },
+        {
+          "value": "off",
+          "name": "Off"
+        }
+      ]
+    }
+  },
+  {
+    "key": "no_mmap",
+    "title": "Disable mmap",
+    "description": "Do not memory-map model (slower load but may reduce pageouts if not using mlock).",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "mlock",
+    "title": "MLock",
+    "description": "Force system to keep model in RAM, preventing swapping/compression.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "cache_type_k",
+    "title": "KV Cache K Type",
+    "description": "KV cache data type for Keys (default: f16).",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "f16",
+      "options": [
+        {
+          "value": "f32",
+          "name": "f32"
+        },
+        {
+          "value": "f16",
+          "name": "f16"
+        },
+        {
+          "value": "bf16",
+          "name": "bf16"
+        },
+        {
+          "value": "q8_0",
+          "name": "q8_0"
+        },
+        {
+          "value": "q4_0",
+          "name": "q4_0"
+        },
+        {
+          "value": "q4_1",
+          "name": "q4_1"
+        },
+        {
+          "value": "iq4_nl",
+          "name": "iq4_nl"
+        },
+        {
+          "value": "q5_0",
+          "name": "q5_0"
+        },
+        {
+          "value": "q5_1",
+          "name": "q5_1"
+        }
+      ]
+    }
+  },
+  {
+    "key": "cache_type_v",
+    "title": "KV Cache V Type",
+    "description": "KV cache data type for Values (default: f16). Quantized values require Flash Attention to be enabled (Auto or On).",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "f16",
+      "options": [
+        {
+          "value": "f32",
+          "name": "f32"
+        },
+        {
+          "value": "f16",
+          "name": "f16"
+        },
+        {
+          "value": "bf16",
+          "name": "bf16"
+        },
+        {
+          "value": "q8_0",
+          "name": "q8_0"
+        },
+        {
+          "value": "q4_0",
+          "name": "q4_0"
+        },
+        {
+          "value": "q4_1",
+          "name": "q4_1"
+        },
+        {
+          "value": "iq4_nl",
+          "name": "iq4_nl"
+        },
+        {
+          "value": "q5_0",
+          "name": "q5_0"
+        },
+        {
+          "value": "q5_1",
+          "name": "q5_1"
+        }
+      ]
+    }
+  },
+  {
+    "key": "defrag_thold",
+    "title": "KV Cache Defragmentation Threshold",
+    "description": "Threshold for KV cache defragmentation (< 0 to disable).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0.1,
+      "placeholder": "0.1",
+      "type": "number",
+      "textAlign": "right",
+      "step": 0.01
+    }
+  },
+  {
+    "key": "rope_scaling",
+    "title": "RoPE Scaling Method",
+    "description": "RoPE frequency scaling method.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "none",
+      "options": [
+        {
+          "value": "none",
+          "name": "None"
+        },
+        {
+          "value": "linear",
+          "name": "Linear"
+        },
+        {
+          "value": "yarn",
+          "name": "YaRN"
+        }
+      ]
+    }
+  },
+  {
+    "key": "rope_scale",
+    "title": "RoPE Scale Factor",
+    "description": "RoPE context scaling factor.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1.0,
+      "placeholder": "1.0",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0,
+      "step": 0.01
+    }
+  },
+  {
+    "key": "rope_freq_base",
+    "title": "RoPE Frequency Base",
+    "description": "RoPE base frequency (0 = loaded from model).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0,
+      "placeholder": "0 (model default)",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "rope_freq_scale",
+    "title": "RoPE Frequency Scale Factor",
+    "description": "RoPE frequency scaling factor.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1.0,
+      "placeholder": "1.0",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0,
+      "step": 0.01
+    }
+  }
+]