@atomic-chat/core 0.5.1 → 0.13.2

package/src/settings/schema/atomic-prism.json

@@ -0,0 +1,433 @@
+[
+  {
+    "key": "version_backend",
+    "title": "Version & Backend",
+    "description": "Version and Backend for PrismML llama.cpp",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "none",
+      "options": [],
+      "recommended": ""
+    }
+  },
+  {
+    "key": "allow_candidate_builds",
+    "title": "Allow unverified PrismML builds",
+    "description": "Offer PrismML releases Atomic Chat has not verified on real hardware yet. Off by default: only verified builds are recommended and offered as updates.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "concurrent_mode",
+    "title": "Concurrent Mode",
+    "description": "Run the model as a multi-slot server: serves multiple chat requests in parallel on one loaded model. Overrides Parallel Sequences and Continuous Batching and exposes Prometheus /metrics.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "concurrent_slots",
+    "title": "Concurrent Slots",
+    "description": "Number of parallel slots when Concurrent Mode is enabled. Per-slot context = ctx_size / slots. Higher values serve more concurrent requests but give each less context.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 8,
+      "placeholder": "8",
+      "type": "number",
+      "textAlign": "right",
+      "min": 2,
+      "max": 16
+    }
+  },
+  {
+    "key": "expose_metrics",
+    "title": "Expose Prometheus /metrics",
+    "description": "Start llama-server with --metrics so the local API server can proxy /v1/metrics?model=<id>. Automatically enabled by Concurrent Mode.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "parallel",
+    "title": "Parallel Sequences",
+    "description": "Number of parallel sequences to decode (0 = llama.cpp default of 1). Set to 1 to also enable -kvu for optimized single-request serving. Ignored when Concurrent Mode is on.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1,
+      "placeholder": "1",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0
+    }
+  },
+  {
+    "key": "cont_batching",
+    "title": "Continuous Batching",
+    "description": "Enable continuous batching (a.k.a dynamic batching) for concurrent requests.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "llamacpp_env",
+    "title": "Environmental variables",
+    "description": "Environmental variables for llama.cpp(KEY=VALUE), separated by ';'",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Eg. GGML_VK_VISIBLE_DEVICES=0,1",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "reasoning_preserve",
+    "title": "Preserve Reasoning",
+    "description": "Preserve supported reasoning traces across conversation turns. Requires a compatible chat template and llama.cpp b9837 or newer.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "extra_args",
+    "title": "Additional llama-server Arguments",
+    "description": "Advanced: append arbitrary llama-server arguments. Separate arguments with spaces and quote values containing spaces. Arguments are passed directly without shell expansion.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "--reasoning-format deepseek --reasoning-budget 4096",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "timeout",
+    "title": "Timeout for llamacpp",
+    "description": "Llamacpp connection and load timeout in seconds. Model loading and chat streaming both use at least 1800s regardless of a lower value here.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "1800",
+      "placeholder": "e.g 1800",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "fit",
+    "title": "Fit context to device memory",
+    "description": "Let llama.cpp pick the largest context window that fits this device's memory (--fit). While on, the context-size slider is set by the engine; turn off to set it by hand. (env: LLAMA_ARG_FIT)",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": true
+    }
+  },
+  {
+    "key": "fit_target",
+    "title": "Fit Target per Device (MiB)",
+    "description": "Target margin per device for fit, comma-separated list of MiB values. Single value is broadcast across all devices. (env: LLAMA_ARG_FIT_TARGET)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "1024",
+      "placeholder": "1024 or MiB0,MiB1,...",
+      "type": "text",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "fit_ctx",
+    "title": "Fit Minimum Context Size",
+    "description": "Minimum context size (tokens) that can be set by --fit option. (env: LLAMA_ARG_FIT_CTX)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 4096,
+      "placeholder": "4096",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "threads",
+    "title": "Threads",
+    "description": "Number of threads to use during generation (-1 for logical cores).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "threads_batch",
+    "title": "Threads (Batch)",
+    "description": "Number of threads for batch and prompt processing (default: same as Threads).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1 (same as Threads)",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "ctx_shift",
+    "title": "Context Shift",
+    "description": "Allow model to cut text in the beginning to accommodate new text in its memory",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "n_predict",
+    "title": "Max Tokens to Predict",
+    "description": "Maximum number of tokens to generate (-1 = infinity).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": -1,
+      "placeholder": "-1",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "ubatch_size",
+    "title": "uBatch Size",
+    "description": "Physical maximum batch size for processing prompts.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 512,
+      "placeholder": "512",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "device",
+    "title": "Devices for Offload",
+    "description": "Comma-separated list of devices to use for offloading (e.g., 'CUDA0', 'CUDA0,CUDA1'). Leave empty to use default/CPU only.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "CUDA0",
+      "type": "text"
+    }
+  },
+  {
+    "key": "flash_attn",
+    "title": "Flash Attention",
+    "description": "Enable Flash Attention for optimized performance.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "auto",
+      "options": [
+        {
+          "value": "auto",
+          "name": "Auto"
+        },
+        {
+          "value": "on",
+          "name": "On"
+        },
+        {
+          "value": "off",
+          "name": "Off"
+        }
+      ]
+    }
+  },
+  {
+    "key": "no_mmap",
+    "title": "Disable mmap",
+    "description": "Do not memory-map model (slower load but may reduce pageouts if not using mlock).",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "mlock",
+    "title": "MLock",
+    "description": "Force system to keep model in RAM, preventing swapping/compression.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": false
+    }
+  },
+  {
+    "key": "cache_type_k",
+    "title": "KV Cache K Type",
+    "description": "KV cache data type for Keys (default: f16).",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "f16",
+      "options": [
+        {
+          "value": "f32",
+          "name": "f32"
+        },
+        {
+          "value": "f16",
+          "name": "f16"
+        },
+        {
+          "value": "bf16",
+          "name": "bf16"
+        },
+        {
+          "value": "q8_0",
+          "name": "q8_0"
+        },
+        {
+          "value": "q4_0",
+          "name": "q4_0"
+        },
+        {
+          "value": "q4_1",
+          "name": "q4_1"
+        },
+        {
+          "value": "iq4_nl",
+          "name": "iq4_nl"
+        },
+        {
+          "value": "q5_0",
+          "name": "q5_0"
+        },
+        {
+          "value": "q5_1",
+          "name": "q5_1"
+        }
+      ]
+    }
+  },
+  {
+    "key": "cache_type_v",
+    "title": "KV Cache V Type",
+    "description": "KV cache data type for Values (default: f16). Quantized values require Flash Attention to be enabled (Auto or On).",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "f16",
+      "options": [
+        {
+          "value": "f32",
+          "name": "f32"
+        },
+        {
+          "value": "f16",
+          "name": "f16"
+        },
+        {
+          "value": "bf16",
+          "name": "bf16"
+        },
+        {
+          "value": "q8_0",
+          "name": "q8_0"
+        },
+        {
+          "value": "q4_0",
+          "name": "q4_0"
+        },
+        {
+          "value": "q4_1",
+          "name": "q4_1"
+        },
+        {
+          "value": "iq4_nl",
+          "name": "iq4_nl"
+        },
+        {
+          "value": "q5_0",
+          "name": "q5_0"
+        },
+        {
+          "value": "q5_1",
+          "name": "q5_1"
+        }
+      ]
+    }
+  },
+  {
+    "key": "defrag_thold",
+    "title": "KV Cache Defragmentation Threshold",
+    "description": "Threshold for KV cache defragmentation (< 0 to disable).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0.1,
+      "placeholder": "0.1",
+      "type": "number",
+      "textAlign": "right",
+      "step": 0.01
+    }
+  },
+  {
+    "key": "rope_scaling",
+    "title": "RoPE Scaling Method",
+    "description": "RoPE frequency scaling method.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "none",
+      "options": [
+        {
+          "value": "none",
+          "name": "None"
+        },
+        {
+          "value": "linear",
+          "name": "Linear"
+        },
+        {
+          "value": "yarn",
+          "name": "YaRN"
+        }
+      ]
+    }
+  },
+  {
+    "key": "rope_scale",
+    "title": "RoPE Scale Factor",
+    "description": "RoPE context scaling factor.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1.0,
+      "placeholder": "1.0",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0,
+      "step": 0.01
+    }
+  },
+  {
+    "key": "rope_freq_base",
+    "title": "RoPE Frequency Base",
+    "description": "RoPE base frequency (0 = loaded from model).",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0,
+      "placeholder": "0 (model default)",
+      "type": "number",
+      "textAlign": "right"
+    }
+  },
+  {
+    "key": "rope_freq_scale",
+    "title": "RoPE Frequency Scale Factor",
+    "description": "RoPE frequency scaling factor.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1.0,
+      "placeholder": "1.0",
+      "type": "number",
+      "textAlign": "right",
+      "min": 0,
+      "step": 0.01
+    }
+  }
+]

package/src/settings/schema/tensorrt-llm.json

@@ -0,0 +1,253 @@
+[
+  {
+    "key": "gpu_id",
+    "title": "GPU",
+    "description": "UUID of the NVIDIA GPU models run on (GPU-… or MIG-…). Leave empty to use the card with the most free memory at load time; if the saved card is gone, the load uses that card and says so. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Card with the most free memory",
+      "type": "text"
+    }
+  },
+  {
+    "key": "context_length",
+    "title": "Context Length",
+    "description": "Maximum prompt plus reply length in tokens, fixed when the model starts. A longer request is refused with context_length_exceeded instead of restarting the model. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 8192,
+      "placeholder": "8192",
+      "type": "number",
+      "min": 512,
+      "max": 1048576
+    }
+  },
+  {
+    "key": "max_output_tokens",
+    "title": "Max Output Tokens",
+    "description": "Longest reply per request. Atomic Chat caps each request's max_tokens to this value on its way to the model; it is not an engine setting. Must be less than the context length. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 4096,
+      "placeholder": "4096",
+      "type": "number",
+      "min": 1,
+      "max": 1048576
+    }
+  },
+  {
+    "key": "kv_cache_free_gpu_memory_fraction",
+    "title": "KV Cache Memory Fraction",
+    "description": "Share of the GPU memory still free after the weights load that the engine reserves for its KV cache. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0.8,
+      "placeholder": "0.8",
+      "type": "number",
+      "min": 0.1,
+      "max": 0.95,
+      "step": 0.05
+    }
+  },
+  {
+    "key": "max_batch_size",
+    "title": "Parallel Requests",
+    "description": "How many requests the engine serves at once (chat, agent and API together); more wait in a queue. Hybrid models such as Qwen3.5 reserve memory for each one up front, so a higher value needs more GPU memory. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 1,
+      "placeholder": "1",
+      "type": "number",
+      "min": 1,
+      "max": 256
+    }
+  },
+  {
+    "key": "kv_cache_max_tokens",
+    "title": "KV Cache Token Limit",
+    "description": "Upper bound of the KV cache in tokens. 0 sizes it for every parallel request at the full context length. The engine uses the smaller of this and the KV cache memory fraction; hybrid models such as Qwen3.5 need it to start. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0,
+      "placeholder": "0",
+      "type": "number",
+      "min": 0,
+      "max": 16777216
+    }
+  },
+  {
+    "key": "cuda_graphs",
+    "title": "CUDA Graphs",
+    "description": "Speeds up token generation, but takes extra GPU memory (about 2 GB for a small model) and a few seconds at load. Auto turns them on only on cards with 12 GB of memory or more, so a model still fits on a small card. Applies from the next model load.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "auto",
+      "options": [
+        {
+          "value": "auto",
+          "name": "Auto"
+        },
+        {
+          "value": "on",
+          "name": "On"
+        },
+        {
+          "value": "off",
+          "name": "Off"
+        }
+      ]
+    }
+  },
+  {
+    "key": "kv_cache_dtype",
+    "title": "KV Cache Precision",
+    "description": "FP8 halves the KV cache memory, so a longer context fits, at a small quality cost on very long contexts. Needs an Ada (RTX 40) or newer card; on older cards it is ignored and the model's own precision is used. Applies from the next model load.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "auto",
+      "options": [
+        {
+          "value": "auto",
+          "name": "Model precision"
+        },
+        {
+          "value": "fp8",
+          "name": "FP8"
+        }
+      ]
+    }
+  },
+  {
+    "key": "load_timeout_seconds",
+    "title": "Load Timeout (seconds)",
+    "description": "How long a load may take before it fails. 0 estimates it from the model's size. Applies from the next model load.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": 0,
+      "placeholder": "0",
+      "type": "number",
+      "min": 0,
+      "max": 3600
+    }
+  },
+  {
+    "key": "enable_prefix_caching",
+    "title": "Prefix Caching",
+    "description": "Reuses the KV cache of a prompt start the engine has already seen (the system prompt, earlier turns of a chat), so a follow-up reads only what is new. Turn it off only to rule it out when chasing a problem. Applies from the next model load.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": true
+    }
+  },
+  {
+    "key": "overlap_scheduler",
+    "title": "Overlap Scheduler",
+    "description": "Plans the engine's next step while the card runs the current one, which speeds up generation. Turn it off only if a model misbehaves with it. Applies from the next model load.",
+    "controllerType": "checkbox",
+    "controllerProps": {
+      "value": true
+    }
+  },
+  {
+    "key": "capacity_scheduler_policy",
+    "title": "Scheduler Policy",
+    "description": "How the engine admits parallel requests. Guaranteed no evict starts a request only when the KV cache can hold it to the end; max utilization starts more at once and pauses some when the cache runs out. Matters only with more than one parallel request. Applies from the next model load.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "guaranteed_no_evict",
+      "options": [
+        {
+          "value": "guaranteed_no_evict",
+          "name": "Guaranteed no evict"
+        },
+        {
+          "value": "max_utilization",
+          "name": "Max utilization"
+        }
+      ]
+    }
+  },
+  {
+    "key": "dtype",
+    "title": "Weight Precision",
+    "description": "Precision of the model's unquantized weights and activations. Auto takes the model's own. A quantized model keeps its quantized weights either way. Applies from the next model load.",
+    "controllerType": "dropdown",
+    "controllerProps": {
+      "value": "auto",
+      "options": [
+        {
+          "value": "auto",
+          "name": "Model precision"
+        },
+        {
+          "value": "float16",
+          "name": "float16"
+        },
+        {
+          "value": "bfloat16",
+          "name": "bfloat16"
+        },
+        {
+          "value": "float32",
+          "name": "float32"
+        }
+      ]
+    }
+  },
+  {
+    "key": "default_temperature",
+    "title": "Default Temperature",
+    "description": "Temperature for a request that does not set one, from 0 to 2. Empty leaves it to the engine. A request's own value always wins. Applies from the next model load or reload.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Model default",
+      "type": "text"
+    }
+  },
+  {
+    "key": "default_top_p",
+    "title": "Default Top P",
+    "description": "Top P for a request that does not set one, above 0 and up to 1. Empty leaves it to the engine. A request's own value always wins. Applies from the next model load or reload.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Model default",
+      "type": "text"
+    }
+  },
+  {
+    "key": "default_top_k",
+    "title": "Default Top K",
+    "description": "Top K for a request that does not set one; -1 or 0 turns it off. Empty leaves it to the engine. A request's own value always wins. Applies from the next model load or reload.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Model default",
+      "type": "text"
+    }
+  },
+  {
+    "key": "default_min_p",
+    "title": "Default Min P",
+    "description": "Min P for a request that does not set one, from 0 to 1. Empty leaves it to the engine. A request's own value always wins. Applies from the next model load or reload.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Model default",
+      "type": "text"
+    }
+  },
+  {
+    "key": "default_repetition_penalty",
+    "title": "Default Repetition Penalty",
+    "description": "Repetition penalty for a request that does not set one, above 0 (1 is none). Empty leaves it to the engine. A request's own value always wins. Applies from the next model load or reload.",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Model default",
+      "type": "text"
+    }
+  }
+]