@miller-tech/uap 1.9.1 → 1.10.0

package/config/model-profiles/gpt-4o.json

@@ -0,0 +1,64 @@
+{
+  "_profile": "gpt-4o",
+  "_description": "OpenAI GPT-4o (gpt-4o-2025-03-27). Multimodal flagship with native tool calling, JSON mode, and 128K context. Fast, capable, and cost-effective for most tasks. Supports parallel function calling and structured outputs.",
+
+  "model": "gpt-4o-2025-03-27",
+  "provider": "openai",
+  "api_base_url": "https://api.openai.com/v1",
+  "max_tokens": 16384,
+  "temperature": 0.7,
+  "top_p": 1.0,
+  "timeout_ms": 120000,
+  "context_window": 128000,
+
+  "optimize_for_tool_calls": true,
+  "enable_thinking": false,
+
+  "tool_calling": {
+    "parallel_tool_calls": true,
+    "tool_choice": "auto",
+    "strict_mode": true,
+    "_comment": "GPT-4o has native parallel function calling. strict_mode ensures function call arguments match the schema exactly."
+  },
+
+  "structured_output": {
+    "json_mode": true,
+    "strict_json_schema": true,
+    "_comment": "GPT-4o supports structured outputs with guaranteed schema adherence via json_schema response format."
+  },
+
+  "dynamic_temperature": {
+    "enabled": true,
+    "decay": 0.5,
+    "floor": 0.2,
+    "_comment": "Lower temperature on retries for more deterministic tool call output."
+  },
+
+  "tool_call_batching": {
+    "enabled": true,
+    "system_prompt_suffix": "When multiple tools are needed, call ALL of them in a single response using parallel function calling.",
+    "_comment": "GPT-4o natively supports parallel function calls."
+  },
+
+  "pricing": {
+    "input_per_1m": 2.5,
+    "output_per_1m": 10.0,
+    "cached_input_per_1m": 1.25,
+    "currency": "USD",
+    "_comment": "As of 2025. Prompt caching available for 50% input discount."
+  },
+
+  "rate_limits": {
+    "requests_per_minute": 5000,
+    "tokens_per_minute": 800000,
+    "_comment": "Tier 3+ limits. Very high throughput."
+  },
+
+  "running_config": {
+    "description": "OpenAI hosted API. Requires OPENAI_API_KEY environment variable.",
+    "env_vars": {
+      "OPENAI_API_KEY": "<your-api-key>",
+      "UAP_MODEL_PROFILE": "gpt-4o"
+    }
+  }
+}

package/config/model-profiles/gpt-o3.json

@@ -0,0 +1,61 @@
+{
+  "_profile": "gpt-o3",
+  "_description": "OpenAI o3 (o3-2025-04-16). Reasoning model with chain-of-thought for complex problem solving. 200K context, native tool use, and configurable reasoning effort. Best for tasks requiring deep analysis, math, science, or multi-step planning.",
+
+  "model": "o3-2025-04-16",
+  "provider": "openai",
+  "api_base_url": "https://api.openai.com/v1",
+  "max_tokens": 100000,
+  "temperature": 1.0,
+  "top_p": 1.0,
+  "timeout_ms": 600000,
+  "context_window": 200000,
+
+  "optimize_for_tool_calls": true,
+  "enable_thinking": true,
+
+  "tool_calling": {
+    "parallel_tool_calls": true,
+    "tool_choice": "auto",
+    "_comment": "o3 supports tool use with reasoning. The model reasons about which tools to call and in what order."
+  },
+
+  "reasoning": {
+    "reasoning_effort": "medium",
+    "options": ["low", "medium", "high"],
+    "_comment": "Controls how much reasoning the model does. 'high' is most thorough but slowest and most expensive. 'low' is fastest."
+  },
+
+  "dynamic_temperature": {
+    "enabled": false,
+    "_comment": "o3 uses fixed temperature=1.0. Reasoning effort controls output quality instead."
+  },
+
+  "tool_call_batching": {
+    "enabled": true,
+    "system_prompt_suffix": "When multiple tools are needed, call ALL of them in a single response.",
+    "_comment": "o3 supports parallel tool calls."
+  },
+
+  "pricing": {
+    "input_per_1m": 2.0,
+    "output_per_1m": 8.0,
+    "cached_input_per_1m": 0.5,
+    "currency": "USD",
+    "_comment": "As of 2025. Reasoning tokens are billed as output tokens."
+  },
+
+  "rate_limits": {
+    "requests_per_minute": 1000,
+    "tokens_per_minute": 200000,
+    "_comment": "Lower throughput than GPT-4o due to reasoning overhead."
+  },
+
+  "running_config": {
+    "description": "OpenAI hosted API. Requires OPENAI_API_KEY environment variable.",
+    "env_vars": {
+      "OPENAI_API_KEY": "<your-api-key>",
+      "UAP_MODEL_PROFILE": "gpt-o3"
+    }
+  }
+}

package/config/model-profiles/kimi-k2.5.json

@@ -0,0 +1,70 @@
+{
+  "_profile": "kimi-k2.5",
+  "_description": "Moonshot Kimi K2.5 (kimi-k2.5). Open-weight MoE model (1T total, 32B active) with agentic tool use, 128K context, and strong coding ability. Runs locally via vLLM or via Moonshot API. Competitive with GPT-4o on coding benchmarks at a fraction of the cost.",
+
+  "model": "kimi-k2.5",
+  "provider": "moonshot",
+  "api_base_url": "https://api.moonshot.cn/v1",
+  "max_tokens": 16384,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "timeout_ms": 180000,
+  "context_window": 131072,
+
+  "optimize_for_tool_calls": true,
+  "enable_thinking": true,
+
+  "tool_calling": {
+    "parallel_tool_calls": true,
+    "tool_choice": "auto",
+    "_comment": "Kimi K2.5 has native tool calling with OpenAI-compatible function calling format. Strong agentic performance."
+  },
+
+  "thinking": {
+    "enabled": true,
+    "_comment": "K2.5 supports thinking mode for complex reasoning. Can be disabled for faster responses."
+  },
+
+  "dynamic_temperature": {
+    "enabled": true,
+    "decay": 0.5,
+    "floor": 0.2,
+    "_comment": "Lower temperature on retries for more deterministic tool call output."
+  },
+
+  "tool_call_batching": {
+    "enabled": true,
+    "system_prompt_suffix": "When multiple tools are needed, call ALL of them in a single response. Do not call one tool and wait - emit all tool calls together.",
+    "_comment": "K2.5 supports parallel function calls."
+  },
+
+  "local_deployment": {
+    "supported": true,
+    "framework": "vLLM",
+    "min_vram_gb": 80,
+    "recommended_vram_gb": 160,
+    "command": "vllm serve moonshotai/Kimi-K2.5 --tensor-parallel-size 4 --max-model-len 131072",
+    "_comment": "Open-weight model. 1T total params, 32B active (MoE). Requires multi-GPU setup for local deployment. Use API for single-GPU setups."
+  },
+
+  "pricing": {
+    "input_per_1m_cny": 2.0,
+    "output_per_1m_cny": 8.0,
+    "currency": "CNY",
+    "_comment": "As of 2025. Moonshot API pricing. Open weights available for self-hosting."
+  },
+
+  "rate_limits": {
+    "requests_per_minute": 500,
+    "tokens_per_minute": 500000,
+    "_comment": "Default tier limits via Moonshot API."
+  },
+
+  "running_config": {
+    "description": "Moonshot hosted API or self-hosted via vLLM. Requires MOONSHOT_API_KEY for hosted, or local vLLM setup.",
+    "env_vars": {
+      "MOONSHOT_API_KEY": "<your-api-key>",
+      "UAP_MODEL_PROFILE": "kimi-k2.5"
+    }
+  }
+}

package/config/model-profiles/llama.json

@@ -0,0 +1,86 @@
+{
+  "_profile": "llama",
+  "_description": "Optimized for Meta Llama 3.1/3.2/3.3 and Llama 4 models via llama.cpp or vLLM. Tuned for Llama's instruction-following strengths with tool calling support. Works with any Llama-family model (8B, 70B, 405B) at any quantization level.",
+
+  "model": "default",
+  "max_tokens": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "top_k": 40,
+  "min_p": 0.05,
+  "repetition_penalty": 1.1,
+  "timeout_ms": 120000,
+  "context_window": 131072,
+
+  "optimize_for_tool_calls": true,
+  "enable_thinking": false,
+
+  "dynamic_temperature": {
+    "enabled": true,
+    "decay": 0.5,
+    "floor": 0.2,
+    "_comment": "On retry, temp = max(floor, base_temp * decay^attempt). Reduces randomness for structured output."
+  },
+
+  "tool_call_batching": {
+    "enabled": true,
+    "system_prompt_suffix": "When multiple tools are needed, call ALL of them in a single response. Do not call one tool and wait - emit all tool calls together.",
+    "_comment": "Reduces round-trips by encouraging multi-tool emission in one turn."
+  },
+
+  "server_optimization": {
+    "flash_attention": true,
+    "kv_cache_type_k": "q8_0",
+    "kv_cache_type_v": "q4_0",
+    "prompt_cache": {
+      "enabled": true,
+      "slot_save_path": "./cache/slots"
+    },
+    "speculative_decoding": {
+      "enabled": false,
+      "_comment": "For Llama 70B+, use Llama-3.2-1B as draft model for 2-3x speedup."
+    },
+    "_comment": "KV q8/q4 split saves ~60% KV VRAM. Flash attn gives 1.5-2x speed on long context."
+  },
+
+  "quantization_profiles": {
+    "current": "Q4_K_M",
+    "upgrade_path": [
+      {
+        "quant": "Q4_K_M",
+        "vram_gb": 5,
+        "accuracy": "95%",
+        "tool_calls": "93%",
+        "note": "Llama 8B"
+      },
+      {
+        "quant": "Q4_K_M",
+        "vram_gb": 40,
+        "accuracy": "95%",
+        "tool_calls": "95%",
+        "note": "Llama 70B"
+      },
+      {
+        "quant": "Q5_K_M",
+        "vram_gb": 48,
+        "accuracy": "97%",
+        "tool_calls": "97%",
+        "note": "Llama 70B"
+      }
+    ],
+    "_comment": "Llama models scale well across quantization levels."
+  },
+
+  "running_config": {
+    "description": "Llama model via llama.cpp. Adjust --model path and --gpu-layers for your hardware.",
+    "llm_server": {
+      "description": "Llama via llama-server on port 8080",
+      "command": "llama-server --model /path/to/llama-model.gguf --port 8080 --host 0.0.0.0 --ctx-size 131072 --gpu-layers 99 --cache-type-k q8_0 --cache-type-v q4_0 -fa on --threads 8 --batch-size 512",
+      "vram_estimate": "~5GB (8B Q4_K_M) / ~40GB (70B Q4_K_M)"
+    },
+    "env_vars": {
+      "TARGET_URL": "http://127.0.0.1:8080",
+      "UAP_MODEL_PROFILE": "llama"
+    }
+  }
+}

package/config/model-profiles/qwen35.json

@@ -0,0 +1,218 @@
+{
+  "model": "qwen3.5-a3b-iq4xs",
+  "max_tokens": 81920,
+  "temperature": 0.3,
+  "top_p": 0.9,
+  "top_k": 20,
+  "min_p": 0.05,
+  "repetition_penalty": 1.0,
+  "stop_sequences": [],
+  "timeout_ms": 120000,
+  "context_window": 262144,
+  "optimize_for_tool_calls": true,
+  "mode_switch_buffer_tokens": 500,
+  "enable_thinking": false,
+  "chat_template_kwargs": {
+    "enable_thinking": false
+  },
+
+  "dynamic_temperature": {
+    "enabled": true,
+    "decay": 0.5,
+    "floor": 0.2,
+    "_comment": "On retry, temp = max(floor, base_temp * decay^attempt). Reduces randomness for structured output."
+  },
+
+  "tool_call_batching": {
+    "enabled": true,
+    "system_prompt_suffix": "When multiple tools are needed, call ALL of them in a single response. Do not call one tool and wait - emit all tool calls together.",
+    "_comment": "Reduces round-trips by encouraging multi-tool emission in one turn."
+  },
+
+  "structured_output": {
+    "grammar_file": "tools/agents/config/tool-call.gbnf",
+    "json_schema_file": "tools/agents/config/tool-call-schema.json",
+    "_comment": "GBNF grammar for per-request use only. Do NOT use --grammar-file on server startup — it forces tool_call output on all requests including plain chat. llama.cpp's autoparser handles tool call detection lazily via the chat template."
+  },
+
+  "server_optimization": {
+    "flash_attention": true,
+    "kv_cache_type_k": "q8_0",
+    "kv_cache_type_v": "q4_0",
+    "prompt_cache": {
+      "enabled": true,
+      "slot_save_path": "./cache/slots"
+    },
+    "speculative_decoding": {
+      "enabled": true,
+      "draft_model": "Qwen3.5-0.8B-Q8_0",
+      "draft_max": 16,
+      "draft_min": 3,
+      "draft_p_min": 0.75,
+      "_comment": "Enable for 2-3x speedup in tokens/sec. Uses Qwen3.5-0.8B-Q8_0 as draft model to propose tokens. Set enabled=true and provide draft model GGUF path. Current setup: main model 17GB, draft model 0.8GB, KV cache ~2-3GB."
+    },
+    "_comment": "KV q8/q4 split saves ~60% KV VRAM. Flash attn gives 1.5-2x speed on long context."
+  },
+
+  "lora": {
+    "enabled": false,
+    "adapter_path": "",
+    "scale": 1.0,
+    "training_config": "config/lora-finetune.yaml",
+    "data_generator": "tools/agents/scripts/generate_lora_training_data.py",
+    "_comment": "Fine-tune a LoRA adapter for +15-20% tool call reliability. See config/lora-finetune.yaml."
+  },
+
+  "quantization_profiles": {
+    "current": "iq4_xs",
+    "upgrade_path": [
+      { "quant": "IQ4_XS", "vram_gb": 17, "accuracy": "96%", "tool_calls": "94%" },
+      { "quant": "Q4_K_M", "vram_gb": 20, "accuracy": "95%", "tool_calls": "95%" },
+      { "quant": "Q5_K_M", "vram_gb": 24, "accuracy": "97%", "tool_calls": "97%" },
+      { "quant": "Q6_K", "vram_gb": 28, "accuracy": "98%", "tool_calls": "98%" }
+    ],
+    "_comment": "Each step up requires ~4-7GB more VRAM but improves instruction following."
+  },
+
+  "running_config": {
+    "description": "Dual-model llama.cpp setup: Qwen3.5 35B A3B for LLM + nomic-embed-text-v2-moe for embeddings. Run as two separate llama-server instances on different ports.",
+
+    "llm_server": {
+      "description": "Instance 1: Main LLM (Qwen3.5 35B A3B) — port 8080",
+      "command": "llama-server --model /path/to/Qwen3.5-35B-A3B-UD-IQ4_XS.gguf --chat-template-file chat_template.jinja --port 8080 --host 0.0.0.0 --ctx-size 131072 --gpu-layers 99 --cache-type-k q8_0 --cache-type-v q4_0 -fa on --threads 8 --batch-size 512 --ubatch-size 256 --mlock --metrics --n-predict 4096 --temp 0.3 --top-p 0.9 --top-k 20 --min-p 0.05",
+      "flags": [
+        "--model /path/to/Qwen3.5-35B-A3B-UD-IQ4_XS.gguf",
+        "--chat-template-file chat_template.jinja",
+        "--n-predict 4096",
+        "--temp 0.3",
+        "--top-p 0.9",
+        "--top-k 20",
+        "--min-p 0.05",
+        "--threads 8",
+        "--ctx-size 131072",
+        "--batch-size 512",
+        "--ubatch-size 256",
+        "--gpu-layers 99",
+        "--cache-type-k q8_0",
+        "--cache-type-v q4_0",
+        "--mlock",
+        "-fa on",
+        "--metrics",
+        "--host 0.0.0.0",
+        "--port 8080"
+      ],
+      "vram_estimate": "~20GB (17GB model + 0.8GB draft + 2-3GB KV cache)"
+    },
+
+    "embedding_server": {
+      "description": "Instance 2: Embedding model (nomic-embed-text-v2-moe) — port 8081. SoTA multilingual MoE embedding model: 475M total / 305M active params, 768 dims, 100+ languages, MIRACL 65.8.",
+      "model": "nomic-embed-text-v2-moe",
+      "model_url": "https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe-GGUF",
+      "recommended_quant": "Q4_K_M",
+      "quant_options": [
+        { "quant": "Q4_K_M", "size_mb": 328, "quality": "good, recommended for most use cases" },
+        { "quant": "Q5_K_M", "size_mb": 354, "quality": "high quality" },
+        { "quant": "Q6_K", "size_mb": 379, "quality": "very high quality, near perfect" },
+        { "quant": "Q8_0", "size_mb": 488, "quality": "extremely high, generally unneeded" },
+        { "quant": "F16", "size_mb": 913, "quality": "full precision, max quality" }
+      ],
+      "download_command": "huggingface-cli download nomic-ai/nomic-embed-text-v2-moe-GGUF nomic-embed-text-v2-moe.Q4_K_M.gguf --local-dir /path/to/models/",
+
+      "command_gpu": "llama-server --model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf --port 8081 --host 0.0.0.0 --embeddings --pooling mean --ctx-size 512 --gpu-layers 99 --batch-size 2048 --ubatch-size 512 --threads 4",
+      "command_cpu": "llama-server --model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf --port 8081 --host 0.0.0.0 --embeddings --pooling mean --ctx-size 512 --gpu-layers 0 --batch-size 2048 --threads 8",
+
+      "flags_gpu": [
+        "--model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf",
+        "--port 8081",
+        "--host 0.0.0.0",
+        "--embeddings",
+        "--pooling mean",
+        "--ctx-size 512",
+        "--gpu-layers 99",
+        "--batch-size 2048",
+        "--ubatch-size 512",
+        "--threads 4"
+      ],
+      "flags_cpu": [
+        "--model /path/to/nomic-embed-text-v2-moe.Q4_K_M.gguf",
+        "--port 8081",
+        "--host 0.0.0.0",
+        "--embeddings",
+        "--pooling mean",
+        "--ctx-size 512",
+        "--gpu-layers 0",
+        "--batch-size 2048",
+        "--threads 8"
+      ],
+      "vram_estimate_gpu": "~380MB (328MB model + ~50MB KV cache)",
+      "vram_estimate_cpu": "0 (runs entirely on CPU RAM, ~400MB)",
+      "task_prefixes": {
+        "search_query": "search_query: ",
+        "search_document": "search_document: ",
+        "_comment": "nomic-embed-text-v2 requires task prefixes. Queries use search_query:, documents use search_document:. The UAP embedding provider handles this automatically."
+      },
+      "env_var": "UAP_EMBEDDING_ENDPOINT=http://localhost:8081",
+      "_comment": "Set UAP_EMBEDDING_ENDPOINT env var to override the default endpoint. The embedding provider auto-detects dimensions from the model response."
+    },
+
+    "dual_model_vram_budget": {
+      "24gb_gpu": {
+        "llm": "~20GB (Qwen3.5 IQ4_XS + draft + KV)",
+        "embedding_gpu": "~380MB (nomic-embed Q4_K_M on GPU)",
+        "total": "~20.4GB — fits with headroom",
+        "recommendation": "Run both on GPU"
+      },
+      "16gb_gpu": {
+        "llm": "~17GB (Qwen3.5 IQ4_XS, no draft, smaller KV)",
+        "embedding_cpu": "0 GPU (run embedding on CPU)",
+        "total": "~17GB GPU + ~400MB CPU",
+        "recommendation": "Run LLM on GPU, embedding on CPU"
+      },
+      "48gb_gpu": {
+        "llm": "~20GB (Qwen3.5 IQ4_XS + draft + KV)",
+        "embedding_gpu": "~380MB",
+        "headroom": "~28GB free for larger context or better quants",
+        "recommendation": "Run both on GPU, consider Q5_K_M or Q6_K for LLM"
+      }
+    }
+  },
+
+  "optimization_recommendations": {
+    "current_performance": {
+      "vram_breakdown": {
+        "main_model": "17GB",
+        "draft_model": "0.8GB",
+        "kv_cache": "~2-3GB",
+        "total_estimated": "~20GB"
+      },
+      "speedup_estimate": "2-3x improvement in tokens/sec compared to standard generation",
+      "notes": "Speculative decoding is working as expected with the current setup. Main model (Qwen3.5-a3b-iq4xs) and draft model (Qwen3.5-0.8B-Q8_0) are optimized for tool calls."
+    },
+    "recommendations": [
+      {
+        "title": "Increase draft-max for longer contexts",
+        "description": "Adjust --draft-max to 32 for longer contexts (up to 32 tokens at once). Current setting is 16. This can improve throughput by 10-20% for longer generation tasks.",
+        "priority": "high",
+        "action": "Update speculative_decoding.draft_max to 32 in config"
+      },
+      {
+        "title": "Monitor KV cache usage",
+        "description": "KV cache is currently ~2-3GB. For longer contexts (131k tokens), consider adjusting --ctx-size or --batch-size if memory pressure occurs.",
+        "priority": "medium",
+        "action": "Monitor memory usage and adjust context window if needed"
+      },
+      {
+        "title": "Consider GPU-layers optimization",
+        "description": "Current setting is --gpu-layers 35. If VRAM allows (20GB+), consider increasing to 36 for better memory management.",
+        "priority": "low",
+        "action": "Evaluate GPU memory and adjust --gpu-layers if needed"
+      },
+      {
+        "title": "Enable flash attention",
+        "description": "Flash attention is already enabled. For further optimization, ensure --flash-attn flag is present in command line.",
+        "priority": "medium",
+        "action": "Verify --flash-attn is enabled in running config"
+      }
+    ]
+  }
+}