@draht/pods 2026.3.2-2

package/dist/model-configs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-configs.d.ts","sourceRoot":"","sources":["../src/model-configs.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AA2BtC;;GAEG;AACH,eAAO,MAAM,cAAc;;;;QAwD1B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,YAAY,8BAExB,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,cAAc,gBAE1B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,YAAY,6BAExB,CAAC","sourcesContent":["import { readFileSync } from \"fs\";\nimport { dirname, join } from \"path\";\nimport { fileURLToPath } from \"url\";\nimport type { GPU } from \"./types.js\";\n\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = dirname(__filename);\n\ninterface ModelConfig {\n\tgpuCount: number;\n\tgpuTypes?: string[];\n\targs: string[];\n\tenv?: Record<string, string>;\n\tnotes?: string;\n}\n\ninterface ModelInfo {\n\tname: string;\n\tconfigs: ModelConfig[];\n\tnotes?: string;\n}\n\ninterface ModelsData {\n\tmodels: Record<string, ModelInfo>;\n}\n\n// Load models configuration - resolve relative to this file\nconst modelsJsonPath = join(__dirname, \"models.json\");\nconst modelsData: ModelsData = JSON.parse(readFileSync(modelsJsonPath, \"utf-8\"));\n\n/**\n * Get the best configuration for a model based on available GPUs\n */\nexport const getModelConfig = (\n\tmodelId: string,\n\tgpus: GPU[],\n\trequestedGpuCount: number,\n): { args: string[]; env?: Record<string, string>; notes?: string } | null => {\n\tconst modelInfo = modelsData.models[modelId];\n\tif (!modelInfo) {\n\t\t// Unknown model, no default config\n\t\treturn null;\n\t}\n\n\t// Extract GPU type from the first GPU name (e.g., \"NVIDIA H200\" -> \"H200\")\n\tconst gpuType = gpus[0]?.name?.replace(\"NVIDIA\", \"\")?.trim()?.split(\" \")[0] || \"\";\n\n\t// Find best matching config\n\tlet bestConfig: ModelConfig | null = null;\n\n\tfor (const config of modelInfo.configs) {\n\t\t// Check GPU count\n\t\tif (config.gpuCount !== requestedGpuCount) {\n\t\t\tcontinue;\n\t\t}\n\n\t\t// Check GPU type if specified\n\t\tif (config.gpuTypes && config.gpuTypes.length > 0) {\n\t\t\tconst typeMatches = config.gpuTypes.some((type) => gpuType.includes(type) || type.includes(gpuType));\n\t\t\tif (!typeMatches) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t}\n\n\t\t// This config matches\n\t\tbestConfig = config;\n\t\tbreak;\n\t}\n\n\t// If no exact match, try to find a config with just the right GPU count\n\tif (!bestConfig) {\n\t\tfor (const config of modelInfo.configs) {\n\t\t\tif (config.gpuCount === requestedGpuCount) {\n\t\t\t\tbestConfig = config;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n\tif (!bestConfig) {\n\t\t// No suitable config found\n\t\treturn null;\n\t}\n\n\treturn {\n\t\targs: [...bestConfig.args],\n\t\tenv: bestConfig.env ? { ...bestConfig.env } : undefined,\n\t\tnotes: bestConfig.notes || modelInfo.notes,\n\t};\n};\n\n/**\n * Check if a model is known\n */\nexport const isKnownModel = (modelId: string): boolean => {\n\treturn modelId in modelsData.models;\n};\n\n/**\n * Get all known models\n */\nexport const getKnownModels = (): string[] => {\n\treturn Object.keys(modelsData.models);\n};\n\n/**\n * Get model display name\n */\nexport const getModelName = (modelId: string): string => {\n\treturn modelsData.models[modelId]?.name || modelId;\n};\n"]}

package/dist/model-configs.js

@@ -0,0 +1,75 @@
+import { readFileSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+// Load models configuration - resolve relative to this file
+const modelsJsonPath = join(__dirname, "models.json");
+const modelsData = JSON.parse(readFileSync(modelsJsonPath, "utf-8"));
+/**
+ * Get the best configuration for a model based on available GPUs
+ */
+export const getModelConfig = (modelId, gpus, requestedGpuCount) => {
+    const modelInfo = modelsData.models[modelId];
+    if (!modelInfo) {
+        // Unknown model, no default config
+        return null;
+    }
+    // Extract GPU type from the first GPU name (e.g., "NVIDIA H200" -> "H200")
+    const gpuType = gpus[0]?.name?.replace("NVIDIA", "")?.trim()?.split(" ")[0] || "";
+    // Find best matching config
+    let bestConfig = null;
+    for (const config of modelInfo.configs) {
+        // Check GPU count
+        if (config.gpuCount !== requestedGpuCount) {
+            continue;
+        }
+        // Check GPU type if specified
+        if (config.gpuTypes && config.gpuTypes.length > 0) {
+            const typeMatches = config.gpuTypes.some((type) => gpuType.includes(type) || type.includes(gpuType));
+            if (!typeMatches) {
+                continue;
+            }
+        }
+        // This config matches
+        bestConfig = config;
+        break;
+    }
+    // If no exact match, try to find a config with just the right GPU count
+    if (!bestConfig) {
+        for (const config of modelInfo.configs) {
+            if (config.gpuCount === requestedGpuCount) {
+                bestConfig = config;
+                break;
+            }
+        }
+    }
+    if (!bestConfig) {
+        // No suitable config found
+        return null;
+    }
+    return {
+        args: [...bestConfig.args],
+        env: bestConfig.env ? { ...bestConfig.env } : undefined,
+        notes: bestConfig.notes || modelInfo.notes,
+    };
+};
+/**
+ * Check if a model is known
+ */
+export const isKnownModel = (modelId) => {
+    return modelId in modelsData.models;
+};
+/**
+ * Get all known models
+ */
+export const getKnownModels = () => {
+    return Object.keys(modelsData.models);
+};
+/**
+ * Get model display name
+ */
+export const getModelName = (modelId) => {
+    return modelsData.models[modelId]?.name || modelId;
+};
+//# sourceMappingURL=model-configs.js.map

package/dist/model-configs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-configs.js","sourceRoot":"","sources":["../src/model-configs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAGpC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAoBtC,4DAA4D;AAC5D,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;AACtD,MAAM,UAAU,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;AAEjF;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAC7B,OAAe,EACf,IAAW,EACX,iBAAyB,EACiD,EAAE,CAAC;IAC7E,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,mCAAmC;QACnC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElF,4BAA4B;IAC5B,IAAI,UAAU,GAAuB,IAAI,CAAC;IAE1C,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACxC,kBAAkB;QAClB,IAAI,MAAM,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;YAC3C,SAAS;QACV,CAAC;QAED,8BAA8B;QAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YACrG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;QACF,CAAC;QAED,sBAAsB;QACtB,UAAU,GAAG,MAAM,CAAC;QACpB,MAAM;IACP,CAAC;IAED,wEAAwE;IACxE,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,MAAM,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;gBAC3C,UAAU,GAAG,MAAM,CAAC;gBACpB,MAAM;YACP,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,2BAA2B;QAC3B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,OAAO;QACN,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC;QAC1B,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS;QACvD,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK;KAC1C,CAAC;AAAA,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAe,EAAW,EAAE,CAAC;IACzD,OAAO,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC;AAAA,CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAa,EAAE,CAAC;IAC7C,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAAA,CACtC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC;IACxD,OAAO,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,IAAI,OAAO,CAAC;AAAA,CACnD,CAAC","sourcesContent":["import { readFileSync } from \"fs\";\nimport { dirname, join } from \"path\";\nimport { fileURLToPath } from \"url\";\nimport type { GPU } from \"./types.js\";\n\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = dirname(__filename);\n\ninterface ModelConfig {\n\tgpuCount: number;\n\tgpuTypes?: string[];\n\targs: string[];\n\tenv?: Record<string, string>;\n\tnotes?: string;\n}\n\ninterface ModelInfo {\n\tname: string;\n\tconfigs: ModelConfig[];\n\tnotes?: string;\n}\n\ninterface ModelsData {\n\tmodels: Record<string, ModelInfo>;\n}\n\n// Load models configuration - resolve relative to this file\nconst modelsJsonPath = join(__dirname, \"models.json\");\nconst modelsData: ModelsData = JSON.parse(readFileSync(modelsJsonPath, \"utf-8\"));\n\n/**\n * Get the best configuration for a model based on available GPUs\n */\nexport const getModelConfig = (\n\tmodelId: string,\n\tgpus: GPU[],\n\trequestedGpuCount: number,\n): { args: string[]; env?: Record<string, string>; notes?: string } | null => {\n\tconst modelInfo = modelsData.models[modelId];\n\tif (!modelInfo) {\n\t\t// Unknown model, no default config\n\t\treturn null;\n\t}\n\n\t// Extract GPU type from the first GPU name (e.g., \"NVIDIA H200\" -> \"H200\")\n\tconst gpuType = gpus[0]?.name?.replace(\"NVIDIA\", \"\")?.trim()?.split(\" \")[0] || \"\";\n\n\t// Find best matching config\n\tlet bestConfig: ModelConfig | null = null;\n\n\tfor (const config of modelInfo.configs) {\n\t\t// Check GPU count\n\t\tif (config.gpuCount !== requestedGpuCount) {\n\t\t\tcontinue;\n\t\t}\n\n\t\t// Check GPU type if specified\n\t\tif (config.gpuTypes && config.gpuTypes.length > 0) {\n\t\t\tconst typeMatches = config.gpuTypes.some((type) => gpuType.includes(type) || type.includes(gpuType));\n\t\t\tif (!typeMatches) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t}\n\n\t\t// This config matches\n\t\tbestConfig = config;\n\t\tbreak;\n\t}\n\n\t// If no exact match, try to find a config with just the right GPU count\n\tif (!bestConfig) {\n\t\tfor (const config of modelInfo.configs) {\n\t\t\tif (config.gpuCount === requestedGpuCount) {\n\t\t\t\tbestConfig = config;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n\tif (!bestConfig) {\n\t\t// No suitable config found\n\t\treturn null;\n\t}\n\n\treturn {\n\t\targs: [...bestConfig.args],\n\t\tenv: bestConfig.env ? { ...bestConfig.env } : undefined,\n\t\tnotes: bestConfig.notes || modelInfo.notes,\n\t};\n};\n\n/**\n * Check if a model is known\n */\nexport const isKnownModel = (modelId: string): boolean => {\n\treturn modelId in modelsData.models;\n};\n\n/**\n * Get all known models\n */\nexport const getKnownModels = (): string[] => {\n\treturn Object.keys(modelsData.models);\n};\n\n/**\n * Get model display name\n */\nexport const getModelName = (modelId: string): string => {\n\treturn modelsData.models[modelId]?.name || modelId;\n};\n"]}

package/dist/models.json

@@ -0,0 +1,295 @@
+{
+	"models": {
+		"Qwen/Qwen2.5-Coder-32B-Instruct": {
+			"name": "Qwen2.5-Coder-32B",
+			"configs": [
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--tool-call-parser", "hermes", "--enable-auto-tool-choice"]
+				},
+				{
+					"gpuCount": 2,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--tensor-parallel-size", "2", "--tool-call-parser", "hermes", "--enable-auto-tool-choice"]
+				}
+			]
+		},
+		"Qwen/Qwen3-Coder-30B-A3B-Instruct": {
+			"name": "Qwen3-Coder-30B",
+			"configs": [
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder"],
+					"notes": "Fits comfortably on single GPU. ~60GB model weight."
+				},
+				{
+					"gpuCount": 2,
+					"gpuTypes": ["H100", "H200"],
+					"args": [
+						"--tensor-parallel-size",
+						"2",
+						"--enable-auto-tool-choice",
+						"--tool-call-parser",
+						"qwen3_coder"
+					],
+					"notes": "For higher throughput/longer context."
+				}
+			]
+		},
+		"Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8": {
+			"name": "Qwen3-Coder-30B-FP8",
+			"configs": [
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder"],
+					"env": {
+						"VLLM_USE_DEEP_GEMM": "1"
+					},
+					"notes": "FP8 quantized, ~30GB model weight. Excellent for single GPU deployment."
+				}
+			]
+		},
+		"Qwen/Qwen3-Coder-480B-A35B-Instruct": {
+			"name": "Qwen3-Coder-480B",
+			"configs": [
+				{
+					"gpuCount": 8,
+					"gpuTypes": ["H200", "H20"],
+					"args": [
+						"--tensor-parallel-size",
+						"8",
+						"--max-model-len",
+						"32000",
+						"--enable-auto-tool-choice",
+						"--tool-call-parser",
+						"qwen3_coder"
+					],
+					"notes": "Cannot serve full 262K context on single node. Reduce max-model-len or increase gpu-memory-utilization."
+				}
+			]
+		},
+		"Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+			"name": "Qwen3-Coder-480B-FP8",
+			"configs": [
+				{
+					"gpuCount": 8,
+					"gpuTypes": ["H200", "H20"],
+					"args": [
+						"--max-model-len",
+						"131072",
+						"--enable-expert-parallel",
+						"--data-parallel-size",
+						"8",
+						"--enable-auto-tool-choice",
+						"--tool-call-parser",
+						"qwen3_coder"
+					],
+					"env": {
+						"VLLM_USE_DEEP_GEMM": "1"
+					},
+					"notes": "Use data-parallel mode (not tensor-parallel) to avoid weight quantization errors."
+				}
+			]
+		},
+		"openai/gpt-oss-20b": {
+			"name": "GPT-OSS-20B",
+			"configs": [
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--async-scheduling"]
+				},
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["B200"],
+					"args": ["--async-scheduling"],
+					"env": {
+						"VLLM_USE_TRTLLM_ATTENTION": "1",
+						"VLLM_USE_TRTLLM_DECODE_ATTENTION": "1",
+						"VLLM_USE_TRTLLM_CONTEXT_ATTENTION": "1",
+						"VLLM_USE_FLASHINFER_MXFP4_MOE": "1"
+					}
+				}
+			],
+			"notes": "Tools/function calls only  via /v1/responses endpoint."
+		},
+		"openai/gpt-oss-120b": {
+			"name": "GPT-OSS-120B",
+			"configs": [
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--async-scheduling", "--gpu-memory-utilization", "0.95", "--max-num-batched-tokens", "1024"],
+					"notes": "Single GPU deployment. Tools/function calls only via /v1/responses endpoint."
+				},
+				{
+					"gpuCount": 2,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--tensor-parallel-size", "2", "--async-scheduling", "--gpu-memory-utilization", "0.94"],
+					"notes": "Recommended for H100/H200. Tools/function calls only via /v1/responses endpoint."
+				},
+				{
+					"gpuCount": 4,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--tensor-parallel-size", "4", "--async-scheduling"],
+					"notes": "Higher throughput. Tools/function calls only via /v1/responses endpoint."
+				},
+				{
+					"gpuCount": 8,
+					"gpuTypes": ["H100", "H200"],
+					"args": ["--tensor-parallel-size", "8", "--async-scheduling"],
+					"notes": "Maximum throughput for evaluation workloads. Tools/function calls only via /v1/responses endpoint."
+				}
+			]
+		},
+		"zai-org/GLM-4.5": {
+			"name": "GLM-4.5",
+			"configs": [
+				{
+					"gpuCount": 16,
+					"gpuTypes": ["H100"],
+					"args": [
+						"--tensor-parallel-size",
+						"16",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					]
+				},
+				{
+					"gpuCount": 8,
+					"gpuTypes": ["H200"],
+					"args": [
+						"--tensor-parallel-size",
+						"8",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					]
+				}
+			],
+			"notes": "Models default to thinking mode. For full 128K context, double the GPU count."
+		},
+		"zai-org/GLM-4.5-FP8": {
+			"name": "GLM-4.5-FP8",
+			"configs": [
+				{
+					"gpuCount": 8,
+					"gpuTypes": ["H100"],
+					"args": [
+						"--tensor-parallel-size",
+						"8",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					]
+				},
+				{
+					"gpuCount": 4,
+					"gpuTypes": ["H200"],
+					"args": [
+						"--tensor-parallel-size",
+						"4",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					]
+				}
+			]
+		},
+		"zai-org/GLM-4.5-Air-FP8": {
+			"name": "GLM-4.5-Air-FP8",
+			"configs": [
+				{
+					"gpuCount": 2,
+					"gpuTypes": ["H100"],
+					"args": [
+						"--tensor-parallel-size",
+						"2",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					],
+					"env": {
+						"VLLM_ATTENTION_BACKEND": "XFORMERS"
+					},
+					"notes": "FP8 model requires vLLM with proper FP8 support or MTP module"
+				},
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H200"],
+					"args": ["--tool-call-parser", "glm45", "--reasoning-parser", "glm45", "--enable-auto-tool-choice"],
+					"env": {
+						"VLLM_ATTENTION_BACKEND": "XFORMERS"
+					},
+					"notes": "FP8 model requires vLLM with proper FP8 support or MTP module"
+				}
+			]
+		},
+		"zai-org/GLM-4.5-Air": {
+			"name": "GLM-4.5-Air",
+			"configs": [
+				{
+					"gpuCount": 2,
+					"gpuTypes": ["H100", "H200"],
+					"args": [
+						"--tensor-parallel-size",
+						"2",
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice"
+					],
+					"notes": "Non-quantized BF16 version, more compatible"
+				},
+				{
+					"gpuCount": 1,
+					"gpuTypes": ["H200"],
+					"args": [
+						"--tool-call-parser",
+						"glm45",
+						"--reasoning-parser",
+						"glm45",
+						"--enable-auto-tool-choice",
+						"--gpu-memory-utilization",
+						"0.95"
+					],
+					"notes": "Single H200 can fit the BF16 model with high memory utilization"
+				}
+			]
+		},
+		"moonshotai/Kimi-K2-Instruct": {
+			"name": "Kimi-K2",
+			"configs": [
+				{
+					"gpuCount": 16,
+					"gpuTypes": ["H200", "H20"],
+					"args": [
+						"--tensor-parallel-size",
+						"16",
+						"--trust-remote-code",
+						"--enable-auto-tool-choice",
+						"--tool-call-parser",
+						"kimi_k2"
+					],
+					"notes": "Pure TP mode. For >16 GPUs, combine with pipeline-parallelism."
+				}
+			],
+			"notes": "Requires vLLM v0.10.0rc1+. Minimum 16 GPUs for FP8 with 128k context."
+		}
+	}
+}

package/dist/scripts/model_run.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Model runner script - runs sequentially, killed by pi stop
+set -euo pipefail
+
+# These values are replaced before upload by pi CLI
+MODEL_ID="{{MODEL_ID}}"
+NAME="{{NAME}}"
+PORT="{{PORT}}"
+VLLM_ARGS="{{VLLM_ARGS}}"
+
+# Trap to ensure cleanup on exit and kill any child processes
+cleanup() {
+    local exit_code=$?
+    echo "Model runner exiting with code $exit_code"
+    # Kill any child processes
+    pkill -P $$ 2>/dev/null || true
+    exit $exit_code
+}
+trap cleanup EXIT TERM INT
+
+# Force colored output even when not a TTY
+export FORCE_COLOR=1
+export PYTHONUNBUFFERED=1
+export TERM=xterm-256color
+export RICH_FORCE_TERMINAL=1
+export CLICOLOR_FORCE=1
+
+# Source virtual environment
+source /root/venv/bin/activate
+
+echo "========================================="
+echo "Model Run: $NAME"
+echo "Model ID: $MODEL_ID"
+echo "Port: $PORT"
+if [ -n "$VLLM_ARGS" ]; then
+    echo "vLLM Args: $VLLM_ARGS"
+fi
+echo "========================================="
+echo ""
+
+# Download model (with color progress bars)
+echo "Downloading model (will skip if cached)..."
+HF_HUB_ENABLE_HF_TRANSFER=1 hf download "$MODEL_ID"
+
+if [ $? -ne 0 ]; then
+    echo "❌ ERROR: Failed to download model" >&2
+    exit 1
+fi
+
+echo ""
+echo "✅ Model download complete"
+echo ""
+
+# Build vLLM command
+VLLM_CMD="vllm serve '$MODEL_ID' --port $PORT --api-key '$PI_API_KEY'"
+if [ -n "$VLLM_ARGS" ]; then
+    VLLM_CMD="$VLLM_CMD $VLLM_ARGS"
+fi
+
+echo "Starting vLLM server..."
+echo "Command: $VLLM_CMD"
+echo "========================================="
+echo ""
+
+# Run vLLM in background so we can monitor it
+echo "Starting vLLM process..."
+bash -c "$VLLM_CMD" &
+VLLM_PID=$!
+
+# Monitor the vLLM process
+echo "Monitoring vLLM process (PID: $VLLM_PID)..."
+wait $VLLM_PID
+VLLM_EXIT_CODE=$?
+
+if [ $VLLM_EXIT_CODE -ne 0 ]; then
+    echo "❌ ERROR: vLLM exited with code $VLLM_EXIT_CODE" >&2
+    # Make sure to exit the script command too
+    kill -TERM $$ 2>/dev/null || true
+    exit $VLLM_EXIT_CODE
+fi
+
+echo "✅ vLLM exited normally"
+exit 0