@auxot/worker-cli 0.1.0

package/dist/llama-process.js

@@ -0,0 +1,203 @@
+/**
+ * llama.cpp Process Management
+ *
+ * Spawns and manages llama.cpp server process with parameters from GPU key policy.
+ *
+ * Handles:
+ * - Process lifecycle (start, stop, restart)
+ * - Crash detection and restart
+ * - stdout/stderr capture for debugging
+ * - Graceful shutdown
+ */
+import { spawn } from 'child_process';
+import { ensureLlamaBinary } from './llama-binary.js';
+/**
+ * Spawn llama.cpp server process with policy parameters
+ */
+export async function spawnLlamaCpp(options) {
+    // Ensure llama.cpp binary is downloaded
+    const binaryPath = options.binaryPath || await ensureLlamaBinary();
+    const { modelPath, contextSize, parallelism, port = 9002, host = '127.0.0.1', gpuLayers, } = options;
+    // Store original options for restart
+    const originalOptions = {
+        binaryPath,
+        modelPath,
+        contextSize,
+        parallelism,
+        port,
+        host,
+        gpuLayers,
+    };
+    // Build command arguments
+    const args = [
+        '--model', modelPath,
+        '--ctx-size', contextSize.toString(),
+        '--parallel', parallelism.toString(),
+        '--port', port.toString(),
+        '--host', host,
+        '--batch-size', '512',
+        '--threads', '12',
+        '--jinja', // Enable Jinja templating for tool calling
+    ];
+    // Add GPU layers if specified
+    if (gpuLayers !== undefined) {
+        args.push('--n-gpu-layers', gpuLayers.toString());
+    }
+    console.log(`[llama.cpp] Spawning process: ${binaryPath} ${args.join(' ')}`);
+    // Spawn process
+    const childProcess = spawn(binaryPath, args, {
+        stdio: ['ignore', 'pipe', 'pipe'], // stdin: ignore, stdout/stderr: pipe
+        env: process.env,
+    });
+    let isRunning = true;
+    const crashCallbacks = [];
+    // Capture stdout for debugging (with buffering for better formatting)
+    let stdoutBuffer = '';
+    childProcess.stdout?.on('data', (data) => {
+        stdoutBuffer += data.toString();
+        // Process complete lines
+        const lines = stdoutBuffer.split('\n');
+        stdoutBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (trimmed) {
+                console.log(`[llama.cpp stdout] ${trimmed}`);
+            }
+        }
+    });
+    // Capture stderr for debugging (with buffering for better formatting)
+    let stderrBuffer = '';
+    childProcess.stderr?.on('data', (data) => {
+        stderrBuffer += data.toString();
+        // Process complete lines
+        const lines = stderrBuffer.split('\n');
+        stderrBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (trimmed) {
+                console.error(`[llama.cpp stderr] ${trimmed}`);
+            }
+        }
+    });
+    // Handle process exit
+    childProcess.on('exit', (code, signal) => {
+        isRunning = false;
+        // Flush any remaining buffered output
+        if (stdoutBuffer.trim()) {
+            console.log(`[llama.cpp stdout] ${stdoutBuffer.trim()}`);
+            stdoutBuffer = '';
+        }
+        if (stderrBuffer.trim()) {
+            console.error(`[llama.cpp stderr] ${stderrBuffer.trim()}`);
+            stderrBuffer = '';
+        }
+        if (code !== null) {
+            console.log(`[llama.cpp] Process exited with code ${code}`);
+            // Notify crash callbacks (non-zero exit is a crash)
+            if (code !== 0) {
+                for (const callback of crashCallbacks) {
+                    callback(code, signal);
+                }
+            }
+        }
+        else if (signal) {
+            console.log(`[llama.cpp] Process killed with signal ${signal}`);
+            // Notify crash callbacks (killed by signal is a crash unless it's SIGTERM/SIGINT)
+            if (signal !== 'SIGTERM' && signal !== 'SIGINT') {
+                for (const callback of crashCallbacks) {
+                    callback(null, signal);
+                }
+            }
+        }
+    });
+    // Handle process errors
+    childProcess.on('error', (error) => {
+        console.error(`[llama.cpp] Process error:`, error);
+        isRunning = false;
+        // Notify crash callbacks
+        for (const callback of crashCallbacks) {
+            callback(null, null);
+        }
+    });
+    return {
+        process: childProcess,
+        get isRunning() {
+            return isRunning && childProcess.exitCode === null;
+        },
+        async stop() {
+            if (!isRunning) {
+                return;
+            }
+            console.log('[llama.cpp] Stopping process...');
+            isRunning = false;
+            // Try graceful shutdown first (SIGTERM)
+            childProcess.kill('SIGTERM');
+            // Wait up to 5 seconds for graceful shutdown
+            await new Promise((resolve) => {
+                const timeout = setTimeout(() => {
+                    // Force kill if still running
+                    if (childProcess.exitCode === null) {
+                        console.warn('[llama.cpp] Process did not exit gracefully, force killing...');
+                        childProcess.kill('SIGKILL');
+                    }
+                    resolve();
+                }, 5000);
+                childProcess.once('exit', () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+        },
+        async restart() {
+            console.log('[llama.cpp] Restarting process...');
+            // Stop current process
+            const stopPromise = (async () => {
+                if (childProcess && isRunning) {
+                    childProcess.kill('SIGTERM');
+                    // Wait for graceful shutdown
+                    await new Promise((resolve) => {
+                        const timeout = setTimeout(() => {
+                            if (childProcess && isRunning) {
+                                console.warn('[llama.cpp] Process did not exit gracefully, force killing...');
+                                childProcess.kill('SIGKILL');
+                            }
+                            resolve();
+                        }, 5000);
+                        childProcess.once('exit', () => {
+                            clearTimeout(timeout);
+                            resolve();
+                        });
+                    });
+                }
+            })();
+            await stopPromise;
+            isRunning = false;
+            // Create a new process with the same options
+            return await spawnLlamaCpp(originalOptions);
+        },
+        onCrash(callback) {
+            crashCallbacks.push(callback);
+        },
+    };
+}
+/**
+ * Wait for llama.cpp server to be ready (health check)
+ */
+export async function waitForLlamaReady(url, timeoutMs = 30000) {
+    const startTime = Date.now();
+    const checkInterval = 500; // Check every 500ms
+    while (Date.now() - startTime < timeoutMs) {
+        try {
+            const response = await fetch(`${url}/v1/models`);
+            if (response.ok) {
+                console.log('[llama.cpp] Server is ready');
+                return;
+            }
+        }
+        catch (error) {
+            // Server not ready yet, continue waiting
+        }
+        await new Promise((resolve) => setTimeout(resolve, checkInterval));
+    }
+    throw new Error(`llama.cpp server did not become ready within ${timeoutMs}ms`);
+}

package/dist/llama.js

@@ -0,0 +1,207 @@
+/**
+ * llama.cpp Integration
+ *
+ * Simple dumb router: forwards OpenAI-format jobs to llama.cpp and streams back tokens.
+ * NO prompt templates or business logic - server controls all prompts.
+ */
+import { createParser } from 'eventsource-parser';
+import { logClientToLlama, logLlamaToClient } from './debug.js';
+/**
+ * Process job by forwarding to llama.cpp
+ *
+ * Dumb router - receives OpenAI format, forwards to llama.cpp, streams back tokens.
+ *
+ * If job.max_tokens is not provided, uses capabilities.max_tokens_default to override
+ * llama.cpp's OpenAI API layer default (which is 2048, too restrictive).
+ *
+ * Supports cancellation via AbortSignal.
+ */
+export async function processJob(job, llamaUrl, capabilities, abortSignal, onToken) {
+    const startTime = Date.now();
+    // Prepare request to llama.cpp (OpenAI compatible format)
+    const request = {
+        model: 'default',
+        messages: job.messages,
+        temperature: job.temperature ?? 0.7,
+        stream: true,
+    };
+    // Include tools if provided (for function calling)
+    if (job.tools && job.tools.length > 0) {
+        request.tools = job.tools;
+    }
+    // Handle max_tokens:
+    // - If job specifies max_tokens, use it (explicit limit)
+    // - Otherwise, use capabilities.max_tokens_default from /props (server's default, often -1 for unlimited)
+    // - This overrides llama.cpp's OpenAI API compat layer default of 2048
+    if (job.max_tokens !== undefined) {
+        request.max_tokens = job.max_tokens;
+    }
+    else if (capabilities.max_tokens_default !== undefined) {
+        request.max_tokens = capabilities.max_tokens_default;
+    }
+    // Debug log request to llama.cpp (level 2)
+    logClientToLlama(request);
+    const requestBody = JSON.stringify(request);
+    const response = await fetch(`${llamaUrl}/v1/chat/completions`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: requestBody,
+        signal: abortSignal, // Pass abort signal for cancellation
+    });
+    if (!response.ok) {
+        // Try to get error details from response body
+        let errorDetails = '';
+        try {
+            const errorBody = await response.text();
+            errorDetails = errorBody ? ` - ${errorBody.substring(0, 500)}` : '';
+        }
+        catch {
+            // Ignore if we can't read error body
+        }
+        console.error(`[llama.cpp] Request failed:`, {
+            status: response.status,
+            statusText: response.statusText,
+            request: JSON.stringify(request, null, 2),
+            errorBody: errorDetails,
+        });
+        throw new Error(`llama.cpp request failed: ${response.status} ${response.statusText}${errorDetails}`);
+    }
+    if (!response.body) {
+        throw new Error('No response body from llama.cpp');
+    }
+    // Process stream
+    let fullResponse = '';
+    let timings = undefined;
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    // Track tool calls as they stream in
+    const toolCallsMap = new Map();
+    // Create SSE parser
+    const parser = createParser((event) => {
+        // Skip reconnection interval messages
+        if (event.type === 'reconnect-interval')
+            return;
+        // Check for [DONE] signal
+        if (event.data === '[DONE]')
+            return;
+        try {
+            const chunk = JSON.parse(event.data);
+            // Capture timings from the final chunk
+            if (chunk.timings) {
+                timings = chunk.timings;
+            }
+            // Extract content
+            const content = chunk.choices[0]?.delta?.content;
+            if (content) {
+                fullResponse += content;
+                onToken(content);
+            }
+            // Extract tool_calls (streamed incrementally)
+            const toolCalls = chunk.choices[0]?.delta?.tool_calls;
+            if (toolCalls) {
+                for (const tc of toolCalls) {
+                    const index = tc.index ?? 0;
+                    // Get or create tool call entry
+                    if (!toolCallsMap.has(index)) {
+                        toolCallsMap.set(index, {
+                            id: tc.id,
+                            type: 'function',
+                            function: {
+                                name: tc.function?.name,
+                                arguments: tc.function?.arguments || '',
+                            },
+                        });
+                    }
+                    else {
+                        // Append to existing tool call (arguments stream incrementally)
+                        const existing = toolCallsMap.get(index);
+                        if (tc.id)
+                            existing.id = tc.id;
+                        if (tc.function?.name)
+                            existing.function.name = tc.function.name;
+                        if (tc.function?.arguments) {
+                            existing.function.arguments = (existing.function.arguments || '') + tc.function.arguments;
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.error('[SSE Parser] Error parsing chunk:', error);
+        }
+    });
+    try {
+        while (true) {
+            // Check for abort before reading next chunk
+            if (abortSignal.aborted) {
+                break; // Exit gracefully with partial response
+            }
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            // Decode and feed to SSE parser
+            const text = decoder.decode(value, { stream: true });
+            // Debug log raw SSE chunk (level 2)
+            logLlamaToClient(text);
+            parser.feed(text);
+        }
+    }
+    catch (error) {
+        // If aborted, return partial response gracefully
+        if (error instanceof Error && error.name === 'AbortError') {
+            console.log(`[Job ${job.job_id}] Cancelled - returning partial response`);
+            // Don't throw - return what we have so far
+        }
+        else {
+            throw error; // Re-throw non-abort errors
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    // Use real timing data from llama.cpp if available
+    let durationMs;
+    let inputTokens;
+    let outputTokens;
+    if (timings) {
+        // TypeScript has narrowing issues with optional types in closures, so we explicitly type
+        const t = timings;
+        durationMs = Math.round(t.prompt_ms + t.predicted_ms);
+        inputTokens = t.prompt_n;
+        outputTokens = t.predicted_n;
+        console.log(`[Job ${job.job_id}] Completed in ${durationMs}ms (${inputTokens} prompt, ${outputTokens} predicted)`);
+    }
+    else {
+        // Fallback to estimates if no timing data
+        console.warn(`[Job ${job.job_id}] No timing data from llama.cpp`);
+        durationMs = Date.now() - startTime;
+        inputTokens = Math.ceil(JSON.stringify(job.messages).length / 4);
+        outputTokens = Math.ceil(fullResponse.length / 4);
+    }
+    // Convert tool calls map to array (if any)
+    const finalToolCalls = toolCallsMap.size > 0
+        ? Array.from(toolCallsMap.values())
+            .filter(tc => tc.id && tc.function.name) // Only include complete tool calls
+            .map(tc => ({
+            id: tc.id,
+            type: 'function',
+            function: {
+                name: tc.function.name,
+                arguments: tc.function.arguments || '{}',
+            },
+        }))
+        : undefined;
+    // Log tool calls if present
+    if (finalToolCalls && finalToolCalls.length > 0) {
+        console.log(`[Job ${job.job_id}] Detected ${finalToolCalls.length} tool call(s):`, finalToolCalls.map(tc => tc.function.name).join(', '));
+    }
+    return {
+        fullResponse,
+        durationMs,
+        inputTokens,
+        outputTokens,
+        tool_calls: finalToolCalls,
+    };
+}

package/dist/model-downloader.js

@@ -0,0 +1,145 @@
+/**
+ * Model Downloader
+ *
+ * Downloads GGUF model files from Hugging Face.
+ *
+ * Features:
+ * - Progress reporting
+ * - Resumable downloads (HTTP Range requests)
+ * - Integrity verification (file size)
+ * - Caching (checks if file exists)
+ */
+import { createWriteStream, existsSync, statSync } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+/**
+ * Download a model file from Hugging Face
+ *
+ * @param entry Model registry entry
+ * @param outputPath Full path where the file should be saved
+ * @param onProgress Optional progress callback (bytes downloaded, total bytes)
+ * @returns Path to downloaded file
+ */
+export async function downloadModel(entry, outputPath, onProgress) {
+    // Create output directory if it doesn't exist
+    const outputDir = dirname(outputPath);
+    if (!existsSync(outputDir)) {
+        await mkdir(outputDir, { recursive: true });
+    }
+    // Check if file already exists
+    if (existsSync(outputPath)) {
+        const stats = statSync(outputPath);
+        // If file size matches expected size, skip download
+        if (entry.file_size_bytes && stats.size === entry.file_size_bytes) {
+            console.log(`  ✓ Model already downloaded (${formatBytes(stats.size)})`);
+            return outputPath;
+        }
+        // If file exists but size doesn't match, delete it and re-download
+        if (entry.file_size_bytes && stats.size !== entry.file_size_bytes) {
+            console.log(`  ⊘ Existing file size mismatch (${formatBytes(stats.size)} vs ${formatBytes(entry.file_size_bytes)})`);
+            console.log(`  ⊘ Re-downloading...`);
+            // Delete the file - we'll download fresh
+            const { unlink } = await import('node:fs/promises');
+            await unlink(outputPath);
+        }
+    }
+    // Build Hugging Face download URL
+    // Format: https://huggingface.co/{repo_id}/resolve/main/{filename}
+    const downloadUrl = `https://huggingface.co/${entry.huggingface_id}/resolve/main/${entry.file_name}`;
+    console.log(`  Downloading from: ${entry.huggingface_id}`);
+    console.log(`  File: ${entry.file_name}`);
+    if (entry.file_size_bytes) {
+        console.log(`  Size: ${formatBytes(entry.file_size_bytes)}`);
+    }
+    // Download with progress tracking
+    let downloadedBytes = 0;
+    const totalBytes = entry.file_size_bytes || 0;
+    // Check if we can resume (partial file exists)
+    let startByte = 0;
+    if (existsSync(outputPath)) {
+        const stats = statSync(outputPath);
+        startByte = stats.size;
+        if (startByte > 0 && startByte < totalBytes) {
+            console.log(`  Resuming from ${formatBytes(startByte)}...`);
+            downloadedBytes = startByte;
+        }
+    }
+    const response = await fetch(downloadUrl, {
+        headers: startByte > 0 ? {
+            'Range': `bytes=${startByte}-`,
+        } : {},
+    });
+    if (!response.ok) {
+        if (response.status === 416) {
+            // Range not satisfiable - file already fully downloaded
+            if (existsSync(outputPath)) {
+                console.log(`  ✓ Download complete`);
+                return outputPath;
+            }
+        }
+        throw new Error(`Download failed: ${response.status} ${response.statusText}`);
+    }
+    const contentLength = response.headers.get('content-length');
+    const totalSize = contentLength ? parseInt(contentLength, 10) + startByte : totalBytes;
+    // Open file for writing (append if resuming)
+    const fileStream = createWriteStream(outputPath, { flags: startByte > 0 ? 'a' : 'w' });
+    // Stream response to file
+    const reader = response.body?.getReader();
+    if (!reader) {
+        throw new Error('Response body is not readable');
+    }
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            fileStream.write(value);
+            downloadedBytes += value.length;
+            // Report progress
+            if (onProgress) {
+                onProgress(downloadedBytes, totalSize);
+            }
+            else if (totalSize > 0) {
+                // Simple progress log every 10MB
+                if (downloadedBytes % (10 * 1024 * 1024) < value.length) {
+                    const percent = ((downloadedBytes / totalSize) * 100).toFixed(1);
+                    process.stdout.write(`\r  Progress: ${percent}% (${formatBytes(downloadedBytes)} / ${formatBytes(totalSize)})`);
+                }
+            }
+        }
+        fileStream.end();
+        // Wait for file stream to finish
+        await new Promise((resolve, reject) => {
+            fileStream.on('finish', resolve);
+            fileStream.on('error', reject);
+        });
+        if (totalSize > 0 && downloadedBytes !== totalSize) {
+            throw new Error(`Download incomplete: ${downloadedBytes} bytes downloaded, expected ${totalSize}`);
+        }
+        if (onProgress) {
+            // Clear progress line
+            process.stdout.write('\r');
+        }
+        else {
+            process.stdout.write('\r');
+        }
+        console.log(`  ✓ Download complete (${formatBytes(downloadedBytes)})`);
+        return outputPath;
+    }
+    catch (error) {
+        fileStream.destroy();
+        throw error;
+    }
+}
+/**
+ * Format bytes to human-readable string
+ */
+function formatBytes(bytes) {
+    if (bytes === 0)
+        return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
+}

package/dist/model-resolver.js

@@ -0,0 +1,80 @@
+/**
+ * Model Path Resolver
+ *
+ * Resolves model paths from policy using the model registry.
+ * Downloads models if not cached.
+ */
+import { loadRegistry, getModels } from '@auxot/model-registry';
+import { join } from 'path';
+import { homedir } from 'os';
+import { downloadModel } from './model-downloader.js';
+import { existsSync } from 'node:fs';
+/**
+ * Get model entry from registry based on policy
+ */
+export function getModelFromPolicy(policy) {
+    const registry = loadRegistry();
+    // Find model matching policy (model_name and quantization)
+    const matchingModels = getModels(registry, {
+        model_name: policy.model_name,
+    });
+    // Filter by quantization (case-insensitive)
+    const model = matchingModels.find((m) => m.quantization.toLowerCase() === policy.quantization.toLowerCase());
+    return model || null;
+}
+/**
+ * Get model path from policy, downloading if necessary
+ *
+ * Returns the local path for the model file.
+ * Downloads the model if it doesn't exist or is incomplete.
+ *
+ * @param policy GPU key policy
+ * @param onProgress Optional progress callback (downloaded, total)
+ * @returns Model file path, or null if model not found in registry
+ */
+export async function ensureModelDownloaded(policy, onProgress) {
+    const model = getModelFromPolicy(policy);
+    if (!model) {
+        console.error(`  ✗ Model not found in registry: ${policy.model_name} (${policy.quantization})`);
+        return null;
+    }
+    // Build model path
+    const modelsDir = process.env.AUXOT_MODELS_DIR || join(homedir(), '.auxot', 'models');
+    const modelDir = join(modelsDir, model.huggingface_id.replace('/', '_'));
+    const modelPath = join(modelDir, model.file_name);
+    // Check if model exists and is valid
+    if (existsSync(modelPath)) {
+        const { statSync } = await import('node:fs');
+        const stats = statSync(modelPath);
+        // If file size matches expected size, use cached model
+        if (model.file_size_bytes && stats.size === model.file_size_bytes) {
+            return modelPath;
+        }
+    }
+    // Download model
+    console.log(`  Downloading model: ${model.model_name} (${model.quantization})`);
+    try {
+        await downloadModel(model, modelPath, onProgress);
+        return modelPath;
+    }
+    catch (error) {
+        console.error(`  ✗ Download failed:`, error);
+        throw error;
+    }
+}
+/**
+ * Get model path without downloading (for checking if model exists)
+ *
+ * @param policy GPU key policy
+ * @returns Model file path, or null if model not found in registry
+ */
+export function getModelPath(policy) {
+    const model = getModelFromPolicy(policy);
+    if (!model) {
+        return null;
+    }
+    const modelsDir = process.env.AUXOT_MODELS_DIR || join(homedir(), '.auxot', 'models');
+    const modelDir = join(modelsDir, model.huggingface_id.replace('/', '_'));
+    const modelPath = join(modelDir, model.file_name);
+    return modelPath;
+}