@tenex-chat/backend 0.9.1

package/src/llm/providers/key-manager.ts

@@ -0,0 +1,238 @@
+/**
+ * API Key Manager - Handles multi-key rotation, health tracking, and fallback
+ *
+ * This module provides key management for providers that have multiple API keys.
+ * It supports:
+ * - Random key selection from a pool of healthy keys
+ * - Failure tracking with configurable time windows
+ * - Temporary key disabling when failure threshold is exceeded
+ * - Automatic re-enabling after the disable window expires
+ *
+ * @module
+ */
+
+import { logger } from "@/utils/logger";
+
+/**
+ * Clock interface for injectable time source (enables deterministic testing)
+ */
+export interface Clock {
+    now(): number;
+}
+
+const systemClock: Clock = { now: () => Date.now() };
+
+/**
+ * Configuration for key health tracking
+ */
+export interface KeyManagerConfig {
+    /** Time window (ms) for counting failures. Failures older than this are ignored. Default: 60000 (1 min) */
+    failureWindowMs: number;
+    /** Number of failures within the window that triggers temporary disabling. Default: 3 */
+    failureThreshold: number;
+    /** How long (ms) a key stays disabled after hitting the threshold. Default: 300000 (5 min) */
+    disableDurationMs: number;
+    /** Injectable clock for testing. Defaults to system clock. */
+    clock?: Clock;
+}
+
+const DEFAULT_CONFIG: KeyManagerConfig = {
+    failureWindowMs: 60_000,
+    failureThreshold: 3,
+    disableDurationMs: 300_000,
+};
+
+/**
+ * Health state for a single API key
+ */
+interface KeyHealth {
+    /** Timestamps of recent failures */
+    failures: number[];
+    /** When the key becomes re-enabled (0 = not disabled) */
+    disabledUntil: number;
+}
+
+/**
+ * Manages API key pools, health tracking, and selection for LLM providers.
+ *
+ * Each provider can have one or more API keys. The manager tracks failures
+ * per key and temporarily disables keys that fail too often, allowing
+ * automatic fallback to healthy keys.
+ */
+export class KeyManager {
+    private keys: Map<string, string[]> = new Map();
+    private health: Map<string, KeyHealth> = new Map();
+    private config: KeyManagerConfig;
+    private clock: Clock;
+
+    constructor(config?: Partial<KeyManagerConfig>) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.clock = config?.clock ?? systemClock;
+    }
+
+    /**
+     * Register API keys for a provider.
+     * Normalizes single keys to arrays for uniform handling.
+     */
+    registerKeys(providerId: string, apiKey: string | string[]): void {
+        const keys = Array.isArray(apiKey) ? [...apiKey] : [apiKey];
+        if (keys.length === 0) {
+            return;
+        }
+        this.keys.set(providerId, keys);
+
+        // Initialize health tracking for new keys
+        for (const key of keys) {
+            const healthKey = this.healthKey(providerId, key);
+            if (!this.health.has(healthKey)) {
+                this.health.set(healthKey, { failures: [], disabledUntil: 0 });
+            }
+        }
+
+        if (keys.length > 1) {
+            logger.debug(`[KeyManager] Registered ${keys.length} keys for provider "${providerId}"`);
+        }
+    }
+
+    /**
+     * Select a random healthy key for a provider.
+     * Returns undefined if no healthy keys are available.
+     */
+    selectKey(providerId: string): string | undefined {
+        const keys = this.keys.get(providerId);
+        if (!keys || keys.length === 0) {
+            return undefined;
+        }
+
+        const healthy = keys.filter(key => this.isKeyHealthy(providerId, key));
+
+        if (healthy.length === 0) {
+            logger.warn(`[KeyManager] No healthy keys available for provider "${providerId}", trying all keys`);
+            // Fall back to all keys when everything is disabled — better than nothing
+            return keys[Math.floor(Math.random() * keys.length)];
+        }
+
+        return healthy[Math.floor(Math.random() * healthy.length)];
+    }
+
+    /**
+     * Report a key failure. If the failure threshold is reached within the
+     * configured window, the key is temporarily disabled.
+     */
+    reportFailure(providerId: string, apiKey: string): void {
+        const hKey = this.healthKey(providerId, apiKey);
+        const health = this.health.get(hKey);
+        if (!health) {
+            return;
+        }
+
+        const now = this.clock.now();
+        health.failures.push(now);
+
+        // Prune old failures outside the window
+        this.pruneFailures(health, now);
+
+        if (health.failures.length >= this.config.failureThreshold) {
+            health.disabledUntil = now + this.config.disableDurationMs;
+            const keyPreview = apiKey.slice(0, 8) + "...";
+            logger.warn(
+                `[KeyManager] Key ${keyPreview} for "${providerId}" temporarily disabled ` +
+                `(${health.failures.length} failures in ${this.config.failureWindowMs}ms window). ` +
+                `Re-enables in ${this.config.disableDurationMs / 1000}s`
+            );
+        }
+    }
+
+    /**
+     * Check whether a provider has multiple keys registered
+     */
+    hasMultipleKeys(providerId: string): boolean {
+        const keys = this.keys.get(providerId);
+        return !!keys && keys.length > 1;
+    }
+
+    /**
+     * Get the number of currently healthy keys for a provider
+     */
+    getHealthyKeyCount(providerId: string): number {
+        const keys = this.keys.get(providerId);
+        if (!keys) return 0;
+        return keys.filter(key => this.isKeyHealthy(providerId, key)).length;
+    }
+
+    /**
+     * Get all registered provider IDs
+     */
+    getRegisteredProviders(): string[] {
+        return Array.from(this.keys.keys());
+    }
+
+    /**
+     * Reset all state (for testing)
+     */
+    reset(): void {
+        this.keys.clear();
+        this.health.clear();
+    }
+
+    /**
+     * Check if a specific key is currently healthy (not temporarily disabled)
+     */
+    private isKeyHealthy(providerId: string, apiKey: string): boolean {
+        const hKey = this.healthKey(providerId, apiKey);
+        const health = this.health.get(hKey);
+        if (!health) return true;
+
+        const now = this.clock.now();
+
+        // Re-enable if disable period has passed
+        if (health.disabledUntil > 0 && now >= health.disabledUntil) {
+            health.disabledUntil = 0;
+            health.failures = [];
+            return true;
+        }
+
+        return health.disabledUntil === 0;
+    }
+
+    /**
+     * Remove failures outside the tracking window
+     */
+    private pruneFailures(health: KeyHealth, now: number): void {
+        const cutoff = now - this.config.failureWindowMs;
+        health.failures = health.failures.filter(ts => ts > cutoff);
+    }
+
+    /**
+     * Create a composite key for the health map
+     */
+    private healthKey(providerId: string, apiKey: string): string {
+        return `${providerId}:${apiKey}`;
+    }
+}
+
+/**
+ * Singleton instance shared across the application
+ */
+export const keyManager = new KeyManager();
+
+/**
+ * Resolve an API key that may be a single string or an array.
+ * For services that only need a single key (embeddings, image gen),
+ * this returns the first key from an array or the string itself.
+ */
+export function resolveApiKey(apiKey: string | string[] | undefined): string | undefined {
+    if (!apiKey) return undefined;
+    if (Array.isArray(apiKey)) return apiKey[0];
+    return apiKey;
+}
+
+/**
+ * Check whether an API key value represents a configured, usable key.
+ * Handles string, string[], undefined, and empty values.
+ */
+export function hasApiKey(apiKey: string | string[] | undefined): boolean {
+    if (!apiKey) return false;
+    if (Array.isArray(apiKey)) return apiKey.some(k => k.length > 0);
+    return apiKey.length > 0 && apiKey !== "none";
+}

package/src/llm/providers/ollama-models.ts

@@ -0,0 +1,105 @@
+import { logger } from "@/utils/logger";
+
+export interface OllamaModel {
+    name: string;
+    size: string;
+    modified: string;
+    digest: string;
+}
+
+/**
+ * Fetch available models from local Ollama instance
+ */
+export async function fetchOllamaModels(): Promise<OllamaModel[]> {
+    try {
+        const baseUrl = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
+        const response = await fetch(`${baseUrl}/api/tags`, {
+            method: "GET",
+            headers: {
+                "Content-Type": "application/json",
+            },
+        });
+
+        if (!response.ok) {
+            logger.warn(`Failed to fetch Ollama models: ${response.status}`);
+            return [];
+        }
+
+        interface OllamaResponse {
+            models?: Array<{
+                name: string;
+                size: number;
+                modified_at: string;
+                digest: string;
+            }>;
+        }
+
+        const data = (await response.json()) as OllamaResponse;
+        const models = data.models || [];
+
+        logger.debug(`Fetched ${models.length} Ollama models`);
+
+        return models.map((model) => ({
+            name: model.name,
+            size: formatSize(model.size),
+            modified: model.modified_at,
+            digest: model.digest,
+        }));
+    } catch (error) {
+        logger.warn("Failed to fetch Ollama models", {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        return [];
+    }
+}
+
+/**
+ * Format bytes to human readable size
+ */
+function formatSize(bytes: number): string {
+    if (!bytes) return "unknown";
+    const gb = bytes / (1024 * 1024 * 1024);
+    if (gb >= 1) {
+        return `${gb.toFixed(1)}GB`;
+    }
+    const mb = bytes / (1024 * 1024);
+    return `${mb.toFixed(0)}MB`;
+}
+
+/**
+ * Get popular Ollama models for quick selection
+ */
+export function getPopularOllamaModels(): Record<string, string[]> {
+    return {
+        "Small Models (< 4GB)": [
+            "llama3.2:1b",
+            "llama3.2:3b",
+            "phi3:mini",
+            "gemma2:2b",
+            "qwen2.5:1.5b",
+            "qwen2.5:3b",
+        ],
+        "Medium Models (4-8GB)": [
+            "llama3.1:8b",
+            "mistral:7b",
+            "gemma2:9b",
+            "qwen2.5:7b",
+            "deepseek-coder-v2:16b",
+        ],
+        "Large Models (> 8GB)": [
+            "llama3.1:70b",
+            "mixtral:8x7b",
+            "qwen2.5:14b",
+            "qwen2.5:32b",
+            "qwen2.5:72b",
+            "deepseek-coder-v2:236b",
+        ],
+        "Specialized Models": [
+            "codellama:7b",
+            "codellama:13b",
+            "starcoder2:3b",
+            "starcoder2:7b",
+            "sqlcoder:7b",
+        ],
+    };
+}

package/src/llm/providers/openrouter-models.ts

@@ -0,0 +1,102 @@
+/**
+ * OpenRouter model fetching utilities
+ */
+
+export interface OpenRouterModel {
+    id: string;
+    name: string;
+    description?: string;
+    context_length: number;
+    pricing: {
+        prompt: string;
+        completion: string;
+    };
+    top_provider?: {
+        max_completion_tokens?: number;
+    };
+}
+
+export interface OpenRouterModelsResponse {
+    data: OpenRouterModel[];
+}
+
+/**
+ * Fetch available models from OpenRouter API
+ */
+export async function fetchOpenRouterModels(): Promise<OpenRouterModel[]> {
+    try {
+        const response = await fetch("https://openrouter.ai/api/v1/models", {
+            headers: {
+                Accept: "application/json",
+            },
+        });
+
+        if (!response.ok) {
+            throw new Error(`Failed to fetch models: ${response.statusText}`);
+        }
+
+        const data = (await response.json()) as OpenRouterModelsResponse;
+
+        // Sort models by popularity/relevance (you can customize this)
+        return data.data.sort((a, b) => {
+            // Prioritize commonly used models
+            const priority = [
+                "openai/gpt-4",
+                "openai/gpt-4-turbo",
+                "anthropic/claude-3-5-sonnet",
+                "anthropic/claude-3-opus",
+                "google/gemini-2.0-flash",
+                "google/gemini-pro",
+            ];
+
+            const aIndex = priority.indexOf(a.id);
+            const bIndex = priority.indexOf(b.id);
+
+            if (aIndex !== -1 && bIndex !== -1) return aIndex - bIndex;
+            if (aIndex !== -1) return -1;
+            if (bIndex !== -1) return 1;
+
+            return a.name.localeCompare(b.name);
+        });
+    } catch (error) {
+        console.error("Error fetching OpenRouter models:", error);
+        return [];
+    }
+}
+
+/**
+ * Get popular models grouped by provider
+ */
+export function getPopularModels(): Record<string, string[]> {
+    return {
+        OpenAI: [
+            "openai/gpt-4",
+            "openai/gpt-4-turbo",
+            "openai/gpt-3.5-turbo",
+            "openai/o1-preview",
+            "openai/o1-mini",
+        ],
+        Anthropic: [
+            "anthropic/claude-3-5-sonnet",
+            "anthropic/claude-3-opus",
+            "anthropic/claude-3-haiku",
+            "anthropic/claude-3-5-haiku",
+        ],
+        Google: [
+            "google/gemini-2.0-flash-thinking-exp",
+            "google/gemini-2.0-flash-exp",
+            "google/gemini-pro",
+            "google/gemini-pro-1.5",
+        ],
+        Meta: [
+            "meta-llama/llama-3.1-405b-instruct",
+            "meta-llama/llama-3.1-70b-instruct",
+            "meta-llama/llama-3.1-8b-instruct",
+        ],
+        Mistral: [
+            "mistralai/mistral-large",
+            "mistralai/mixtral-8x22b-instruct",
+            "mistralai/mixtral-8x7b-instruct",
+        ],
+    };
+}

package/src/llm/providers/provider-ids.ts

@@ -0,0 +1,18 @@
+/**
+ * Provider ID constants
+ *
+ * These constants are the canonical source of truth for provider IDs.
+ * Use these instead of magic strings to prevent typos and mismatches.
+ */
+
+export const PROVIDER_IDS = {
+    CLAUDE_CODE: "claude-code",
+    CODEX_APP_SERVER: "codex-app-server",
+    OPENROUTER: "openrouter",
+    ANTHROPIC: "anthropic",
+    OPENAI: "openai",
+    OLLAMA: "ollama",
+    MOCK: "mock",
+} as const;
+
+export type ProviderId = (typeof PROVIDER_IDS)[keyof typeof PROVIDER_IDS];