@tenex-chat/backend 0.9.1

package/src/llm/utils/ConfigurationTester.ts

@@ -0,0 +1,229 @@
+import type { CompleteEvent, ContentEvent, StreamErrorEvent } from "@/llm/types";
+import { config as configService } from "@/services/ConfigService";
+import type { TenexLLMs } from "@/services/config/types";
+import { isMetaModelConfiguration } from "@/services/config/types";
+import chalk from "chalk";
+import inquirer from "inquirer";
+import { z } from "zod";
+import { llmServiceFactory } from "../LLMServiceFactory";
+
+/**
+ * Extended type for editor use - includes providers
+ */
+type TenexLLMsWithProviders = TenexLLMs & {
+    providers: Record<string, { apiKey: string | string[] }>;
+};
+
+/**
+ * Tests LLM configurations
+ */
+export class ConfigurationTester {
+    static async test(llmsConfig: TenexLLMsWithProviders): Promise<void> {
+        const configNames = Object.keys(llmsConfig.configurations);
+
+        if (configNames.length === 0) {
+            console.log(chalk.yellow("⚠️  No configurations to test"));
+            return;
+        }
+
+        const { name } = await inquirer.prompt([
+            {
+                type: "select",
+                name: "name",
+                message: "Select configuration to test:",
+                choices: configNames.map((n) => {
+                    const cfg = llmsConfig.configurations[n];
+                    const isMeta = isMetaModelConfiguration(cfg);
+                    const label = n === llmsConfig.default ? `${n} (default)` : n;
+                    return {
+                        name: isMeta ? `${label} [meta model]` : label,
+                        value: n,
+                    };
+                }),
+            },
+        ]);
+
+        try {
+            // Load full config first (needed for getLLMConfig and MCP server configs)
+            await configService.loadConfig();
+
+            // Use getLLMConfig to resolve meta models to their default variant
+            const llmConfig = configService.getLLMConfig(name);
+            const rawConfig = llmsConfig.configurations[name];
+            const isMeta = isMetaModelConfiguration(rawConfig);
+
+            console.log(chalk.yellow(`\nTesting configuration "${name}"${isMeta ? " (meta model - using default variant)" : ""}...`));
+            console.log(chalk.gray(`Provider: ${llmConfig.provider}, Model: ${llmConfig.model}`));
+
+            // Initialize providers before testing
+            await llmServiceFactory.initializeProviders(llmsConfig.providers);
+
+            // Create the service using the factory
+            const service = llmServiceFactory.createService(llmConfig);
+
+            console.log(chalk.cyan("📡 Sending test message..."));
+            const handleContent = (event: ContentEvent): void => {
+                process.stdout.write(chalk.cyan(event.delta));
+            };
+            service.on("content", handleContent);
+
+            const completePromise = new Promise<CompleteEvent>((resolve) => {
+                service.once("complete", resolve);
+            });
+            const errorPromise = new Promise<never>((_resolve, reject) => {
+                service.once("stream-error", (event: StreamErrorEvent) => {
+                    reject(event.error);
+                });
+            });
+
+            const completion = Promise.race([completePromise, errorPromise]);
+
+            console.log(chalk.white("Response: "));
+            const [, completeEvent] = await Promise.all([
+                service.stream(
+                    [{ role: "user", content: "Say 'Hello, TENEX!' in exactly those words." }],
+                    {}
+                ),
+                completion,
+            ]);
+
+            process.stdout.write("\n");
+            console.log(chalk.green("✅ Test successful!"));
+
+            // Show usage stats if available
+            if (completeEvent.usage) {
+                const usage = completeEvent.usage;
+                const inputTokens = usage.inputTokens ?? "?";
+                const outputTokens = usage.outputTokens ?? "?";
+                const totalTokens = usage.totalTokens ?? "?";
+                console.log(
+                    chalk.gray(`\nTokens: ${inputTokens} + ${outputTokens} = ${totalTokens}`)
+                );
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+        } catch (error: unknown) {
+            console.log(chalk.red("\n❌ Test failed!"));
+
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (errorMessage) {
+                console.log(chalk.red(`Error: ${errorMessage}`));
+            }
+
+            // Check for common issues
+            if (errorMessage?.includes("401") || errorMessage?.includes("Unauthorized")) {
+                console.log(chalk.yellow("\n💡 Invalid or expired API key"));
+            } else if (errorMessage?.includes("404")) {
+                console.log(chalk.yellow(`\n💡 Model for configuration '${name}' may not be available`));
+            } else if (errorMessage?.includes("rate limit")) {
+                console.log(chalk.yellow("\n💡 Rate limit hit. Please wait and try again"));
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, 3000));
+        }
+    }
+
+    /**
+     * Test a configuration for summarization using generateObject
+     */
+    static async testSummarization(llmsConfig: TenexLLMsWithProviders, configName: string): Promise<void> {
+        const rawConfig = llmsConfig.configurations[configName];
+        if (!rawConfig) {
+            console.log(chalk.red(`❌ Configuration "${configName}" not found`));
+            return;
+        }
+
+        // Use getLLMConfig to resolve meta models to their default variant
+        const llmConfig = configService.getLLMConfig(configName);
+        const isMeta = isMetaModelConfiguration(rawConfig);
+
+        console.log(chalk.yellow(`\nTesting summarization with "${configName}"${isMeta ? " (meta model - using default variant)" : ""}...`));
+        console.log(chalk.gray(`Provider: ${llmConfig.provider}, Model: ${llmConfig.model}`));
+
+        // Schema that mimics what we'd use for kind 513 summaries
+        const SummarySchema = z.object({
+            title: z.string().describe("A brief title for the summary"),
+            summary: z.string().describe("A concise summary of the conversation"),
+            keyPoints: z.array(z.string()).describe("Key points from the conversation"),
+        });
+
+        try {
+            // Load full config (needed for MCP server configs in agent providers)
+            await configService.loadConfig();
+
+            // Initialize providers before testing
+            await llmServiceFactory.initializeProviders(llmsConfig.providers);
+
+            // Create the service using the factory
+            const service = llmServiceFactory.createService(llmConfig);
+
+            console.log(chalk.cyan("📡 Testing generateObject..."));
+
+            const testConversation = `
+User: I need help setting up authentication for my web app.
+Assistant: I can help with that. What authentication method are you considering?
+User: I'm thinking OAuth with Google and GitHub.
+Assistant: Great choice. OAuth is secure and user-friendly. Let me outline the steps...
+`;
+
+            const result = await service.generateObject(
+                [
+                    {
+                        role: "system",
+                        content:
+                            "You are a helpful assistant that summarizes conversations. Generate a structured summary.",
+                    },
+                    {
+                        role: "user",
+                        content: `Summarize this conversation:\n${testConversation}`,
+                    },
+                ],
+                SummarySchema
+            );
+
+            console.log(chalk.green("\n✅ generateObject test successful!"));
+            console.log(chalk.white("\nGenerated summary:"));
+            console.log(chalk.cyan(`  Title: ${result.object.title}`));
+            console.log(chalk.cyan(`  Summary: ${result.object.summary}`));
+            console.log(chalk.cyan("  Key Points:"));
+            for (const point of result.object.keyPoints) {
+                console.log(chalk.cyan(`    • ${point}`));
+            }
+
+            // Show usage stats if available
+            if (result.usage) {
+                const { inputTokens, outputTokens, totalTokens } = result.usage;
+                console.log(chalk.gray(`\nTokens: ${inputTokens} + ${outputTokens} = ${totalTokens}`));
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+        } catch (error: unknown) {
+            console.log(chalk.red("\n❌ generateObject test failed!"));
+
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (errorMessage) {
+                console.log(chalk.red(`Error: ${errorMessage}`));
+            }
+
+            // Check for common issues
+            if (errorMessage?.includes("401") || errorMessage?.includes("Unauthorized")) {
+                console.log(chalk.yellow("\n💡 Invalid or expired API key"));
+            } else if (errorMessage?.includes("404")) {
+                console.log(chalk.yellow(`\n💡 Model '${llmConfig.model}' may not be available`));
+            } else if (errorMessage?.includes("rate limit")) {
+                console.log(chalk.yellow("\n💡 Rate limit hit. Please wait and try again"));
+            } else if (
+                errorMessage?.includes("structured output") ||
+                errorMessage?.includes("json")
+            ) {
+                console.log(
+                    chalk.yellow(
+                        "\n💡 This model may not support structured output (generateObject)"
+                    )
+                );
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, 3000));
+        }
+    }
+}

package/src/llm/utils/ModelSelector.ts

@@ -0,0 +1,212 @@
+import chalk from "chalk";
+import inquirer from "inquirer";
+import { fetchOllamaModels, getPopularOllamaModels } from "../providers/ollama-models";
+import { fetchOpenRouterModels, getPopularModels } from "../providers/openrouter-models";
+
+/**
+ * Utility class for interactive model selection
+ * Extracted from LLMConfigEditor to reduce complexity
+ */
+export class ModelSelector {
+    /**
+     * Select an Ollama model interactively
+     */
+    static async selectOllamaModel(currentModel?: string): Promise<string> {
+        console.log(chalk.gray("Fetching available Ollama models..."));
+
+        const ollamaModels = await fetchOllamaModels();
+
+        if (ollamaModels.length > 0) {
+            console.log(chalk.green(`✓ Found ${ollamaModels.length} installed models`));
+
+            const choices = [
+                ...ollamaModels.map((m) => ({
+                    name: `${m.name} ${chalk.gray(`(${m.size})`)}`,
+                    value: m.name,
+                })),
+                new inquirer.Separator(),
+                { name: chalk.cyan("→ Type model name manually"), value: "__manual__" },
+            ];
+
+            const { selectedModel } = await inquirer.prompt([
+                {
+                    type: "select",
+                    name: "selectedModel",
+                    message: "Select model:",
+                    choices,
+                    default: currentModel,
+                    pageSize: 15,
+                },
+            ]);
+
+            if (selectedModel === "__manual__") {
+                return await ModelSelector.promptManualModel(currentModel || "llama3.1:8b");
+            }
+
+            return selectedModel;
+        }
+        console.log(chalk.yellow("⚠️  No Ollama models found. Make sure Ollama is running."));
+        console.log(chalk.gray("Showing popular models (you'll need to pull them first)."));
+
+        const popular = getPopularOllamaModels();
+        const choices = [];
+        for (const [category, models] of Object.entries(popular)) {
+            choices.push(new inquirer.Separator(`--- ${category} ---`));
+            choices.push(
+                ...models.map((m) => ({
+                    name: m,
+                    value: m,
+                }))
+            );
+        }
+
+        choices.push(new inquirer.Separator());
+        choices.push({ name: chalk.cyan("→ Type model name manually"), value: "__manual__" });
+
+        const { selectedModel } = await inquirer.prompt([
+            {
+                type: "select",
+                name: "selectedModel",
+                message: "Select model:",
+                default: currentModel,
+                choices,
+                pageSize: 15,
+            },
+        ]);
+
+        if (selectedModel === "__manual__") {
+            return await ModelSelector.promptManualModel(currentModel || "llama3.1:8b");
+        }
+
+        return selectedModel;
+    }
+
+    /**
+     * Select an OpenRouter model interactively
+     */
+    static async selectOpenRouterModel(currentModel?: string): Promise<string> {
+        console.log(chalk.gray("Fetching available OpenRouter models..."));
+
+        const openRouterModels = await fetchOpenRouterModels();
+
+        if (openRouterModels.length > 0) {
+            console.log(chalk.green(`✓ Found ${openRouterModels.length} available models`));
+
+            // Group models by provider
+            const modelsByProvider: Record<string, typeof openRouterModels> = {};
+            for (const model of openRouterModels) {
+                const provider = model.id.split("/")[0] || "other";
+                if (!modelsByProvider[provider]) {
+                    modelsByProvider[provider] = [];
+                }
+                modelsByProvider[provider].push(model);
+            }
+
+            // Build choices
+            const choices = [];
+            const sortedProviders = Object.keys(modelsByProvider).sort();
+
+            for (const provider of sortedProviders) {
+                choices.push(
+                    new inquirer.Separator(chalk.yellow(`--- ${provider.toUpperCase()} ---`))
+                );
+                const providerModels = modelsByProvider[provider];
+
+                for (const model of providerModels) {
+                    const pricing = `$${model.pricing.prompt}/$${model.pricing.completion}/1M`;
+                    const context = `${Math.round(model.context_length / 1000)}k`;
+                    const freeTag = model.id.endsWith(":free") ? chalk.green(" [FREE]") : "";
+
+                    choices.push({
+                        name: `${model.id}${freeTag} ${chalk.gray(`- ${context} context, ${pricing}`)}`,
+                        value: model.id,
+                        short: model.id,
+                    });
+                }
+            }
+
+            choices.push(new inquirer.Separator());
+            choices.push({ name: chalk.cyan("→ Type model ID manually"), value: "__manual__" });
+
+            const { selectedModel } = await inquirer.prompt([
+                {
+                    type: "select",
+                    name: "selectedModel",
+                    message: "Select model:",
+                    choices,
+                    default: currentModel,
+                    pageSize: 20,
+                    loop: false,
+                },
+            ]);
+
+            if (selectedModel === "__manual__") {
+                return await ModelSelector.promptManualModel(currentModel || "openai/gpt-4");
+            }
+
+            return selectedModel;
+        }
+        console.log(chalk.yellow("⚠️  Failed to fetch models from OpenRouter API"));
+        console.log(
+            chalk.gray("You can still enter a model ID manually or select from popular models.")
+        );
+
+        const { selectionMethod } = await inquirer.prompt([
+            {
+                type: "select",
+                name: "selectionMethod",
+                message: "How would you like to select the model?",
+                choices: [
+                    { name: "Quick select from popular models", value: "quick" },
+                    { name: "Type model ID manually", value: "manual" },
+                ],
+            },
+        ]);
+
+        if (selectionMethod === "quick") {
+            const popular = getPopularModels();
+            const choices = [];
+            for (const [category, models] of Object.entries(popular)) {
+                choices.push(new inquirer.Separator(`--- ${category} ---`));
+                choices.push(
+                    ...models.map((m) => ({
+                        name: m,
+                        value: m,
+                    }))
+                );
+            }
+
+            const { selectedModel } = await inquirer.prompt([
+                {
+                    type: "select",
+                    name: "selectedModel",
+                    message: "Select model:",
+                    default: currentModel,
+                    choices,
+                    pageSize: 15,
+                },
+            ]);
+            return selectedModel;
+        }
+        return await ModelSelector.promptManualModel(currentModel || "openai/gpt-4");
+    }
+
+    /**
+     * Prompt for manual model entry
+     */
+    private static async promptManualModel(defaultValue: string): Promise<string> {
+        const { inputModel } = await inquirer.prompt([
+            {
+                type: "input",
+                name: "inputModel",
+                message: "Enter model name/ID:",
+                default: defaultValue,
+                validate: (input: string) => {
+                    if (!input.trim()) return "Model name is required";
+                    return true;
+                },
+            },
+        ]);
+        return inputModel;
+    }
+}

package/src/llm/utils/ProviderConfigUI.ts

@@ -0,0 +1,177 @@
+import { PROVIDER_IDS } from "@/llm/providers/provider-ids";
+import { resolveApiKey, hasApiKey } from "@/llm/providers/key-manager";
+import type { TenexLLMs } from "@/services/config/types";
+import chalk from "chalk";
+import inquirer from "inquirer";
+
+/**
+ * Extended type for editor use - includes providers
+ */
+type TenexLLMsWithProviders = TenexLLMs & {
+    providers: Record<string, { apiKey: string | string[] }>;
+};
+
+/**
+ * UI utilities for provider configuration
+ * Extracted from LLMConfigEditor to separate concerns
+ */
+export class ProviderConfigUI {
+    /**
+     * Get provider display names
+     */
+    static getProviderDisplayName(provider: string): string {
+        const names: Record<string, string> = {
+            [PROVIDER_IDS.OPENROUTER]: "OpenRouter (300+ models)",
+            [PROVIDER_IDS.ANTHROPIC]: "Anthropic (Claude)",
+            [PROVIDER_IDS.OPENAI]: "OpenAI (GPT)",
+            [PROVIDER_IDS.OLLAMA]: "Ollama (Local models)",
+            [PROVIDER_IDS.CLAUDE_CODE]: "Claude Code",
+            [PROVIDER_IDS.CODEX_APP_SERVER]: "Codex App Server (GPT-5.1/5.2)",
+        };
+        return names[provider] || provider;
+    }
+
+    /**
+     * Configure a specific provider interactively
+     */
+    static async configureProvider(
+        provider: string,
+        currentProviders?: Record<string, { apiKey: string | string[] }>
+    ): Promise<{ apiKey: string | string[] }> {
+        if (provider === PROVIDER_IDS.CLAUDE_CODE || provider === PROVIDER_IDS.CODEX_APP_SERVER) {
+            // Agent providers don't require an API key
+            console.log(
+                chalk.green(
+                    `✓ ${ProviderConfigUI.getProviderDisplayName(provider)} provider configured (no API key required)`
+                )
+            );
+            return { apiKey: "none" }; // Doesn't use API keys
+        }
+        if (provider === PROVIDER_IDS.OLLAMA) {
+            // For Ollama, ask for base URL instead of API key
+            const currentUrl = resolveApiKey(currentProviders?.[provider]?.apiKey) || "local";
+            const { ollamaConfig } = await inquirer.prompt([
+                {
+                    type: "select",
+                    name: "ollamaConfig",
+                    message: "Ollama configuration:",
+                    choices: [
+                        { name: "Use local Ollama (http://localhost:11434)", value: "local" },
+                        { name: "Use custom Ollama URL", value: "custom" },
+                    ],
+                    default: currentUrl === "local" ? "local" : "custom",
+                },
+            ]);
+
+            let baseUrl = "local";
+            if (ollamaConfig === "custom") {
+                const { customUrl } = await inquirer.prompt([
+                    {
+                        type: "input",
+                        name: "customUrl",
+                        message: "Enter Ollama base URL:",
+                        default: currentUrl !== "local" ? currentUrl : "http://localhost:11434",
+                        validate: (input: string) => {
+                            if (!input.trim()) return "URL is required";
+                            try {
+                                new URL(input);
+                                return true;
+                            } catch {
+                                return "Please enter a valid URL";
+                            }
+                        },
+                    },
+                ]);
+                baseUrl = customUrl;
+            }
+
+            return { apiKey: baseUrl };
+        }
+        // For other providers, ask for API key
+        const existingKey = currentProviders?.[provider]?.apiKey;
+
+        // If the provider already has a multi-key array, preserve it
+        if (Array.isArray(existingKey) && existingKey.length > 1) {
+            console.log(
+                chalk.cyan(
+                    `  ℹ  ${ProviderConfigUI.getProviderDisplayName(provider)} has ${existingKey.length} API keys configured (multi-key rotation).`
+                )
+            );
+            console.log(
+                chalk.gray(
+                    "     To edit multi-key configs, modify providers.json directly."
+                )
+            );
+            const { keep } = await inquirer.prompt([
+                {
+                    type: "confirm",
+                    name: "keep",
+                    message: "Keep existing multi-key configuration?",
+                    default: true,
+                },
+            ]);
+            if (keep) {
+                return { apiKey: existingKey };
+            }
+        }
+
+        const currentKey = resolveApiKey(existingKey);
+        const { apiKey } = await inquirer.prompt([
+            {
+                type: "password",
+                name: "apiKey",
+                message: `Enter API key for ${ProviderConfigUI.getProviderDisplayName(provider)} (press Enter to keep existing):`,
+                default: currentKey,
+                mask: "*",
+                validate: (input: string) => {
+                    // Allow empty input if there's an existing key
+                    if (!input.trim() && !currentKey) return "API key is required";
+                    return true;
+                },
+            },
+        ]);
+
+        return { apiKey: apiKey || currentKey || "" };
+    }
+
+    /**
+     * Display current configuration status
+     */
+    static displayCurrentConfig(llmsConfig: TenexLLMsWithProviders): void {
+        console.log(chalk.bold("Configured Providers:"));
+        const providers = Object.keys(llmsConfig.providers).filter(
+            (p) => hasApiKey(llmsConfig.providers[p]?.apiKey)
+        );
+        if (providers.length === 0) {
+            console.log(chalk.gray("  None configured"));
+        } else {
+            for (const p of providers) {
+                console.log(chalk.green(`  ✓ ${ProviderConfigUI.getProviderDisplayName(p)}`));
+            }
+        }
+
+        console.log(chalk.bold("\nConfigurations:"));
+        const configNames = Object.keys(llmsConfig.configurations);
+        if (configNames.length === 0) {
+            console.log(chalk.gray("  None"));
+        } else {
+            for (const name of configNames) {
+                const config = llmsConfig.configurations[name];
+                const isDefault = name === llmsConfig.default;
+                const isSummarization = name === llmsConfig.summarization;
+                const marker = isDefault || isSummarization ? chalk.cyan("• ") : "  ";
+
+                const tags: string[] = [];
+                if (isDefault) tags.push("default");
+                if (isSummarization) tags.push("summarization");
+                const tagStr = tags.length > 0 ? chalk.gray(` (${tags.join(", ")})`) : "";
+
+                // Handle meta models differently - they don't have a single model
+                const configDisplay = config.provider === "meta"
+                    ? `meta (${Object.keys((config as { variants: Record<string, unknown> }).variants).length} variants)`
+                    : `${config.provider}:${"model" in config ? config.model : "unknown"}`;
+                console.log(`  ${marker}${name}${tagStr}: ${configDisplay}`);
+            }
+        }
+    }
+}