@tenex-chat/backend 0.9.5 → 0.9.6

package/src/llm/utils/ConfigurationTester.ts

@@ -1,71 +1,52 @@
 import type { CompleteEvent, ContentEvent, StreamErrorEvent } from "@/llm/types";
 import { config as configService } from "@/services/ConfigService";
 import type { TenexLLMs } from "@/services/config/types";
-import { isMetaModelConfiguration } from "@/services/config/types";
-import chalk from "chalk";
-import inquirer from "inquirer";
-import { z } from "zod";
 import { llmServiceFactory } from "../LLMServiceFactory";
 
-/**
- * Extended type for editor use - includes providers
- */
 type TenexLLMsWithProviders = TenexLLMs & {
     providers: Record<string, { apiKey: string | string[] }>;
 };
 
-/**
- * Tests LLM configurations
- */
-export class ConfigurationTester {
-    static async test(llmsConfig: TenexLLMsWithProviders): Promise<void> {
-        const configNames = Object.keys(llmsConfig.configurations);
+export type TestResult = { success: true } | { success: false; error: string };
+
+function silenceConsole(): () => void {
+    const origLog = console.log;
+    const origWarn = console.warn;
+    const origError = console.error;
+    const origInfo = console.info;
+    const noop = () => {};
+    console.log = noop;
+    console.warn = noop;
+    console.error = noop;
+    console.info = noop;
+    return () => {
+        console.log = origLog;
+        console.warn = origWarn;
+        console.error = origError;
+        console.info = origInfo;
+    };
+}
 
-        if (configNames.length === 0) {
-            console.log(chalk.yellow("⚠️  No configurations to test"));
-            return;
+export class ConfigurationTester {
+    /**
+     * Run a silent test against a configuration. Returns pass/fail result.
+     * All console output is suppressed during the test.
+     */
+    static async runTest(llmsConfig: TenexLLMsWithProviders, configName: string): Promise<TestResult> {
+        if (!llmsConfig.configurations[configName]) {
+            return { success: false, error: "configuration not found" };
         }
 
-        const { name } = await inquirer.prompt([
-            {
-                type: "select",
-                name: "name",
-                message: "Select configuration to test:",
-                choices: configNames.map((n) => {
-                    const cfg = llmsConfig.configurations[n];
-                    const isMeta = isMetaModelConfiguration(cfg);
-                    const label = n === llmsConfig.default ? `${n} (default)` : n;
-                    return {
-                        name: isMeta ? `${label} [meta model]` : label,
-                        value: n,
-                    };
-                }),
-            },
-        ]);
+        const restoreConsole = silenceConsole();
 
         try {
-            // Load full config first (needed for getLLMConfig and MCP server configs)
             await configService.loadConfig();
+            const llmConfig = configService.getLLMConfig(configName);
 
-            // Use getLLMConfig to resolve meta models to their default variant
-            const llmConfig = configService.getLLMConfig(name);
-            const rawConfig = llmsConfig.configurations[name];
-            const isMeta = isMetaModelConfiguration(rawConfig);
-
-            console.log(chalk.yellow(`\nTesting configuration "${name}"${isMeta ? " (meta model - using default variant)" : ""}...`));
-            console.log(chalk.gray(`Provider: ${llmConfig.provider}, Model: ${llmConfig.model}`));
-
-            // Initialize providers before testing
             await llmServiceFactory.initializeProviders(llmsConfig.providers);
-
-            // Create the service using the factory
             const service = llmServiceFactory.createService(llmConfig);
 
-            console.log(chalk.cyan("📡 Sending test message..."));
-            const handleContent = (event: ContentEvent): void => {
-                process.stdout.write(chalk.cyan(event.delta));
-            };
-            service.on("content", handleContent);
+            service.on("content", (_event: ContentEvent) => {});
 
             const completePromise = new Promise<CompleteEvent>((resolve) => {
                 service.once("complete", resolve);
@@ -76,154 +57,30 @@ export class ConfigurationTester {
                 });
             });
 
-            const completion = Promise.race([completePromise, errorPromise]);
-
-            console.log(chalk.white("Response: "));
-            const [, completeEvent] = await Promise.all([
+            await Promise.all([
                 service.stream(
                     [{ role: "user", content: "Say 'Hello, TENEX!' in exactly those words." }],
                     {}
                 ),
-                completion,
+                Promise.race([completePromise, errorPromise]),
             ]);
 
-            process.stdout.write("\n");
-            console.log(chalk.green("✅ Test successful!"));
-
-            // Show usage stats if available
-            if (completeEvent.usage) {
-                const usage = completeEvent.usage;
-                const inputTokens = usage.inputTokens ?? "?";
-                const outputTokens = usage.outputTokens ?? "?";
-                const totalTokens = usage.totalTokens ?? "?";
-                console.log(
-                    chalk.gray(`\nTokens: ${inputTokens} + ${outputTokens} = ${totalTokens}`)
-                );
-            }
-
-            await new Promise((resolve) => setTimeout(resolve, 2000));
-        } catch (error: unknown) {
-            console.log(chalk.red("\n❌ Test failed!"));
-
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            if (errorMessage) {
-                console.log(chalk.red(`Error: ${errorMessage}`));
-            }
-
-            // Check for common issues
-            if (errorMessage?.includes("401") || errorMessage?.includes("Unauthorized")) {
-                console.log(chalk.yellow("\n💡 Invalid or expired API key"));
-            } else if (errorMessage?.includes("404")) {
-                console.log(chalk.yellow(`\n💡 Model for configuration '${name}' may not be available`));
-            } else if (errorMessage?.includes("rate limit")) {
-                console.log(chalk.yellow("\n💡 Rate limit hit. Please wait and try again"));
-            }
-
-            await new Promise((resolve) => setTimeout(resolve, 3000));
-        }
-    }
-
-    /**
-     * Test a configuration for summarization using generateObject
-     */
-    static async testSummarization(llmsConfig: TenexLLMsWithProviders, configName: string): Promise<void> {
-        const rawConfig = llmsConfig.configurations[configName];
-        if (!rawConfig) {
-            console.log(chalk.red(`❌ Configuration "${configName}" not found`));
-            return;
-        }
-
-        // Use getLLMConfig to resolve meta models to their default variant
-        const llmConfig = configService.getLLMConfig(configName);
-        const isMeta = isMetaModelConfiguration(rawConfig);
-
-        console.log(chalk.yellow(`\nTesting summarization with "${configName}"${isMeta ? " (meta model - using default variant)" : ""}...`));
-        console.log(chalk.gray(`Provider: ${llmConfig.provider}, Model: ${llmConfig.model}`));
-
-        // Schema that mimics what we'd use for kind 513 summaries
-        const SummarySchema = z.object({
-            title: z.string().describe("A brief title for the summary"),
-            summary: z.string().describe("A concise summary of the conversation"),
-            keyPoints: z.array(z.string()).describe("Key points from the conversation"),
-        });
-
-        try {
-            // Load full config (needed for MCP server configs in agent providers)
-            await configService.loadConfig();
-
-            // Initialize providers before testing
-            await llmServiceFactory.initializeProviders(llmsConfig.providers);
-
-            // Create the service using the factory
-            const service = llmServiceFactory.createService(llmConfig);
-
-            console.log(chalk.cyan("📡 Testing generateObject..."));
-
-            const testConversation = `
-User: I need help setting up authentication for my web app.
-Assistant: I can help with that. What authentication method are you considering?
-User: I'm thinking OAuth with Google and GitHub.
-Assistant: Great choice. OAuth is secure and user-friendly. Let me outline the steps...
-`;
-
-            const result = await service.generateObject(
-                [
-                    {
-                        role: "system",
-                        content:
-                            "You are a helpful assistant that summarizes conversations. Generate a structured summary.",
-                    },
-                    {
-                        role: "user",
-                        content: `Summarize this conversation:\n${testConversation}`,
-                    },
-                ],
-                SummarySchema
-            );
-
-            console.log(chalk.green("\n✅ generateObject test successful!"));
-            console.log(chalk.white("\nGenerated summary:"));
-            console.log(chalk.cyan(`  Title: ${result.object.title}`));
-            console.log(chalk.cyan(`  Summary: ${result.object.summary}`));
-            console.log(chalk.cyan("  Key Points:"));
-            for (const point of result.object.keyPoints) {
-                console.log(chalk.cyan(`    • ${point}`));
-            }
-
-            // Show usage stats if available
-            if (result.usage) {
-                const { inputTokens, outputTokens, totalTokens } = result.usage;
-                console.log(chalk.gray(`\nTokens: ${inputTokens} + ${outputTokens} = ${totalTokens}`));
-            }
-
-            await new Promise((resolve) => setTimeout(resolve, 2000));
+            return { success: true };
         } catch (error: unknown) {
-            console.log(chalk.red("\n❌ generateObject test failed!"));
-
             const errorMessage = error instanceof Error ? error.message : String(error);
-            if (errorMessage) {
-                console.log(chalk.red(`Error: ${errorMessage}`));
-            }
-
-            // Check for common issues
+            let hint = errorMessage;
             if (errorMessage?.includes("401") || errorMessage?.includes("Unauthorized")) {
-                console.log(chalk.yellow("\n💡 Invalid or expired API key"));
+                hint = "invalid or expired API key";
             } else if (errorMessage?.includes("404")) {
-                console.log(chalk.yellow(`\n💡 Model '${llmConfig.model}' may not be available`));
+                hint = "model not available";
             } else if (errorMessage?.includes("rate limit")) {
-                console.log(chalk.yellow("\n💡 Rate limit hit. Please wait and try again"));
-            } else if (
-                errorMessage?.includes("structured output") ||
-                errorMessage?.includes("json")
-            ) {
-                console.log(
-                    chalk.yellow(
-                        "\n💡 This model may not support structured output (generateObject)"
-                    )
-                );
+                hint = "rate limited";
             }
-
-            await new Promise((resolve) => setTimeout(resolve, 3000));
+            return { success: false, error: hint };
+        } finally {
+            // Delay restore so async logger stragglers are swallowed
+            await new Promise((resolve) => setTimeout(resolve, 200));
+            restoreConsole();
         }
     }
 }

package/src/llm/utils/ModelSelector.ts

@@ -1,7 +1,9 @@
 import chalk from "chalk";
 import inquirer from "inquirer";
-import { fetchOllamaModels, getPopularOllamaModels } from "../providers/ollama-models";
+import { amber, inquirerTheme } from "@/utils/cli-theme";
+import { fetchOllamaModels } from "../providers/ollama-models";
 import { fetchOpenRouterModels, getPopularModels } from "../providers/openrouter-models";
+import { ensureCacheLoaded, getProviderModels } from "./models-dev-cache";
 
 /**
  * Utility class for interactive model selection
@@ -9,7 +11,7 @@ import { fetchOpenRouterModels, getPopularModels } from "../providers/openrouter
  */
 export class ModelSelector {
     /**
-     * Select an Ollama model interactively
+     * Select an Ollama model interactively with fuzzy search
      */
     static async selectOllamaModel(currentModel?: string): Promise<string> {
         console.log(chalk.gray("Fetching available Ollama models..."));
@@ -19,23 +21,38 @@ export class ModelSelector {
         if (ollamaModels.length > 0) {
             console.log(chalk.green(`✓ Found ${ollamaModels.length} installed models`));
 
-            const choices = [
+            const allChoices = [
                 ...ollamaModels.map((m) => ({
                     name: `${m.name} ${chalk.gray(`(${m.size})`)}`,
                     value: m.name,
+                    short: m.name,
                 })),
-                new inquirer.Separator(),
-                { name: chalk.cyan("→ Type model name manually"), value: "__manual__" },
+                { name: chalk.cyan("→ Type model name manually"), value: "__manual__", short: "manual" },
             ];
 
             const { selectedModel } = await inquirer.prompt([
                 {
-                    type: "select",
+                    type: "search",
                     name: "selectedModel",
                     message: "Select model:",
-                    choices,
+                    source: (term: string | undefined) => {
+                        if (!term) return allChoices;
+                        const lower = term.toLowerCase();
+                        const filtered = allChoices.filter(
+                            (c) =>
+                                c.value === "__manual__" ||
+                                c.value.toLowerCase().includes(lower)
+                        );
+                        return filtered;
+                    },
                     default: currentModel,
-                    pageSize: 15,
+                    theme: {
+                        ...inquirerTheme,
+                        style: {
+                            ...inquirerTheme.style,
+                            searchTerm: (text: string) => amber(text || chalk.gray("Search models...")),
+                        },
+                    },
                 },
             ]);
 
@@ -45,44 +62,13 @@ export class ModelSelector {
 
             return selectedModel;
         }
-        console.log(chalk.yellow("⚠️  No Ollama models found. Make sure Ollama is running."));
-        console.log(chalk.gray("Showing popular models (you'll need to pull them first)."));
-
-        const popular = getPopularOllamaModels();
-        const choices = [];
-        for (const [category, models] of Object.entries(popular)) {
-            choices.push(new inquirer.Separator(`--- ${category} ---`));
-            choices.push(
-                ...models.map((m) => ({
-                    name: m,
-                    value: m,
-                }))
-            );
-        }
-
-        choices.push(new inquirer.Separator());
-        choices.push({ name: chalk.cyan("→ Type model name manually"), value: "__manual__" });
-
-        const { selectedModel } = await inquirer.prompt([
-            {
-                type: "select",
-                name: "selectedModel",
-                message: "Select model:",
-                default: currentModel,
-                choices,
-                pageSize: 15,
-            },
-        ]);
-
-        if (selectedModel === "__manual__") {
-            return await ModelSelector.promptManualModel(currentModel || "llama3.1:8b");
-        }
-
-        return selectedModel;
+        console.log(amber("⚠️  No local Ollama models were discovered."));
+        console.log(chalk.gray("Check `ollama list` and run `ollama pull <model>` if needed."));
+        return await ModelSelector.promptManualModel(currentModel || "llama3.1:8b");
     }
 
     /**
-     * Select an OpenRouter model interactively
+     * Select an OpenRouter model interactively with fuzzy search
      */
     static async selectOpenRouterModel(currentModel?: string): Promise<string> {
         console.log(chalk.gray("Fetching available OpenRouter models..."));
@@ -92,51 +78,42 @@ export class ModelSelector {
         if (openRouterModels.length > 0) {
             console.log(chalk.green(`✓ Found ${openRouterModels.length} available models`));
 
-            // Group models by provider
-            const modelsByProvider: Record<string, typeof openRouterModels> = {};
-            for (const model of openRouterModels) {
-                const provider = model.id.split("/")[0] || "other";
-                if (!modelsByProvider[provider]) {
-                    modelsByProvider[provider] = [];
-                }
-                modelsByProvider[provider].push(model);
-            }
+            const allChoices = openRouterModels.map((model) => {
+                const pricing = `$${model.pricing.prompt}/$${model.pricing.completion}/1M`;
+                const context = `${Math.round(model.context_length / 1000)}k`;
+                const freeTag = model.id.endsWith(":free") ? chalk.green(" [FREE]") : "";
 
-            // Build choices
-            const choices = [];
-            const sortedProviders = Object.keys(modelsByProvider).sort();
-
-            for (const provider of sortedProviders) {
-                choices.push(
-                    new inquirer.Separator(chalk.yellow(`--- ${provider.toUpperCase()} ---`))
-                );
-                const providerModels = modelsByProvider[provider];
-
-                for (const model of providerModels) {
-                    const pricing = `$${model.pricing.prompt}/$${model.pricing.completion}/1M`;
-                    const context = `${Math.round(model.context_length / 1000)}k`;
-                    const freeTag = model.id.endsWith(":free") ? chalk.green(" [FREE]") : "";
-
-                    choices.push({
-                        name: `${model.id}${freeTag} ${chalk.gray(`- ${context} context, ${pricing}`)}`,
-                        value: model.id,
-                        short: model.id,
-                    });
-                }
-            }
+                return {
+                    name: `${model.id}${freeTag} ${chalk.gray(`- ${context} ctx, ${pricing}`)}`,
+                    value: model.id,
+                    short: model.id,
+                };
+            });
 
-            choices.push(new inquirer.Separator());
-            choices.push({ name: chalk.cyan("→ Type model ID manually"), value: "__manual__" });
+            allChoices.push({ name: chalk.cyan("→ Type model ID manually"), value: "__manual__", short: "manual" });
 
             const { selectedModel } = await inquirer.prompt([
                 {
-                    type: "select",
+                    type: "search",
                     name: "selectedModel",
                     message: "Select model:",
-                    choices,
+                    source: (term: string | undefined) => {
+                        if (!term) return allChoices;
+                        const lower = term.toLowerCase();
+                        return allChoices.filter(
+                            (c) =>
+                                c.value === "__manual__" ||
+                                c.value.toLowerCase().includes(lower)
+                        );
+                    },
                     default: currentModel,
-                    pageSize: 20,
-                    loop: false,
+                    theme: {
+                        ...inquirerTheme,
+                        style: {
+                            ...inquirerTheme.style,
+                            searchTerm: (text: string) => amber(text || chalk.gray("Search models...")),
+                        },
+                    },
                 },
             ]);
 
@@ -146,7 +123,7 @@ export class ModelSelector {
 
             return selectedModel;
         }
-        console.log(chalk.yellow("⚠️  Failed to fetch models from OpenRouter API"));
+        console.log(amber("⚠️  Failed to fetch models from OpenRouter API"));
         console.log(
             chalk.gray("You can still enter a model ID manually or select from popular models.")
         );
@@ -160,30 +137,44 @@ export class ModelSelector {
                     { name: "Quick select from popular models", value: "quick" },
                     { name: "Type model ID manually", value: "manual" },
                 ],
+                theme: inquirerTheme,
             },
         ]);
 
         if (selectionMethod === "quick") {
             const popular = getPopularModels();
-            const choices = [];
+            const popularChoices: Array<{ name: string; value: string; short: string }> = [];
             for (const [category, models] of Object.entries(popular)) {
-                choices.push(new inquirer.Separator(`--- ${category} ---`));
-                choices.push(
-                    ...models.map((m) => ({
-                        name: m,
+                for (const m of models) {
+                    popularChoices.push({
+                        name: `${m} ${chalk.gray(`(${category})`)}`,
                         value: m,
-                    }))
-                );
+                        short: m,
+                    });
+                }
             }
 
             const { selectedModel } = await inquirer.prompt([
                 {
-                    type: "select",
+                    type: "search",
                     name: "selectedModel",
                     message: "Select model:",
-                    default: currentModel,
-                    choices,
-                    pageSize: 15,
+                    source: (term: string | undefined) => {
+                        if (!term) return popularChoices;
+                        const lower = term.toLowerCase();
+                        return popularChoices.filter(
+                            (c) =>
+                                c.value.toLowerCase().includes(lower) ||
+                                c.name.toLowerCase().includes(lower)
+                        );
+                    },
+                    theme: {
+                        ...inquirerTheme,
+                        style: {
+                            ...inquirerTheme.style,
+                            searchTerm: (text: string) => amber(text || chalk.gray("Search models...")),
+                        },
+                    },
                 },
             ]);
             return selectedModel;
@@ -191,6 +182,78 @@ export class ModelSelector {
         return await ModelSelector.promptManualModel(currentModel || "openai/gpt-4");
     }
 
+    /**
+     * Select a model from models.dev data (for Anthropic, OpenAI, etc.)
+     * Returns { id, name } so callers can use the human name for config naming.
+     */
+    static async selectModelsDevModel(
+        provider: string,
+        defaultModel?: string
+    ): Promise<{ id: string; name: string }> {
+        await ensureCacheLoaded();
+        const models = getProviderModels(provider);
+
+        if (models.length > 0) {
+            const allChoices = [
+                ...models.map((m) => {
+                    const ctx = m.limit?.context
+                        ? `${Math.round(m.limit.context / 1000)}k ctx`
+                        : "";
+                    const pricing = m.cost
+                        ? `$${m.cost.input}/$${m.cost.output}/M`
+                        : "";
+                    const meta = [ctx, pricing].filter(Boolean).join(", ");
+
+                    return {
+                        name: `${m.name} ${chalk.gray(`(${m.id})`)} ${chalk.gray(meta ? `- ${meta}` : "")}`,
+                        value: m.id,
+                        short: m.id,
+                        humanName: m.name,
+                    };
+                }),
+                { name: chalk.cyan("→ Type model ID manually"), value: "__manual__", short: "manual", humanName: "" },
+            ];
+
+            const { selectedModel } = await inquirer.prompt([
+                {
+                    type: "search",
+                    name: "selectedModel",
+                    message: "Select model:",
+                    source: (term: string | undefined) => {
+                        if (!term) return allChoices;
+                        const lower = term.toLowerCase();
+                        return allChoices.filter(
+                            (c) =>
+                                c.value === "__manual__" ||
+                                c.value.toLowerCase().includes(lower) ||
+                                c.humanName.toLowerCase().includes(lower)
+                        );
+                    },
+                    default: defaultModel,
+                    theme: {
+                        ...inquirerTheme,
+                        style: {
+                            ...inquirerTheme.style,
+                            searchTerm: (text: string) => amber(text || chalk.gray("Search models...")),
+                        },
+                    },
+                },
+            ]);
+
+            if (selectedModel === "__manual__") {
+                const id = await ModelSelector.promptManualModel(defaultModel || "");
+                return { id, name: id };
+            }
+
+            const selected = allChoices.find((c) => c.value === selectedModel);
+            return { id: selectedModel, name: selected?.humanName || selectedModel };
+        }
+
+        // No models.dev data available — fall back to manual
+        const id = await ModelSelector.promptManualModel(defaultModel || "");
+        return { id, name: id };
+    }
+
     /**
      * Prompt for manual model entry
      */
@@ -205,6 +268,7 @@ export class ModelSelector {
                     if (!input.trim()) return "Model name is required";
                     return true;
                 },
+                theme: inquirerTheme,
             },
         ]);
         return inputModel;