@abacus-ai/cli 2.0.0-canary.1 → 2.0.0-canary.11

package/src/__e2e__/llm-models.e2e.test.ts

@@ -1,402 +0,0 @@
-import * as fs from "fs";
-import * as path from "path";
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-const RESULTS_FILE = path.join(process.cwd(), "llm-models-test-results.txt");
-
-const API_BASE_URL = (() => {
-  const env = process.env.CODELLM_ENV || "prod";
-  const urls: Record<string, string> = {
-    prod: "https://apps.abacus.ai/api/v0",
-    preprod: "https://preprod-apps.abacus.ai/api/v0",
-    staging: "https://staging-apps.abacus.ai/api/v0",
-    "staging-latest": "https://staging-latest-apps.abacus.ai/api/v0",
-  };
-  const url = urls[env] || urls.prod;
-  console.log(`🌐 Using API environment: ${env} (${url})`);
-  return url;
-})();
-
-async function createConversationWithRealAPI(
-  apiKey: string,
-): Promise<{ conversationId: string; response: any }> {
-  const response = await fetch(`${API_BASE_URL}/_createCodellmDeploymentConversation`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      APIKEY: apiKey,
-    },
-    body: JSON.stringify({
-      deploymentType: "CODE_LLM_AGENT",
-    }),
-  });
-
-  const data = (await response.json()) as { result?: { deploymentConversationId: string } };
-  console.log("📝 Real API conversation response:", JSON.stringify(data, null, 2));
-  return {
-    conversationId: data.result?.deploymentConversationId || "test-id",
-    response: data,
-  };
-}
-
-async function fetchAvailableLLMs(
-  apiKey: string,
-): Promise<{ llmName: string; displayName: string }[]> {
-  const response = await fetch(`${API_BASE_URL}/_listCodeBots`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      APIKEY: apiKey,
-    },
-    body: JSON.stringify({
-      supportsThoughtStreaming: true,
-    }),
-  });
-
-  const data = (await response.json()) as { result?: { llmName: string; displayName: string }[] };
-  const llms = (data.result || []).map((llm: any) => ({
-    llmName: llm.llmName,
-    displayName: llm.displayName || llm.llmName,
-  }));
-  console.log(`📋 Found ${llms.length} available LLMs`);
-  return llms;
-}
-
-async function sendMessageWithRealAPI(
-  apiKey: string,
-  conversationId: string,
-  message: string,
-  llmName: string = "",
-): Promise<{ fullResponse: string; hasToolUse: boolean; toolsUsed: string[]; hasError: boolean }> {
-  const url = `${API_BASE_URL}/_codeLLMSendAgentMessageSSE`;
-
-  // Use proper tool object format as expected by the API
-  const supportedTools = [
-    {
-      name: "list_dir",
-      arguments: ["relativeWorkspacePath", "explanation"],
-      description: "List files in a directory",
-    },
-    {
-      name: "read_file",
-      arguments: [
-        "targetFile",
-        "startLineOneIndexed",
-        "endLineOneIndexed",
-        "shouldReadEntireFile",
-        "explanation",
-      ],
-      description: "Read a file",
-    },
-    {
-      name: "edit",
-      arguments: [
-        "targetFile",
-        "codeEdit",
-        "overwriteFile",
-        "instructions",
-        "startLine",
-        "endLine",
-      ],
-      description: "Create or edit a file",
-    },
-    { name: "delete_file", arguments: ["targetFile", "explanation"], description: "Delete a file" },
-    {
-      name: "grep",
-      arguments: [
-        "pattern",
-        "path",
-        "glob",
-        "output_mode",
-        "-B",
-        "-A",
-        "-C",
-        "-n",
-        "-i",
-        "type",
-        "head_limit",
-        "offset",
-        "multiline",
-      ],
-      description: "A powerful search tool built on ripgrep",
-    },
-    { name: "file_search", arguments: ["explanation", "query"], description: "Search for a file" },
-    {
-      name: "semantic_search_server",
-      arguments: ["command", "explanation"],
-      description: "Semantic search",
-    },
-    {
-      name: "run_in_terminal",
-      arguments: ["command", "explanation", "isBackground", "id"],
-      description: "Run a command in terminal",
-    },
-    { name: "get_terminal_output", arguments: ["id"], description: "Get terminal output" },
-    { name: "fetch_url", arguments: ["url"], description: "Fetch content from a URL" },
-    { name: "web_search", arguments: ["query", "explanation"], description: "Search the web" },
-    {
-      name: "upload_image",
-      arguments: ["targetFile", "explanation"],
-      description: "Upload an image",
-    },
-    { name: "todo_write", arguments: ["todos"], description: "Create task list" },
-    { name: "enter_plan_mode", arguments: [], description: "Enter plan mode" },
-    { name: "exit_plan_mode", arguments: ["planFilePath"], description: "Exit plan mode" },
-    { name: "subagent", arguments: ["agent", "task"], description: "Switch to subagent" },
-  ];
-
-  const body = {
-    llmName: llmName,
-    supportedTools: supportedTools,
-    mcpTools: [],
-    message: message,
-    deploymentConversationId: conversationId,
-    folderPaths: JSON.stringify([]),
-    rules: JSON.stringify([]),
-    requestId: `test-${Date.now()}`,
-    docInfos: [],
-    userChanges: JSON.stringify({}),
-    problemCount: 0,
-    runningTerminals: JSON.stringify([]),
-    supportsV2: true,
-    supportsThoughtStreaming: true,
-    supportsAbsolutePath: false,
-    isPrintMode: false,
-    supportsParallelToolUse: true,
-  };
-
-  const response = await fetch(url, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      APIKEY: apiKey,
-      "REAI-UI": "1",
-      Accept: "text/event-stream",
-    },
-    body: JSON.stringify(body),
-  });
-
-  let fullResponse = "";
-  let hasToolUse = false;
-  let hasError = false;
-  const toolsUsed: string[] = [];
-
-  const toolPatterns = [
-    "toolUseRequest",
-    '"name":"edit"',
-    '"name":"read_file"',
-    '"name":"list_dir"',
-    '"name":"delete_file"',
-    '"name":"grep"',
-    '"name":"file_search"',
-    '"name":"run_in_terminal"',
-    '"name":"enter_plan_mode"',
-    '"name":"exit_plan_mode"',
-  ];
-
-  if (!response.ok) {
-    const errorText = await response.text();
-    fullResponse = errorText;
-    console.log(`📡 API Error Response: ${errorText}`);
-    hasError = true;
-  } else {
-    const reader = response.body?.getReader();
-    const decoder = new TextDecoder();
-
-    if (reader) {
-      try {
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) {
-            break;
-          }
-
-          const chunk = decoder.decode(value, { stream: true });
-          fullResponse += chunk;
-
-          for (const pattern of toolPatterns) {
-            if (chunk.includes(pattern)) {
-              hasToolUse = true;
-              if (!toolsUsed.includes(pattern)) {
-                toolsUsed.push(pattern);
-              }
-            }
-          }
-
-          if (chunk.toLowerCase().includes("internal server error")) {
-            hasError = true;
-          }
-
-          // Check for success: false in JSON responses
-          if (chunk.includes('"success": false') || chunk.includes('"success":false')) {
-            hasError = true;
-          }
-        }
-      } finally {
-        reader.releaseLock();
-      }
-    }
-  }
-
-  console.log(`📡 Real API response (first 500 chars): ${fullResponse.substring(0, 500)}`);
-  console.log(`🔧 Tools detected: ${toolsUsed.join(", ") || "none"}`);
-  return { fullResponse, hasToolUse, toolsUsed, hasError };
-}
-
-describe.concurrent("LLM Models E2E Tests", () => {
-  const apiKey = process.env.ABACUS_API_KEY;
-  if (!apiKey) {
-    throw new Error("ABACUS_API_KEY is not set");
-  }
-
-  beforeEach(() => {
-    vi.spyOn(process, "exit").mockImplementation((() => {}) as unknown as typeof process.exit);
-  });
-
-  afterEach(() => {
-    vi.restoreAllMocks();
-  });
-
-  it("should test ALL available LLMs for edit tool usage", async () => {
-    console.log(`\n🔑 Using API key: ${apiKey.substring(0, 8)}...`);
-    console.log("\n🔑 Using REAL API to test ALL available LLMs...\n");
-
-    const availableLLMs = await fetchAvailableLLMs(apiKey);
-
-    if (availableLLMs.length === 0) {
-      console.error("❌ No LLMs available from API");
-      return;
-    }
-
-    console.log(`\n📋 Testing ${availableLLMs.length} LLMs:\n`);
-    availableLLMs.forEach((llm, i) => {
-      console.log(`   ${i + 1}. ${llm.displayName} (${llm.llmName})`);
-    });
-    console.log("");
-
-    const results: {
-      llmName: string;
-      displayName: string;
-      hasToolUse: boolean;
-      toolsUsed: string[];
-      error?: string;
-    }[] = [];
-
-    for (let i = 0; i < availableLLMs.length; i++) {
-      const llm = availableLLMs[i];
-      console.log(`\n--- Testing LLM ${i + 1}/${availableLLMs.length}: ${llm.displayName} ---`);
-
-      const { conversationId, response } = await createConversationWithRealAPI(apiKey);
-
-      if (!response.success) {
-        console.error(`❌ Failed to create conversation for ${llm.displayName}`);
-        results.push({
-          ...llm,
-          hasToolUse: false,
-          toolsUsed: [],
-          error: "Failed to create conversation",
-        });
-        continue;
-      }
-
-      console.log(`✅ Created conversation: ${conversationId}`);
-
-      try {
-        const testMessage = `Create a simple Python file called hello_${llm.llmName.replace(/[^a-z0-9]/g, "_")}.py that prints "Hello from ${llm.displayName}!"`;
-
-        const { fullResponse, hasToolUse, toolsUsed, hasError } = await sendMessageWithRealAPI(
-          apiKey,
-          conversationId,
-          testMessage,
-          llm.llmName,
-        );
-
-        if (hasError) {
-          console.error(`❌ ${llm.displayName} - Error detected`);
-          results.push({
-            ...llm,
-            hasToolUse: false,
-            toolsUsed: [],
-            error: "Internal server error",
-          });
-        } else {
-          results.push({ ...llm, hasToolUse, toolsUsed });
-
-          if (hasToolUse) {
-            console.log(`✅ ${llm.displayName} - Tool usage detected: ${toolsUsed.join(", ")}`);
-          } else {
-            console.error(`❌ ${llm.displayName} - No tool usage detected`);
-            console.error(`   Response: ${fullResponse.substring(0, 500)}`);
-          }
-        }
-      } catch (error) {
-        console.error(
-          `❌ Error testing ${llm.displayName}: ${error instanceof Error ? error.message : String(error)}`,
-        );
-        results.push({ ...llm, hasToolUse: false, toolsUsed: [], error: String(error) });
-      }
-
-      await sleep(1000);
-    }
-
-    console.log("\n\n📊 SUMMARY - LLM Tool Usage Test Results:\n");
-    console.log("=".repeat(60));
-
-    const withToolUse = results.filter((r) => r.hasToolUse);
-    const withoutToolUse = results.filter((r) => !r.hasToolUse && !r.error);
-    const withErrors = results.filter((r) => r.error);
-
-    console.log("\n✅ LLMs WITH tool usage (GOOD):");
-    withToolUse.forEach((r) =>
-      console.log(`   - ${r.displayName} (${r.llmName}): ${r.toolsUsed.join(", ")}`),
-    );
-
-    console.log("\n❌ LLMs WITHOUT tool usage (BAD):");
-    withoutToolUse.forEach((r) => console.log(`   - ${r.displayName} (${r.llmName})`));
-
-    if (withErrors.length > 0) {
-      console.log("\n⚠️  LLMs with errors:");
-      withErrors.forEach((r) => console.log(`   - ${r.displayName}: ${r.error}`));
-    }
-
-    console.log("\n" + "=".repeat(60));
-    console.log(`Total: ${results.length} LLMs tested`);
-    console.log(`  ✅ With tool usage: ${withToolUse.length}`);
-    console.log(`  ❌ No tool usage: ${withoutToolUse.length}`);
-    console.log(`  ⚠️  Errors: ${withErrors.length}`);
-
-    // Write results to file for debugging
-    const fileContent = [
-      "📊 LLM Tool Usage Test Results",
-      "=".repeat(60),
-      "",
-      "✅ LLMs WITH tool usage (GOOD):",
-      ...withToolUse.map((r) => `   - ${r.displayName} (${r.llmName}): ${r.toolsUsed.join(", ")}`),
-      "",
-      "❌ LLMs WITHOUT tool usage (BAD):",
-      ...withoutToolUse.map((r) => `   - ${r.displayName} (${r.llmName})`),
-      "",
-      "⚠️  LLMs with errors:",
-      ...withErrors.map((r) => `   - ${r.displayName}: ${r.error}`),
-      "",
-      "=".repeat(60),
-      `Total: ${results.length} LLMs tested`,
-      `  ✅ With tool usage: ${withToolUse.length}`,
-      `  ❌ No tool usage: ${withoutToolUse.length}`,
-      `  ⚠️  Errors: ${withErrors.length}`,
-    ].join("\n");
-    fs.writeFileSync(RESULTS_FILE, fileContent);
-    console.log(`\n📄 Results written to: ${RESULTS_FILE}`);
-
-    // Test fails if any LLM returns Internal Server Error or other errors
-    expect(withErrors.length).toBe(0);
-    // Test fails if any LLM doesn't show tool usage
-    expect(withoutToolUse.length).toBe(0);
-    // Sanity check: all LLMs should show tool usage
-    expect(withToolUse.length).toBe(results.length);
-    console.log(
-      `\n✅ Test completed - ALL ${withToolUse.length} LLMs showed tool usage as expected\n`,
-    );
-  }, 600000);
-});

package/src/__e2e__/mcp/mcp-callback-flow.e2e.test.tsx

@@ -1,71 +0,0 @@
-import type { SSEEvent } from "@codellm/api";
-
-import { describe, it, expect, vi } from "vitest";
-
-import { createMockAbacusClient } from "../helpers/test-helpers.js";
-
-/**
- * Integration Tests - These test the full streaming flow by examining how
- * the async generator events update the AgentProvider state.
- *
- * Note: Full app integration tests are complex due to provider dependencies.
- * These tests verify the streaming mechanism in isolation.
- */
-describe.concurrent("MCP Integration - Streaming Flow Tests", () => {
-  /**
-   * Test that verifies the AgentRunnerCore callback mechanism works correctly.
-   * We import the actual callback handlers and verify they update status.
-   */
-  it("onToolExecutionStart callback sets ExecutingTool status", async () => {
-    // This test verifies that the status enum values exist (basic sanity check)
-    const { AgentStatus } = await import("../../providers/agent.tsx");
-
-    expect(AgentStatus.Idle).toBe("idle");
-    expect(AgentStatus.Submitted).toBe("submitted");
-    expect(AgentStatus.Streaming).toBe("streaming");
-    expect(AgentStatus.ExecutingTool).toBe("executing-tool");
-
-    console.log("[Integration Test] AgentStatus enum values verified");
-    console.log("[Integration Test] Callback mechanism verified ✓");
-  });
-
-  /**
-   * Test that verifies the mock API client correctly simulates tool requests
-   * via the async generator streaming pattern.
-   */
-  it("Mock API client simulates tool request flow correctly via async generator", async () => {
-    const mockClient = createMockAbacusClient();
-
-    const events: SSEEvent[] = [];
-
-    console.log("[Mock Test] Starting streaming call simulation...");
-    const stream = mockClient.streamAgentMessage({
-      deploymentConversationId: "test-conversation-id",
-      message: "Create a hello world file",
-      supportedTools: [],
-    });
-
-    for await (const event of stream) {
-      events.push(event);
-      console.log("[Mock Test] Event received:", JSON.stringify(event).slice(0, 100));
-    }
-
-    console.log("[Mock Test] Streaming completed");
-    console.log(`[Mock Test] Events received: ${events.length}`);
-
-    // Verify the mock sent the expected events
-    expect(events.length).toBe(2);
-
-    // First event should be text segment
-    expect(events[0]?.["segment"]).toBeDefined();
-    console.log("[Mock Test] ✓ Text segment event received");
-
-    // Second event should be tool request
-    expect(events[1]?.["toolUseRequest"]).toBeDefined();
-    const toolReq = events[1]?.["toolUseRequest"] as { name: string };
-    expect(toolReq.name).toBe("edit");
-    console.log("[Mock Test] ✓ Tool request event received");
-
-    console.log("[Mock Test] Mock API streaming flow verified ✓");
-  }, 10000);
-});

package/src/__e2e__/mcp/mcp-full-app-ui.e2e.test.tsx

@@ -1,167 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-/**
- * Full App UI Rendering Tests
- * These tests render the app with proper provider wrapper and verify UI updates.
- */
-describe.sequential("MCP Full App UI Tests", () => {
-  /**
-   * Test that renders StatusIndicator inside the full provider stack
-   * and verifies it updates correctly when status changes.
-   */
-  it("StatusIndicator in full provider stack shows correct UI", async () => {
-    const { render, cleanup } = await import("../../lib/test-utils.tsx");
-    const { createTestWrapper } = await import("../helpers/test-helpers.ts");
-    const { StatusIndicator } = await import("../../components/status-indicator.tsx");
-    const { AgentStatus } = await import("../../providers/agent.tsx");
-    const React = await import("react");
-    const stripAnsi = (await import("strip-ansi")).default;
-
-    const TestWrapper = createTestWrapper();
-
-    console.log("[Full Stack Test] Rendering StatusIndicator with full provider stack...");
-
-    try {
-      // Render StatusIndicator inside the full provider wrapper
-      const instance = render(
-        React.createElement(
-          TestWrapper,
-          null,
-          React.createElement(StatusIndicator, { status: AgentStatus.ExecutingTool }),
-        ),
-        false,
-      );
-
-      await sleep(500);
-
-      const frame = instance.lastFrame() ?? "";
-      const plainFrame = stripAnsi(frame);
-
-      console.log("[Full Stack Test] Rendered frame:", plainFrame);
-
-      // Check if the indicator is visible
-      const hasIndicator = plainFrame.includes("...");
-
-      if (hasIndicator) {
-        console.log("[Full Stack Test] ✓ StatusIndicator rendered correctly with full providers");
-        expect(plainFrame).toContain("...");
-      } else {
-        console.log("[Full Stack Test] Frame content:", plainFrame.slice(0, 200));
-      }
-
-      // Test status transition within provider stack
-      console.log("[Full Stack Test] Testing status transitions...");
-
-      instance.rerender(
-        React.createElement(
-          TestWrapper,
-          null,
-          React.createElement(StatusIndicator, { status: AgentStatus.Idle }),
-        ),
-      );
-
-      const idleFrame = stripAnsi(instance.lastFrame() ?? "");
-      const idleHasIndicator = idleFrame.includes("...");
-
-      console.log("[Full Stack Test] Idle status - indicator visible:", idleHasIndicator);
-
-      // Idle should hide the indicator
-      if (!idleHasIndicator || idleFrame.trim() === "") {
-        console.log("[Full Stack Test] ✓ Idle correctly hides indicator");
-      }
-
-      console.log("[Full Stack Test] Full provider stack test completed ✓");
-    } finally {
-      cleanup();
-    }
-  }, 10000);
-
-  /**
-   * Test that simulates the full message → tool execution → UI update flow
-   * using mocked services within the provider stack.
-   */
-  it("Full provider stack handles tool execution status flow", async () => {
-    const { render, cleanup } = await import("../../lib/test-utils.tsx");
-    const { createTestWrapper } = await import("../helpers/test-helpers.ts");
-    const { StatusIndicator } = await import("../../components/status-indicator.tsx");
-    const { AgentStatus } = await import("../../providers/agent.tsx");
-    const React = await import("react");
-    const stripAnsi = (await import("strip-ansi")).default;
-
-    const TestWrapper = createTestWrapper();
-
-    console.log("[Flow Test] Simulating MCP tool execution flow with full providers...");
-
-    // Simulate the complete status flow
-    const statusFlow = [
-      { status: AgentStatus.Idle, name: "Initial (Idle)", shouldShow: false },
-      { status: AgentStatus.Submitted, name: "Message Submitted", shouldShow: true },
-      { status: AgentStatus.Streaming, name: "Streaming Response", shouldShow: true },
-      { status: AgentStatus.ExecutingTool, name: "MCP Tool Executing", shouldShow: true },
-      { status: AgentStatus.Streaming, name: "Streaming (after tool)", shouldShow: true },
-      { status: AgentStatus.Idle, name: "Complete (Idle)", shouldShow: false },
-    ];
-
-    const results: { name: string; visible: boolean; expected: boolean; match: boolean }[] = [];
-
-    try {
-      const instance = render(
-        React.createElement(
-          TestWrapper,
-          null,
-          React.createElement(StatusIndicator, { status: AgentStatus.Idle }),
-        ),
-        false,
-      );
-
-      await sleep(300);
-
-      for (const { status, name, shouldShow } of statusFlow) {
-        instance.rerender(
-          React.createElement(TestWrapper, null, React.createElement(StatusIndicator, { status })),
-        );
-
-        // Small delay to let React process
-        await sleep(50);
-
-        const frame = instance.lastFrame() ?? "";
-        const plainFrame = stripAnsi(frame);
-        const isVisible = plainFrame.includes("...") || plainFrame.includes("Resuming");
-
-        results.push({
-          name,
-          visible: isVisible,
-          expected: shouldShow,
-          match: isVisible === shouldShow,
-        });
-      }
-
-      console.log("[Flow Test] Status flow results:");
-      results.forEach((r) => {
-        const icon = r.match ? "✓" : "✗";
-        console.log(
-          `  ${icon} ${r.name}: ${r.visible ? "VISIBLE" : "hidden"} (expected: ${r.expected ? "VISIBLE" : "hidden"})`,
-        );
-      });
-
-      // Verify all states matched expectations
-      const allMatch = results.every((r) => r.match);
-      console.log(`[Flow Test] All states matched: ${allMatch ? "✓ YES" : "✗ NO"}`);
-
-      // The critical assertion: ExecutingTool MUST be visible
-      const executingToolResult = results.find((r) => r.name === "MCP Tool Executing");
-      expect(executingToolResult?.visible).toBe(true);
-
-      // All results should match expectations
-      for (const result of results) {
-        expect(result.match).toBe(true);
-      }
-
-      console.log("[Flow Test] Full provider stack flow test completed ✓");
-    } finally {
-      cleanup();
-    }
-  }, 10000);
-});