@abacus-ai/cli 1.106.25008 → 2.0.0-canary.1

package/src/__e2e__/conversation-throttle.e2e.test.ts

@@ -0,0 +1,453 @@
+import type { AbacusClient } from "@codellm/api";
+
+import { AuthManager } from "@codellm/auth";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import React from "react";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+import { AgentModeProvider } from "../context/agent-mode.js";
+import { ExtensionFileProvider } from "../context/extension-file.js";
+import { NotificationProvider } from "../context/notification.js";
+import { ShellSizeProvider } from "../context/shell-size.js";
+import { ShellTitleProvider } from "../context/shell-title.js";
+import { Timing as _Timing } from "../lib/timing.js";
+import { AgentProvider } from "../providers/agent.js";
+import { ApiClientProvider } from "../providers/api-client.js";
+import { ThemeProvider } from "../theme/index.js";
+import {
+  renderE2E as _renderE2E,
+  simulateText as _simulateText,
+  simulateEnter as _simulateEnter,
+  waitForFrame as _waitForFrame,
+} from "./helpers/test-helpers.js";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export const _checkForUpgradePrompts = (frame: string, context: string): boolean => {
+  const frameLower = frame.toLowerCase();
+  const upgradeTerms = [
+    "upgrade",
+    "subscription",
+    "limit reached",
+    "pro plan",
+    "premium",
+    "credit limit",
+    "compute_point_budgeting",
+    "billing page",
+  ];
+  const foundTerms = upgradeTerms.filter((term) => frameLower.includes(term));
+
+  if (foundTerms.length > 0) {
+    console.error(`\n❌ UPGRADE/BILLING PROMPT DETECTED in ${context}!`);
+    console.error(`   Found terms: ${foundTerms.join(", ")}`);
+    console.error(`   Frame content:\n${frame}\n`);
+    return true;
+  }
+  console.log(`✅ No upgrade prompt in ${context}`);
+  return false;
+};
+
+const API_BASE_URL = (() => {
+  const env = process.env.CODELLM_ENV || "prod";
+  const urls: Record<string, string> = {
+    prod: "https://apps.abacus.ai/api/v0",
+    preprod: "https://preprod-apps.abacus.ai/api/v0",
+    staging: "https://staging-apps.abacus.ai/api/v0",
+    "staging-latest": "https://staging-latest-apps.abacus.ai/api/v0",
+  };
+  const url = urls[env] || urls.prod;
+  console.log(`🌐 Using API environment: ${env} (${url})`);
+  return url;
+})();
+
+async function createConversationWithRealAPI(
+  apiKey: string,
+): Promise<{ conversationId: string; response: any }> {
+  const response = await fetch(`${API_BASE_URL}/_createCodellmDeploymentConversation`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      APIKEY: apiKey,
+    },
+    body: JSON.stringify({
+      deploymentType: "CODE_LLM_AGENT",
+    }),
+  });
+
+  const data = (await response.json()) as { result?: { deploymentConversationId: string } };
+  console.log("📝 Real API conversation response:", JSON.stringify(data, null, 2));
+  return {
+    conversationId: data.result?.deploymentConversationId || "test-id",
+    response: data,
+  };
+}
+
+async function fetchAvailableLLMs(
+  apiKey: string,
+): Promise<{ llmName: string; displayName: string }[]> {
+  const response = await fetch(`${API_BASE_URL}/_listCodeBots`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      APIKEY: apiKey,
+    },
+    body: JSON.stringify({
+      supportsThoughtStreaming: true,
+    }),
+  });
+
+  const data = (await response.json()) as { result?: { llmName: string; displayName: string }[] };
+  const llms = (data.result || []).map((llm: any) => ({
+    llmName: llm.llmName,
+    displayName: llm.displayName || llm.llmName,
+  }));
+  console.log(`📋 Found ${llms.length} available LLMs`);
+  return llms;
+}
+
+async function sendMessageWithRealAPI(
+  apiKey: string,
+  conversationId: string,
+  message: string,
+  llmName: string = "",
+): Promise<{ fullResponse: string; hasUpgradePrompt: boolean }> {
+  const url = `${API_BASE_URL}/_codeLLMSendAgentMessageSSE`;
+
+  const body = {
+    llmName: llmName,
+    supportedTools: [],
+    mcpTools: [],
+    message: message,
+    deploymentConversationId: conversationId,
+    folderPaths: JSON.stringify([]),
+    rules: JSON.stringify([]),
+    requestId: `test-${Date.now()}`,
+    docInfos: [],
+    userChanges: JSON.stringify({}),
+    problemCount: 0,
+    runningTerminals: JSON.stringify([]),
+    supportsV2: true,
+    supportsThoughtStreaming: true,
+    supportsAbsolutePath: false,
+    isPrintMode: false,
+    supportsParallelToolUse: true,
+  };
+
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      APIKEY: apiKey,
+      "REAI-UI": "1",
+      Accept: "text/event-stream",
+    },
+    body: JSON.stringify(body),
+  });
+
+  let fullResponse = "";
+  let hasUpgradePrompt = false;
+
+  const upgradeTerms = [
+    "upgrade",
+    "subscription",
+    "limit reached",
+    "pro plan",
+    "premium",
+    "credit limit",
+    "compute_point_budgeting",
+    "billing page",
+    "billing_page",
+  ];
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    fullResponse = errorText;
+    console.log(`📡 API Error Response: ${errorText}`);
+    const lowerError = errorText.toLowerCase();
+    hasUpgradePrompt = upgradeTerms.some((term) => lowerError.includes(term));
+  } else {
+    const reader = response.body?.getReader();
+    const decoder = new TextDecoder();
+
+    if (reader) {
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+
+          const chunk = decoder.decode(value, { stream: true });
+          fullResponse += chunk;
+
+          const lowerChunk = chunk.toLowerCase();
+          if (upgradeTerms.some((term) => lowerChunk.includes(term))) {
+            hasUpgradePrompt = true;
+          }
+        }
+      } finally {
+        reader.releaseLock();
+      }
+    }
+  }
+
+  console.log(`📡 Real API message response (first 500 chars): ${fullResponse.substring(0, 500)}`);
+  return { fullResponse, hasUpgradePrompt };
+}
+
+export function _createApiClientWithRealCalls(apiKey: string): AbacusClient {
+  const mockStorage: Storage = {
+    length: 0,
+    clear: vi.fn(),
+    getItem: (key: string) => {
+      if (key === "codellm.apikey") {
+        return apiKey;
+      }
+      return null;
+    },
+    key: vi.fn().mockReturnValue(null),
+    removeItem: vi.fn(),
+    setItem: vi.fn(),
+  };
+
+  return {
+    storage: mockStorage,
+    getUserInfo: vi.fn().mockResolvedValue({ result: { email: "test@test.com" } }),
+    sendAgentMessageSSE: vi.fn().mockResolvedValue({}),
+    streamAgentMessage: vi.fn().mockImplementation(function () {
+      console.log("📡 streamAgentMessage called with real API key");
+      async function* generate() {
+        await sleep(500);
+
+        yield {
+          segment: "I will help you with that request.",
+          counter: 1,
+          message_id: "real-message-id",
+        };
+
+        yield {
+          toolUseRequest: {
+            id: "tool-1",
+            type: "builtin",
+            name: "edit",
+            input: {
+              targetFile: "test.py",
+              codeEdit: 'print("test")',
+              overwriteFile: true,
+              instructions: "Creating test file",
+            },
+          },
+          counter: 2,
+          message_id: "real-message-id",
+        };
+      }
+      return generate();
+    }),
+    cancelConversationRequest: vi.fn().mockResolvedValue({}),
+    getLLMOptions: vi.fn().mockResolvedValue({ result: [] }),
+    getConversation: vi.fn().mockResolvedValue({ result: { history: [] } }),
+    createConversation: vi.fn().mockImplementation(async () => {
+      const result = await createConversationWithRealAPI(apiKey);
+      return { success: true, result: { deploymentConversationId: result.conversationId } };
+    }),
+    createCodellmDeploymentConversation: vi.fn().mockImplementation(async () => {
+      const result = await createConversationWithRealAPI(apiKey);
+      return { success: true, result: { deploymentConversationId: result.conversationId } };
+    }),
+    listConversations: vi.fn().mockResolvedValue({ result: [] }),
+    listExternalApplications: vi.fn().mockResolvedValue({ success: true, result: [] }),
+    deleteConversation: vi.fn().mockResolvedValue({}),
+    clearConversation: vi.fn().mockResolvedValue({}),
+    recordUserAction: vi.fn().mockResolvedValue({}),
+    getTotalCredits: vi.fn().mockResolvedValue({ result: { credits: 0 } }),
+  } as unknown as AbacusClient;
+}
+
+export function _createTestWrapperWithRealAPI(
+  apiClient: AbacusClient,
+): React.FC<{ children: React.ReactElement }> {
+  const queryClient = new QueryClient({
+    defaultOptions: {
+      queries: {
+        retry: false,
+      },
+    },
+  });
+  const mockSecureStorage = {
+    getSecret: vi.fn().mockResolvedValue(null),
+    setSecret: vi.fn().mockResolvedValue(undefined),
+    deleteSecret: vi.fn().mockResolvedValue(undefined),
+  };
+  const mockAuthManager = new AuthManager(mockSecureStorage);
+  return function TestWrapper({ children }: { children: React.ReactElement }) {
+    return React.createElement(
+      ThemeProvider,
+      null,
+      React.createElement(
+        ApiClientProvider,
+        { client: apiClient, authManager: mockAuthManager },
+        React.createElement(
+          QueryClientProvider,
+          { client: queryClient },
+          React.createElement(
+            ShellSizeProvider,
+            null,
+            React.createElement(
+              ShellTitleProvider,
+              null,
+              React.createElement(
+                NotificationProvider,
+                null,
+                React.createElement(
+                  ExtensionFileProvider,
+                  null,
+                  React.createElement(
+                    AgentModeProvider,
+                    null,
+                    React.createElement(
+                      AgentProvider,
+                      {
+                        apiClient,
+                      },
+                      children,
+                    ),
+                  ),
+                ),
+              ),
+            ),
+          ),
+        ),
+      ),
+    );
+  };
+}
+
+describe.concurrent("CLI Conversation Limits E2E Tests", () => {
+  const apiKey = process.env.ABACUS_THROTTLED_API_KEY || process.env.ABACUS_API_KEY;
+  if (!apiKey) {
+    throw new Error("ABACUS_THROTTLED_API_KEY or ABACUS_API_KEY is not set");
+  }
+
+  beforeEach(() => {
+    vi.spyOn(process, "exit").mockImplementation((() => {}) as unknown as typeof process.exit);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("should test ALL available LLMs for upgrade/billing prompts", async () => {
+    console.log(`\n🔑 Using API key: ${apiKey.substring(0, 8)}...`);
+
+    console.log("\n🔑 Using REAL API to test ALL available LLMs...\n");
+
+    // First, fetch all available LLMs
+    const availableLLMs = await fetchAvailableLLMs(apiKey);
+
+    if (availableLLMs.length === 0) {
+      console.error("❌ No LLMs available from API");
+      return;
+    }
+
+    console.log(`\n📋 Testing ${availableLLMs.length} LLMs:\n`);
+    availableLLMs.forEach((llm, i) => {
+      console.log(`   ${i + 1}. ${llm.displayName} (${llm.llmName})`);
+    });
+    console.log("");
+
+    const results: {
+      llmName: string;
+      displayName: string;
+      hasUpgradePrompt: boolean;
+      error?: string;
+    }[] = [];
+
+    for (let i = 0; i < availableLLMs.length; i++) {
+      const llm = availableLLMs[i];
+      console.log(`\n--- Testing LLM ${i + 1}/${availableLLMs.length}: ${llm.displayName} ---`);
+
+      // Create a new conversation for each LLM test
+      const { conversationId, response } = await createConversationWithRealAPI(apiKey);
+
+      if (!response.success) {
+        console.error(`❌ Failed to create conversation for ${llm.displayName}`);
+        results.push({ ...llm, hasUpgradePrompt: false, error: "Failed to create conversation" });
+        continue;
+      }
+
+      console.log(`✅ Created conversation: ${conversationId}`);
+
+      // Send a message with this specific LLM
+      try {
+        const { fullResponse, hasUpgradePrompt } = await sendMessageWithRealAPI(
+          apiKey,
+          conversationId,
+          `Hello, this is a test message using ${llm.displayName}`,
+          llm.llmName,
+        );
+
+        results.push({ ...llm, hasUpgradePrompt });
+
+        if (hasUpgradePrompt) {
+          console.log(`✅ ${llm.displayName} - Shows upgrade prompt as expected`);
+        } else {
+          console.error(`❌ MISSING BILLING/UPGRADE PROMPT for ${llm.displayName}!`);
+          console.error(`   Response: ${fullResponse.substring(0, 500)}`);
+        }
+      } catch (error) {
+        console.error(
+          `❌ Error testing ${llm.displayName}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+        results.push({ ...llm, hasUpgradePrompt: false, error: String(error) });
+      }
+
+      // Small delay between LLM tests
+      await sleep(1000);
+    }
+
+    // Summary
+    console.log("\n\n📊 SUMMARY - LLM Billing/Upgrade Prompt Results:\n");
+    console.log("=".repeat(60));
+
+    const withPrompts = results.filter((r) => r.hasUpgradePrompt);
+    const withoutPrompts = results.filter((r) => !r.hasUpgradePrompt && !r.error);
+    const withErrors = results.filter((r) => r.error);
+
+    console.log("\n✅ LLMs WITH upgrade prompts (GOOD):");
+    withPrompts.forEach((r) => console.log(`   - ${r.displayName} (${r.llmName})`));
+
+    console.log("\n❌ LLMs WITHOUT upgrade prompts (BAD):");
+    withoutPrompts.forEach((r) => console.log(`   - ${r.displayName} (${r.llmName})`));
+
+    if (withErrors.length > 0) {
+      console.log("\n⚠️  LLMs with errors:");
+      withErrors.forEach((r) => console.log(`   - ${r.displayName}: ${r.error}`));
+    }
+
+    console.log("\n" + "=".repeat(60));
+    console.log(`Total: ${results.length} LLMs tested`);
+    console.log(`  ✅ With prompts: ${withPrompts.length}`);
+    console.log(`  ❌ No prompts: ${withoutPrompts.length}`);
+    console.log(`  ⚠️  Errors: ${withErrors.length}`);
+
+    // Test passes if:
+    // 1. At least one LLM shows upgrade prompts (validates billing detection works)
+    // 2. OR all LLMs responded without errors (no LLMs available to test billing)
+    // Some LLMs may be free or have different billing rules, so we don't require ALL to show prompts
+    if (results.length > 0) {
+      const hasAnySuccessfulResponse = withPrompts.length > 0 || withoutPrompts.length > 0;
+      expect(hasAnySuccessfulResponse).toBe(true);
+
+      // Log warning if some LLMs don't show upgrade prompts (for investigation)
+      if (withoutPrompts.length > 0) {
+        console.log(`\n⚠️  WARNING: ${withoutPrompts.length} LLM(s) did not show upgrade prompts.`);
+        console.log(
+          "   This may be expected if these LLMs are free or have different billing rules.",
+        );
+        console.log("   LLMs without prompts:", withoutPrompts.map((r) => r.llmName).join(", "));
+      }
+    }
+
+    console.log(`\n✅ Test completed - billing/upgrade prompt detection validated\n`);
+  }, 600000); // 10 minute timeout for testing all LLMs
+});

package/src/__e2e__/conversation.e2e.test.tsx

@@ -0,0 +1,56 @@
+import React from "react";
+import { describe, it, expect } from "vitest";
+
+import { App } from "../entrypoints/repl.js";
+import { Timing } from "../lib/timing.js";
+import {
+  renderWithProviders,
+  simulateText,
+  simulateEnter,
+  waitForFrame,
+} from "./helpers/test-helpers.js";
+
+describe.concurrent("Agent Conversation E2E Tests", () => {
+  it("should render App component", async () => {
+    const timing = new Timing();
+
+    const { lastFrame, unmount } = renderWithProviders(React.createElement(App, { timing }));
+
+    await waitForFrame(lastFrame, (frame) => frame.length > 0, 3000);
+
+    const frame = lastFrame();
+    expect(frame).toBeDefined();
+
+    unmount();
+  });
+
+  it("should handle empty input in composer", async () => {
+    const timing = new Timing();
+
+    const { lastFrame, stdin, unmount } = renderWithProviders(React.createElement(App, { timing }));
+
+    await waitForFrame(lastFrame, (frame) => frame.length > 0, 3000);
+
+    simulateEnter(stdin);
+
+    const frame = lastFrame();
+    expect(frame).toBeDefined();
+
+    unmount();
+  });
+
+  it("should accept text input", async () => {
+    const timing = new Timing();
+
+    const { lastFrame, stdin, unmount } = renderWithProviders(React.createElement(App, { timing }));
+
+    await waitForFrame(lastFrame, (frame) => frame.length > 0, 3000);
+
+    simulateText(stdin, "test message");
+
+    const frame = lastFrame();
+    expect(frame).toBeDefined();
+
+    unmount();
+  });
+});