apteva 0.4.11 → 0.4.14

package/src/mcp-platform.ts

@@ -2,6 +2,8 @@
 // This allows the meta agent (Apteva Assistant) to control the platform
 
 import { AgentDB, ProjectDB, McpServerDB, SkillDB, TelemetryDB, generateId } from "./db";
+import { TestCaseDB, TestRunDB } from "./db-tests";
+import { runTest, runAll } from "./test-runner";
 import { getProvidersWithStatus, PROVIDERS } from "./providers";
 import { startAgentProcess, setAgentStatus, toApiAgent, META_AGENT_ID, agentFetch } from "./routes/api/agent-utils";
 import { agentProcesses } from "./server";
@@ -48,24 +50,47 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "create_agent",
-    description: "Create a new AI agent. Requires a name, provider, and model. The provider must have an API key configured.",
+    description: `Create a new AI agent. The provider must have an API key configured — use list_providers first to check.
+
+PROVIDERS & MODELS (use list_providers to see which have keys):
+- anthropic: claude-sonnet-4-5 (recommended), claude-haiku-4-5 (fast/cheap)
+- openai: gpt-4o (recommended), gpt-4o-mini (fast/cheap)
+- groq: llama-3.3-70b-versatile (recommended), llama-3.1-8b-instant (fast)
+- gemini: gemini-3-pro-preview (recommended), gemini-3-flash-preview (fast)
+- xai: grok-2 (recommended), grok-2-mini (fast)
+- together: moonshotai/Kimi-K2.5 (recommended), moonshotai/Kimi-K2-Thinking (reasoning)
+- fireworks: accounts/fireworks/models/kimi-k2p5, accounts/fireworks/models/kimi-k2-thinking
+- moonshot: moonshot-v1-128k (recommended), moonshot-v1-32k (fast)
+- ollama: llama3.3, llama3.2, qwen2.5, mistral, deepseek-r1 (local, no API key needed)
+
+FEATURES (all optional, default false):
+- memory: Persistent memory across conversations — agent remembers past interactions. Requires OpenAI key for embeddings.
+- tasks: Task scheduling — agent can create, schedule, and track tasks. Supports recurring tasks.
+- vision: Image & PDF understanding — agent can analyze uploaded images and PDFs.
+- mcp: MCP tool use — agent can use tools from assigned MCP servers. Enable this if you plan to assign MCP servers.
+- files: File management — agent can read, write, and manage files in its workspace.
+
+TIPS:
+- Always provide a descriptive system_prompt that tells the agent what it does and how to behave.
+- Assign to a project_id to organize agents. Use list_projects to see available projects.
+- After creating, use start_agent to run it. Then assign MCP servers or skills as needed.`,
     inputSchema: {
       type: "object",
       properties: {
-        name: { type: "string", description: "Agent name" },
-        provider: { type: "string", description: "LLM provider ID (e.g. anthropic, openai, groq, gemini, xai, together, fireworks, ollama)" },
-        model: { type: "string", description: "Model ID (e.g. claude-sonnet-4-5, gpt-4o, llama-3.3-70b-versatile)" },
-        system_prompt: { type: "string", description: "System prompt for the agent (optional)" },
-        project_id: { type: "string", description: "Project ID to assign the agent to (optional)" },
+        name: { type: "string", description: "Agent name (e.g. 'Customer Support', 'Code Reviewer')" },
+        provider: { type: "string", description: "LLM provider ID: anthropic, openai, groq, gemini, xai, together, fireworks, moonshot, ollama" },
+        model: { type: "string", description: "Model ID — see tool description for full list per provider" },
+        system_prompt: { type: "string", description: "Instructions for the agent. Describe its role, personality, and capabilities. This is the most important field for agent behavior." },
+        project_id: { type: "string", description: "Project ID to assign the agent to (optional). Use list_projects to find IDs." },
         features: {
           type: "object",
-          description: "Feature flags (optional). All default to false.",
+          description: "Feature flags to enable. All default to false. See tool description for details on each feature.",
           properties: {
-            memory: { type: "boolean" },
-            tasks: { type: "boolean" },
-            vision: { type: "boolean" },
-            mcp: { type: "boolean" },
-            files: { type: "boolean" },
+            memory: { type: "boolean", description: "Persistent memory across conversations (requires OpenAI key for embeddings)" },
+            tasks: { type: "boolean", description: "Task scheduling and tracking" },
+            vision: { type: "boolean", description: "Image and PDF understanding" },
+            mcp: { type: "boolean", description: "MCP tool use — required if assigning MCP servers" },
+            files: { type: "boolean", description: "File read/write in agent workspace" },
           },
         },
       },
@@ -74,17 +99,27 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "update_agent",
-    description: "Update an existing agent's configuration. Only provide fields you want to change.",
+    description: "Update an existing agent's configuration. Only provide fields you want to change. If the agent is running, restart it after updating for changes to take effect.",
     inputSchema: {
       type: "object",
       properties: {
         agent_id: { type: "string", description: "The agent ID to update" },
-        name: { type: "string", description: "New name" },
-        model: { type: "string", description: "New model ID" },
-        provider: { type: "string", description: "New provider ID" },
-        system_prompt: { type: "string", description: "New system prompt" },
-        project_id: { type: "string", description: "New project ID (or null to unassign)" },
-        features: { type: "object", description: "Feature flags to update" },
+        name: { type: "string", description: "New display name" },
+        model: { type: "string", description: "New model ID (see create_agent for available models per provider)" },
+        provider: { type: "string", description: "New provider ID (the new provider must have an API key configured)" },
+        system_prompt: { type: "string", description: "New system prompt / instructions" },
+        project_id: { type: "string", description: "New project ID, or null to unassign from project" },
+        features: {
+          type: "object",
+          description: "Feature flags to update (only provided flags are changed, others remain as-is)",
+          properties: {
+            memory: { type: "boolean" },
+            tasks: { type: "boolean" },
+            vision: { type: "boolean" },
+            mcp: { type: "boolean" },
+            files: { type: "boolean" },
+          },
+        },
       },
       required: ["agent_id"],
     },
@@ -102,7 +137,7 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "start_agent",
-    description: "Start a stopped agent. The agent's provider must have an API key configured.",
+    description: "Start a stopped agent. The agent's provider must have an API key configured. Starting spawns a process, waits for health check, and pushes configuration (model, features, MCP servers, skills). Takes a few seconds.",
     inputSchema: {
       type: "object",
       properties: {
@@ -174,18 +209,26 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "create_mcp_server",
-    description: "Create a new MCP server. For HTTP (remote) servers, provide url and optional headers. For npm package servers, provide a package name.",
+    description: `Create a new MCP server configuration. MCP servers provide tools that agents can use (web search, file access, APIs, etc).
+
+SERVER TYPES:
+- http: Remote MCP server accessible via URL. Provide url and optional auth headers. Ready to use immediately.
+- npm: Node.js MCP server from npm. Provide package name (e.g. '@modelcontextprotocol/server-filesystem'). Needs to be started.
+- pip: Python MCP server from PyPI. Provide package name. Needs to be started.
+- custom: Custom command. Provide command and args. Needs to be started.
+
+After creating, assign to agents with assign_mcp_server_to_agent. HTTP servers work immediately; npm/pip/custom servers need to be started from the MCP page in the UI.`,
     inputSchema: {
       type: "object",
       properties: {
-        name: { type: "string", description: "Server display name" },
-        type: { type: "string", description: "Server type: 'http' (remote URL), 'npm' (npm package), 'pip' (Python package), 'custom' (custom command)" },
-        url: { type: "string", description: "For http type: the remote MCP server URL" },
-        headers: { type: "object", description: "For http type: auth headers (e.g. {\"Authorization\": \"Bearer ...\"})" },
-        package: { type: "string", description: "For npm/pip type: the package name (e.g. '@modelcontextprotocol/server-filesystem')" },
-        command: { type: "string", description: "For custom type: the command to run" },
-        args: { type: "string", description: "Command arguments (optional)" },
-        project_id: { type: "string", description: "Project ID to scope the server to (optional, null = global)" },
+        name: { type: "string", description: "Display name (e.g. 'Filesystem', 'Web Search', 'GitHub')" },
+        type: { type: "string", description: "Server type: http, npm, pip, or custom" },
+        url: { type: "string", description: "For http type: the remote MCP server URL (e.g. 'https://mcp.example.com/sse')" },
+        headers: { type: "object", description: "For http type: auth headers as key-value pairs" },
+        package: { type: "string", description: "For npm/pip type: package name" },
+        command: { type: "string", description: "For custom type: executable command" },
+        args: { type: "string", description: "Command arguments string (optional)" },
+        project_id: { type: "string", description: "Scope to a project (optional). null = available globally to all agents." },
       },
       required: ["name", "type"],
     },
@@ -203,7 +246,7 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "assign_mcp_server_to_agent",
-    description: "Assign an MCP server to an agent so the agent can use its tools. The agent must have MCP feature enabled.",
+    description: "Assign an MCP server to an agent so the agent can use its tools. This automatically enables the MCP feature on the agent. If the agent is running, restart it for changes to take effect.",
     inputSchema: {
       type: "object",
       properties: {
@@ -248,7 +291,7 @@ const PLATFORM_TOOLS = [
   // Skills management
   {
     name: "list_skills",
-    description: "List all installed skills. Skills are reusable instruction sets that give agents specialized capabilities.",
+    description: "List all installed skills. Skills are reusable instruction sets (like prompt templates with tool permissions) that give agents specialized capabilities. Skills can be installed from the SkillsMP marketplace or created locally.",
     inputSchema: {
       type: "object",
       properties: {
@@ -281,7 +324,7 @@ const PLATFORM_TOOLS = [
   },
   {
     name: "assign_skill_to_agent",
-    description: "Assign a skill to an agent so it can use those instructions.",
+    description: "Assign a skill to an agent. The skill's instructions and tool permissions will be pushed to the agent on next start/restart.",
     inputSchema: {
       type: "object",
       properties: {
@@ -314,6 +357,77 @@ const PLATFORM_TOOLS = [
       required: ["skill_id"],
     },
   },
+  // Test tools
+  {
+    name: "list_tests",
+    description: "List all test cases. Tests validate agent workflows by sending a message and using an LLM judge to evaluate the result.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        project_id: { type: "string", description: "Optional project ID to filter tests" },
+      },
+    },
+  },
+  {
+    name: "create_test",
+    description: "Create a new test case for an agent. The test sends a message to the agent, then an LLM judge evaluates the conversation against the success criteria.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "Test name" },
+        agent_id: { type: "string", description: "Agent ID to test" },
+        input_message: { type: "string", description: "Message to send to the agent" },
+        eval_criteria: { type: "string", description: "Natural language success criteria for the LLM judge. E.g. 'The agent should use the post_tweet tool and confirm the post was made.'" },
+        description: { type: "string", description: "Optional description" },
+        timeout_ms: { type: "number", description: "Timeout in ms (default 60000)" },
+      },
+      required: ["name", "agent_id", "input_message", "eval_criteria"],
+    },
+  },
+  {
+    name: "run_test",
+    description: "Run a test case. The agent must be running. Returns pass/fail with LLM judge reasoning.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        test_id: { type: "string", description: "Test case ID to run. Use list_tests to find IDs." },
+      },
+      required: ["test_id"],
+    },
+  },
+  {
+    name: "run_all_tests",
+    description: "Run all test cases (or specific ones). Returns summary of pass/fail results.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        test_case_ids: { type: "array", items: { type: "string" }, description: "Optional array of test case IDs. If empty, runs all tests." },
+      },
+    },
+  },
+  {
+    name: "get_test_results",
+    description: "Get run history for a test case. Shows pass/fail status, judge reasoning, and duration.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        test_id: { type: "string", description: "Test case ID" },
+        limit: { type: "number", description: "Max results to return (default 10)" },
+      },
+      required: ["test_id"],
+    },
+  },
+  {
+    name: "delete_test",
+    description: "Delete a test case and all its run history.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        test_id: { type: "string", description: "Test case ID to delete" },
+      },
+      required: ["test_id"],
+    },
+  },
 ];
 
 // Tool execution handlers
@@ -725,6 +839,91 @@ async function executeTool(name: string, args: Record<string, any>): Promise<{ c
         return { content: [{ type: "text", text: `Skill "${skill.name}" deleted${agentsWithSkill.length > 0 ? ` (unassigned from ${agentsWithSkill.length} agent(s))` : ""}` }] };
       }
 
+      // Test tools
+      case "list_tests": {
+        const tests = TestCaseDB.findAll(args.project_id);
+        const result = tests.map(tc => {
+          const agent = AgentDB.findById(tc.agent_id);
+          const lastRun = TestRunDB.getLatestByTestCase(tc.id);
+          return {
+            id: tc.id,
+            name: tc.name,
+            agent_id: tc.agent_id,
+            agent_name: agent?.name || "Unknown",
+            input_message: tc.input_message,
+            eval_criteria: tc.eval_criteria,
+            timeout_ms: tc.timeout_ms,
+            last_status: lastRun?.status || null,
+            last_reasoning: lastRun?.judge_reasoning || null,
+          };
+        });
+        return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+      }
+
+      case "create_test": {
+        const agent = AgentDB.findById(args.agent_id);
+        if (!agent) {
+          return { content: [{ type: "text", text: `Agent not found: ${args.agent_id}` }], isError: true };
+        }
+        const tc = TestCaseDB.create({
+          name: args.name,
+          agent_id: args.agent_id,
+          input_message: args.input_message,
+          eval_criteria: args.eval_criteria,
+          description: args.description,
+          timeout_ms: args.timeout_ms,
+        });
+        return { content: [{ type: "text", text: `Test "${tc.name}" created (id: ${tc.id}) for agent "${agent.name}". Use run_test to execute it.` }] };
+      }
+
+      case "run_test": {
+        const tc = TestCaseDB.findById(args.test_id);
+        if (!tc) {
+          return { content: [{ type: "text", text: `Test not found: ${args.test_id}` }], isError: true };
+        }
+        const result = await runTest(tc);
+        const agent = AgentDB.findById(tc.agent_id);
+        return { content: [{ type: "text", text: `Test "${tc.name}" (agent: ${agent?.name || tc.agent_id}): ${result.status.toUpperCase()}${result.duration_ms ? ` in ${(result.duration_ms / 1000).toFixed(1)}s` : ""}\n\nJudge: ${result.judge_reasoning || result.error || "No reasoning"}` }] };
+      }
+
+      case "run_all_tests": {
+        const results = await runAll(args.test_case_ids);
+        const passed = results.filter(r => r.status === "passed").length;
+        const failed = results.filter(r => r.status === "failed").length;
+        const errors = results.filter(r => r.status === "error").length;
+        const lines = results.map(r => {
+          const tc = TestCaseDB.findById(r.test_case_id);
+          return `- ${tc?.name || r.test_case_id}: ${r.status.toUpperCase()}${r.judge_reasoning ? ` — ${r.judge_reasoning}` : ""}${r.error ? ` — Error: ${r.error}` : ""}`;
+        });
+        return { content: [{ type: "text", text: `Test Results: ${passed} passed, ${failed} failed, ${errors} errors (${results.length} total)\n\n${lines.join("\n")}` }] };
+      }
+
+      case "get_test_results": {
+        const tc = TestCaseDB.findById(args.test_id);
+        if (!tc) {
+          return { content: [{ type: "text", text: `Test not found: ${args.test_id}` }], isError: true };
+        }
+        const runs = TestRunDB.findByTestCase(args.test_id, args.limit || 10);
+        const result = runs.map(r => ({
+          id: r.id,
+          status: r.status,
+          duration_ms: r.duration_ms,
+          judge_reasoning: r.judge_reasoning,
+          error: r.error,
+          created_at: r.created_at,
+        }));
+        return { content: [{ type: "text", text: `Run history for "${tc.name}":\n${JSON.stringify(result, null, 2)}` }] };
+      }
+
+      case "delete_test": {
+        const tc = TestCaseDB.findById(args.test_id);
+        if (!tc) {
+          return { content: [{ type: "text", text: `Test not found: ${args.test_id}` }], isError: true };
+        }
+        TestCaseDB.delete(args.test_id);
+        return { content: [{ type: "text", text: `Test "${tc.name}" deleted.` }] };
+      }
+
       default:
         return { content: [{ type: "text", text: `Unknown tool: ${name}` }], isError: true };
     }
@@ -772,7 +971,19 @@ export async function handlePlatformMcpRequest(req: Request): Promise<Response>
           name: "apteva-platform",
           version: "1.0.0",
         },
-        instructions: "This MCP server provides tools to control the Apteva AI agent platform. You can create, start, stop, and manage agents, projects, and view system status.",
+        instructions: `This MCP server controls the Apteva AI agent management platform.
+
+You can manage:
+- AGENTS: Create, configure, start, stop, and delete AI agents. Each agent has a provider (LLM), model, system prompt, and optional features (memory, tasks, vision, MCP tools, files).
+- PROJECTS: Organize agents into projects for grouping.
+- MCP SERVERS: Tool integrations that give agents capabilities (web search, file access, APIs). Assign servers to agents.
+- SKILLS: Reusable instruction sets that specialize agent behavior. Assign skills to agents.
+- PROVIDERS: View which LLM providers have API keys configured.
+- TESTS: Create and run automated tests for agent workflows. Tests send a message to an agent, then an LLM judge evaluates the response against success criteria. Use list_tests, create_test, run_test, run_all_tests, get_test_results, delete_test.
+
+Typical workflow: list_providers → create_agent → assign MCP servers/skills → start_agent.
+Test workflow: create_test (set agent, message, eval criteria) → run_test → check results.
+Always use list_providers first to check which providers have API keys before creating agents.`,
       };
       break;
     }