apteva 0.4.31 → 0.4.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apteva",
-  "version": "0.4.31",
+  "version": "0.4.41",
   "description": "Run AI agents locally. Multi-provider support for Claude, GPT, Gemini, Llama, and more.",
   "type": "module",
   "main": "./dist/index.js",
@@ -72,14 +72,15 @@
     "ink-spinner": "^5.0.0",
     "ink-text-input": "^6.0.0",
     "react": "19",
-    "react-dom": "19"
+    "react-dom": "19",
+    "recharts": "^3.7.0"
   },
   "optionalDependencies": {
-    "@apteva/apteva-darwin-arm64": "0.4.31",
-    "@apteva/apteva-darwin-x64": "0.4.31",
-    "@apteva/apteva-linux-arm64": "0.4.31",
-    "@apteva/apteva-linux-x64": "0.4.31",
-    "@apteva/apteva-win32-x64": "0.4.31",
+    "@apteva/apteva-darwin-arm64": "0.4.33",
+    "@apteva/apteva-darwin-x64": "0.4.33",
+    "@apteva/apteva-linux-arm64": "0.4.33",
+    "@apteva/apteva-linux-x64": "0.4.33",
+    "@apteva/apteva-win32-x64": "0.4.33",
     "@apteva/agent-darwin-arm64": "^1.33.73",
     "@apteva/agent-darwin-x64": "^1.33.73",
     "@apteva/agent-linux-arm64": "^1.33.73",

package/src/crypto.ts

@@ -136,12 +136,13 @@ export function validateKeyFormat(provider: string, key: string): { valid: boole
     return { valid: false, error: "API key cannot be empty" };
   }
 
-  // Ollama and local browser providers use base URL instead of API key
-  if (provider === "ollama" || provider === "browserengine" || provider === "chrome") {
+  // Ollama and CDP use URLs instead of API keys
+  if (provider === "ollama" || provider === "cdp") {
     if (trimmed.startsWith("http://") || trimmed.startsWith("https://") || trimmed.startsWith("ws://") || trimmed.startsWith("wss://")) {
       return { valid: true };
     }
-    return { valid: false, error: `${provider} requires a valid URL (e.g., http://localhost:8098)` };
+    const example = provider === "cdp" ? "ws://localhost:9222" : "http://localhost:11434";
+    return { valid: false, error: `${provider} requires a valid URL (e.g., ${example})` };
   }
 
   // Multi-field providers store JSON objects — validate the inner api_key

package/src/db.ts

@@ -20,7 +20,7 @@ export interface AgentBuiltinTools {
 
 export interface OperatorConfig {
   enabled: boolean;
-  browser_provider?: string; // "browserbase" | "steel" | "browserengine" | "chrome"
+  browser_provider?: string; // "browserengine" | "browserbase" | "steel" | "cdp"
   display_width?: number;
   display_height?: number;
   max_actions_per_turn?: number;
@@ -328,6 +328,12 @@ export function initDatabase(dataDir: string): Database {
   db.run("PRAGMA busy_timeout = 5000");
   db.run("PRAGMA foreign_keys = ON");
 
+  // Performance PRAGMAs
+  db.run("PRAGMA synchronous = NORMAL"); // Safe with WAL, much faster than FULL
+  db.run("PRAGMA cache_size = -20000"); // 20MB page cache (negative = KB)
+  db.run("PRAGMA mmap_size = 30000000"); // 30MB memory-mapped I/O
+  db.run("PRAGMA temp_store = MEMORY"); // Keep temp tables in memory
+
   // Run migrations
   runMigrations();
 
@@ -486,6 +492,7 @@ function runMigrations() {
         );
         CREATE INDEX IF NOT EXISTS idx_telemetry_agent ON telemetry_events(agent_id);
         CREATE INDEX IF NOT EXISTS idx_telemetry_time ON telemetry_events(timestamp);
+        CREATE INDEX IF NOT EXISTS idx_telemetry_agent_time ON telemetry_events(agent_id, timestamp DESC);
         CREATE INDEX IF NOT EXISTS idx_telemetry_category ON telemetry_events(category);
         CREATE INDEX IF NOT EXISTS idx_telemetry_level ON telemetry_events(level);
         CREATE INDEX IF NOT EXISTS idx_telemetry_trace ON telemetry_events(trace_id);
@@ -835,6 +842,31 @@ function runMigrations() {
         CREATE UNIQUE INDEX IF NOT EXISTS idx_provider_keys_unique ON provider_keys(provider_id, COALESCE(project_id, ''));
       `,
     },
+    {
+      name: "034_repair_provider_keys_project_support",
+      sql: `
+        -- Repair: migrations 022 and 029 may have failed but were marked as applied.
+        -- Recreate provider_keys with project_id support from whatever current state.
+        CREATE TABLE IF NOT EXISTS provider_keys_repair (
+          id TEXT PRIMARY KEY,
+          provider_id TEXT NOT NULL,
+          encrypted_key TEXT NOT NULL,
+          key_hint TEXT,
+          is_valid INTEGER DEFAULT 1,
+          last_tested_at TEXT,
+          created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+          project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
+          name TEXT
+        );
+        INSERT OR IGNORE INTO provider_keys_repair (id, provider_id, encrypted_key, key_hint, is_valid, last_tested_at, created_at)
+          SELECT id, provider_id, encrypted_key, key_hint, is_valid, last_tested_at, created_at FROM provider_keys;
+        DROP TABLE IF EXISTS provider_keys;
+        ALTER TABLE provider_keys_repair RENAME TO provider_keys;
+        CREATE INDEX IF NOT EXISTS idx_provider_keys_provider ON provider_keys(provider_id);
+        CREATE INDEX IF NOT EXISTS idx_provider_keys_project ON provider_keys(project_id);
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_provider_keys_unique ON provider_keys(provider_id, COALESCE(project_id, ''));
+      `,
+    },
   ];
 
   // Check which migrations have been applied
@@ -1056,10 +1088,10 @@ export const AgentDB = {
 
   // Find agents that have a specific skill
   findBySkill(skillId: string): Agent[] {
-    // SQLite JSON query: check if skills array contains the skillId
+    // Use json_each to properly search the JSON array (avoids full table scan with LIKE)
     const rows = db.query(
-      `SELECT * FROM agents WHERE skills LIKE ? ORDER BY created_at DESC`
-    ).all(`%"${skillId}"%`) as AgentRow[];
+      `SELECT DISTINCT a.* FROM agents a, json_each(a.skills) AS s WHERE s.value = ? ORDER BY a.created_at DESC`
+    ).all(skillId) as AgentRow[];
     return rows.map(rowToAgent);
   },
 
@@ -1091,14 +1123,23 @@ export const AgentDB = {
     return row.count;
   },
 
-  // Get decrypted API key for an agent
+  // In-memory cache for decrypted API keys (avoids expensive scryptSync on every request)
+  _apiKeyCache: new Map<string, string>(),
+
+  // Get decrypted API key for an agent (cached)
   getApiKey(id: string): string | null {
+    // Check cache first
+    const cached = this._apiKeyCache.get(id);
+    if (cached) return cached;
+
     const agent = this.findById(id);
     if (!agent || !agent.api_key_encrypted) {
       return null;
     }
     try {
-      return decrypt(agent.api_key_encrypted);
+      const key = decrypt(agent.api_key_encrypted);
+      if (key) this._apiKeyCache.set(id, key);
+      return key;
     } catch {
       return null;
     }
@@ -1118,6 +1159,8 @@ export const AgentDB = {
       [encrypted, now, id]
     );
 
+    // Update cache
+    this._apiKeyCache.set(id, newApiKey);
     return newApiKey;
   },
 
@@ -1129,7 +1172,9 @@ export const AgentDB = {
     // If agent already has a key, return it
     if (agent.api_key_encrypted) {
       try {
-        return decrypt(agent.api_key_encrypted);
+        const key = decrypt(agent.api_key_encrypted);
+        if (key) this._apiKeyCache.set(id, key);
+        return key;
       } catch {
         // Key is corrupted, regenerate
       }
@@ -1395,7 +1440,7 @@ export const ProviderKeysDB = {
 
   // Find all keys for a provider (global + all projects)
   findAllByProvider(providerId: string): ProviderKey[] {
-    const rows = db.query("SELECT * FROM provider_keys WHERE provider_id = ? ORDER BY project_id NULLS FIRST, created_at DESC").all(providerId) as ProviderKeyRow[];
+    const rows = db.query("SELECT * FROM provider_keys WHERE provider_id = ? ORDER BY (project_id IS NOT NULL), created_at DESC").all(providerId) as ProviderKeyRow[];
     return rows.map(rowToProviderKey);
   },
 
@@ -1407,7 +1452,7 @@ export const ProviderKeysDB = {
 
   // Get all provider keys
   findAll(): ProviderKey[] {
-    const rows = db.query("SELECT * FROM provider_keys ORDER BY provider_id, project_id NULLS FIRST, created_at DESC").all() as ProviderKeyRow[];
+    const rows = db.query("SELECT * FROM provider_keys ORDER BY provider_id, (project_id IS NOT NULL), created_at DESC").all() as ProviderKeyRow[];
     return rows.map(rowToProviderKey);
   },
 
@@ -1535,6 +1580,22 @@ export const McpServerDB = {
     return rows.map(rowToMcpServer);
   },
 
+  // Light version: skips expensive decryption for listing endpoints
+  findAllLight(): McpServer[] {
+    const rows = db.query("SELECT * FROM mcp_servers ORDER BY created_at DESC").all() as McpServerRow[];
+    return rows.map(rowToMcpServerLight);
+  },
+
+  // Light batch load by IDs: skips decryption (used by toApiAgentsBatch)
+  findByIdsLight(ids: string[]): Map<string, McpServer> {
+    if (ids.length === 0) return new Map();
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = db.query(`SELECT * FROM mcp_servers WHERE id IN (${placeholders})`).all(...ids) as McpServerRow[];
+    const map = new Map<string, McpServer>();
+    for (const row of rows) map.set(row.id, rowToMcpServerLight(row));
+    return map;
+  },
+
   findRunning(): McpServer[] {
     const rows = db.query("SELECT * FROM mcp_servers WHERE status = 'running'").all() as McpServerRow[];
     return rows.map(rowToMcpServer);
@@ -1661,6 +1722,30 @@ export const McpServerDB = {
     const rows = db.query("SELECT * FROM mcp_servers WHERE project_id IS NULL ORDER BY created_at DESC").all() as McpServerRow[];
     return rows.map(rowToMcpServer);
   },
+
+  // Light versions (skip decryption) for listing endpoints
+  findByProjectLight(projectId: string | null): McpServer[] {
+    if (projectId === null) {
+      const rows = db.query("SELECT * FROM mcp_servers WHERE project_id IS NULL ORDER BY created_at DESC").all() as McpServerRow[];
+      return rows.map(rowToMcpServerLight);
+    }
+    const rows = db.query("SELECT * FROM mcp_servers WHERE project_id = ? ORDER BY created_at DESC").all(projectId) as McpServerRow[];
+    return rows.map(rowToMcpServerLight);
+  },
+
+  findForAgentLight(agentProjectId: string | null): McpServer[] {
+    if (agentProjectId === null) {
+      const rows = db.query("SELECT * FROM mcp_servers WHERE project_id IS NULL ORDER BY created_at DESC").all() as McpServerRow[];
+      return rows.map(rowToMcpServerLight);
+    }
+    const rows = db.query("SELECT * FROM mcp_servers WHERE project_id IS NULL OR project_id = ? ORDER BY created_at DESC").all(agentProjectId) as McpServerRow[];
+    return rows.map(rowToMcpServerLight);
+  },
+
+  findGlobalLight(): McpServer[] {
+    const rows = db.query("SELECT * FROM mcp_servers WHERE project_id IS NULL ORDER BY created_at DESC").all() as McpServerRow[];
+    return rows.map(rowToMcpServerLight);
+  },
 };
 
 // MCP Server Tool CRUD operations (for local servers)
@@ -1792,6 +1877,27 @@ function rowToMcpServer(row: McpServerRow): McpServer {
   };
 }
 
+// Light version: skips expensive decryption of env/headers for listing endpoints
+function rowToMcpServerLight(row: McpServerRow): McpServer {
+  return {
+    id: row.id,
+    name: row.name,
+    type: row.type as McpServer["type"],
+    package: row.package,
+    pip_module: row.pip_module,
+    command: row.command,
+    args: row.args,
+    env: {},
+    url: row.url,
+    headers: {},
+    port: row.port,
+    status: row.status as "stopped" | "running",
+    source: row.source,
+    project_id: row.project_id,
+    created_at: row.created_at,
+  };
+}
+
 // Telemetry Event types
 // User types
 export interface User {
@@ -1892,26 +1998,30 @@ export const TelemetryDB = {
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
 
+    // Wrap in transaction for massive speedup (single fsync instead of one per row)
     let inserted = 0;
-    for (const event of events) {
-      const result = stmt.run(
-        event.id,
-        agentId,
-        event.timestamp,
-        event.category,
-        event.type,
-        event.level,
-        event.trace_id || null,
-        event.span_id || null,
-        event.thread_id || null,
-        event.data ? JSON.stringify(event.data) : null,
-        event.metadata ? JSON.stringify(event.metadata) : null,
-        event.duration_ms || null,
-        event.error || null,
-        now
-      );
-      if (result.changes > 0) inserted++;
-    }
+    const insertAll = db.transaction(() => {
+      for (const event of events) {
+        const result = stmt.run(
+          event.id,
+          agentId,
+          event.timestamp,
+          event.category,
+          event.type,
+          event.level,
+          event.trace_id || null,
+          event.span_id || null,
+          event.thread_id || null,
+          event.data ? JSON.stringify(event.data) : null,
+          event.metadata ? JSON.stringify(event.metadata) : null,
+          event.duration_ms || null,
+          event.error || null,
+          now
+        );
+        if (result.changes > 0) inserted++;
+      }
+    });
+    insertAll();
     return inserted;
   },
 
@@ -2810,6 +2920,21 @@ export const SubscriptionDB = {
     return rows.map(rowToSubscription);
   },
 
+  // Batch load subscriptions for multiple agents (1 query instead of N)
+  findByAgentIds(agentIds: string[]): Map<string, Subscription[]> {
+    const result = new Map<string, Subscription[]>();
+    if (agentIds.length === 0) return result;
+    const placeholders = agentIds.map(() => "?").join(",");
+    const rows = db.query(`SELECT * FROM subscriptions WHERE agent_id IN (${placeholders})`).all(...agentIds) as SubscriptionRow[];
+    for (const row of rows) {
+      const sub = rowToSubscription(row);
+      const list = result.get(sub.agent_id) || [];
+      list.push(sub);
+      result.set(sub.agent_id, list);
+    }
+    return result;
+  },
+
   findAll(projectId?: string | null): Subscription[] {
     if (projectId) {
       const rows = db.query("SELECT * FROM subscriptions WHERE project_id = ? ORDER BY created_at DESC").all(projectId) as SubscriptionRow[];

package/src/integrations/agentdojo.ts

@@ -56,18 +56,30 @@ export const AgentDojoProvider: IntegrationProvider = {
       console.error("AgentDojo listApps providers error:", providersRes.status);
     }
 
-    // Index providers by name for quick lookup
+    // Index providers by id and name for quick lookup
+    const providerById = new Map<number, any>();
     const providerByName = new Map<string, any>();
     for (const p of providers) {
+      if (p.id) providerById.set(p.id, p);
       providerByName.set(p.name, p);
       if (p.display_name) providerByName.set(p.display_name.toLowerCase(), p);
     }
 
-    // Map toolkits to apps, enriching auth info from providers
+    // Map toolkits to apps, enriching auth info from providers.
+    // Use auth_provider_id from the toolkit to find the parent provider —
+    // one provider can serve many toolkits (e.g. provider "omnikit" id=49
+    // serves "omnikit-messaging", "omnikit-cms", "omnikit-billing", etc.)
     const apps: IntegrationApp[] = toolkits.map((toolkit: any) => {
       const name = toolkit.name || toolkit.slug;
-      // Try to find matching provider for this toolkit
-      const provider = providerByName.get(name) || providerByName.get(name?.toLowerCase());
+
+      // Primary: match via auth_provider_id (reliable link from API)
+      let provider = toolkit.auth_provider_id
+        ? providerById.get(toolkit.auth_provider_id)
+        : null;
+      // Fallback: match by name
+      if (!provider) {
+        provider = providerByName.get(name) || providerByName.get(name?.toLowerCase());
+      }
 
       let authSchemes: string[];
       if (provider) {
@@ -78,6 +90,19 @@ export const AgentDojoProvider: IntegrationProvider = {
         authSchemes = ["NONE"];
       }
 
+      // Build credential fields from provider auth_config
+      let credentialFields: IntegrationApp["credentialFields"] | undefined;
+      if (provider?.auth_config?.required_fields) {
+        const descriptions = provider.auth_config.field_descriptions || {};
+        const required = new Set(provider.auth_config.required_fields || []);
+        const allFields = [...(provider.auth_config.required_fields || []), ...(provider.auth_config.optional_fields || [])];
+        credentialFields = allFields.map((f: string) => ({
+          name: f,
+          description: descriptions[f] || undefined,
+          required: required.has(f),
+        }));
+      }
+
       return {
         id: String(toolkit.id),
         name: toolkit.display_name || toolkit.name,
@@ -86,13 +111,27 @@ export const AgentDojoProvider: IntegrationProvider = {
         logo: provider?.favicon || toolkit.icon_url || null,
         categories: [],
         authSchemes,
+        providerSlug: provider?.name || undefined,
+        credentialFields,
       };
     });
 
     // Also add any providers that don't match a toolkit (standalone OAuth providers)
     const toolkitNames = new Set(toolkits.map((t: any) => t.name));
+    const toolkitProviderIds = new Set(toolkits.map((t: any) => t.auth_provider_id).filter(Boolean));
     for (const p of providers) {
-      if (!toolkitNames.has(p.name)) {
+      if (!toolkitNames.has(p.name) && !toolkitProviderIds.has(p.id)) {
+        let credentialFields: IntegrationApp["credentialFields"] | undefined;
+        if (p.auth_config?.required_fields) {
+          const descriptions = p.auth_config.field_descriptions || {};
+          const required = new Set(p.auth_config.required_fields || []);
+          const allFields = [...(p.auth_config.required_fields || []), ...(p.auth_config.optional_fields || [])];
+          credentialFields = allFields.map((f: string) => ({
+            name: f,
+            description: descriptions[f] || undefined,
+            required: required.has(f),
+          }));
+        }
         apps.push({
           id: String(p.id),
           name: p.display_name || p.name,
@@ -101,6 +140,7 @@ export const AgentDojoProvider: IntegrationProvider = {
           logo: p.favicon || p.icon_url || null,
           categories: [],
           authSchemes: mapAuthSchemes(p.provider_type),
+          credentialFields,
         });
       }
     }
@@ -146,8 +186,9 @@ export const AgentDojoProvider: IntegrationProvider = {
     redirectUrl: string,
     credentials?: ConnectionCredentials
   ): Promise<ConnectionRequest> {
-    // OAuth flow: no credentials provided, or explicit OAUTH2 scheme
-    if (!credentials?.apiKey) {
+    // OAuth flow: no credentials provided (neither apiKey nor multi-field)
+    const hasCredentials = credentials?.apiKey || (credentials?.fields && Object.keys(credentials.fields).length > 0);
+    if (!hasCredentials) {
       // Init OAuth via MCP API
       const res = await fetch(`${AGENTDOJO_API_BASE}/oauth/init`, {
         method: "POST",
@@ -181,17 +222,58 @@ export const AgentDojoProvider: IntegrationProvider = {
     }
 
     // API key flow: store credential directly
+    // Support arbitrary credential fields via credentials.fields, fall back to single api_key
+    const credentialData = credentials.fields && Object.keys(credentials.fields).length > 0
+      ? credentials.fields
+      : { api_key: credentials.apiKey };
+
+    // Resolve toolkit slug to actual provider name/id — the credential API
+    // needs the provider (e.g. "pushover" id=26), not the toolkit slug
+    // (e.g. "pushover-notifications"). Fetch the toolkit to get auth_provider_id.
+    let providerName = appSlug;
+    let providerId: string | number = appSlug;
+    try {
+      const tkRes = await fetch(`${AGENTDOJO_API_BASE}/toolkits?include_tools=false`, {
+        headers: { "X-API-Key": apiKey, "Content-Type": "application/json" },
+      });
+      if (tkRes.ok) {
+        const tkData = await tkRes.json();
+        const toolkits = tkData.toolkits || tkData.data || [];
+        const toolkit = toolkits.find((t: any) => t.name === appSlug || t.slug === appSlug);
+        if (toolkit?.auth_provider_id) {
+          // Fetch provider details
+          const provRes = await fetch(`${AGENTDOJO_API_BASE}/providers?is_active=true`, {
+            headers: { "X-API-Key": apiKey, "Content-Type": "application/json" },
+          });
+          if (provRes.ok) {
+            const provData = await provRes.json();
+            const providers = provData.providers || provData.data || [];
+            const prov = providers.find((p: any) => p.id === toolkit.auth_provider_id);
+            if (prov) {
+              providerName = prov.name;
+              providerId = prov.id;
+            }
+          }
+        }
+      }
+    } catch (e) {
+      console.warn("[AgentDojo] Failed to resolve provider for toolkit:", appSlug, e);
+    }
+
+    const credBody: Record<string, unknown> = {
+      name: providerName,
+      provider_id: providerId,
+      provider_name: providerName,
+      credential_data: credentialData,
+    };
+    console.log("[AgentDojo] Creating credential:", JSON.stringify(credBody));
     const res = await fetch(`${AGENTDOJO_API_BASE}/credentials`, {
       method: "POST",
       headers: {
         "X-API-Key": apiKey,
         "Content-Type": "application/json",
       },
-      body: JSON.stringify({
-        provider_id: appSlug,
-        provider_name: appSlug,
-        credential_data: { api_key: credentials.apiKey },
-      }),
+      body: JSON.stringify(credBody),
     });
 
     if (!res.ok) {

package/src/integrations/index.ts

@@ -9,6 +9,12 @@ export interface IntegrationApp {
   logo: string | null;
   categories: string[];
   authSchemes: string[]; // e.g., ["OAUTH2", "API_KEY"]
+  providerSlug?: string; // The underlying provider name (one provider can serve multiple toolkits)
+  credentialFields?: { // Fields required for API_KEY auth (from provider auth_config)
+    name: string;
+    description?: string;
+    required?: boolean;
+  }[];
 }
 
 export interface ConnectedAccount {
@@ -32,6 +38,7 @@ export interface ConnectionCredentials {
   bearerToken?: string;
   username?: string;
   password?: string;
+  fields?: Record<string, string>; // Arbitrary credential fields (e.g. { appToken: "...", userKey: "..." })
 }
 
 export interface IntegrationProvider {