onbuzz 4.9.0 → 4.9.5

package/src/tools/composioTool.js

@@ -11,8 +11,10 @@
  * holds the OAuth tokens server-side and auto-refreshes them. Our
  * code never sees raw tokens.
  *
- * API key: `process.env.COMPOSIO_API_KEY`. Required to do anything;
- * obtain at dashboard.composio.dev.
+ * API key: resolved by composioService.getClient() — first checks the
+ * encrypted vault (where the web-ui's Integrations page saves it),
+ * then falls back to process.env.COMPOSIO_API_KEY for ops who set it
+ * out-of-band. Obtain at dashboard.composio.dev.
  *
  * Security note: Composio had a security incident in May 2026 that
  * leaked ~5K GitHub OAuth tokens. Key rotation should be automatable
@@ -25,13 +27,17 @@
  */
 
 import { BaseTool } from './baseTool.js';
+import { ComposioIndexer } from '../services/embeddings/indexers/composioIndexer.js';
+import * as composioService from '../services/composioService.js';
 
 const SUPPORTED_ACTIONS = [
-  'list-toolkits',      // discover available integrations
-  'list-tools',         // for a specific toolkit, list its action slugs + schemas
-  'connect',            // generate a Connect Link URL for the user to OAuth into a toolkit
-  'connection-status',  // check whether the current user has an active connection to a toolkit
-  'execute',            // invoke a Composio action with arguments
+  'list-toolkits',          // discover available integrations
+  'list-tools',             // for a specific toolkit, list its action slugs + schemas
+  'connect',                // generate a Connect Link URL for the user to OAuth into a toolkit
+  'connection-status',      // check whether the current user has an active connection to a toolkit
+  'execute',                // invoke a Composio action with arguments
+  'find-action',            // semantic search over the Composio catalog (needs index)
+  'backfill-actions-index', // (re)build the catalog vector index for find-action
 ];
 
 class ComposioTool extends BaseTool {
@@ -41,8 +47,13 @@ class ComposioTool extends BaseTool {
     this.requiresProject = false;
     this.isAsync = true;
     this.timeout = config.timeout || 60_000;
-    this._sdkClient = null;          // lazy-loaded @composio/core instance
-    this._sdkUnavailableReason = null; // populated if dynamic import fails
+    // SDK client + key resolution are owned by composioService; we
+    // don't keep our own instance cache here because the service's
+    // cache is already keyed by API-key fingerprint and gets
+    // invalidated when the UI rotates the key. Holding a parallel
+    // cache here would mean "agent sees old key" after a rotation —
+    // which is exactly the regression that caused the
+    // "COMPOSIO_API_KEY env var is not set" bug report.
   }
 
   getDescription() {
@@ -106,8 +117,10 @@ Returns: { success: true|false, output: <upstream result>, error? }
 NOTES:
 - Each agent is mapped to a distinct Composio userId (agentId is reused).
   Tokens are isolated per-agent.
-- The org-level API key is read from \`COMPOSIO_API_KEY\` env var.
-- If the env var is missing OR the SDK isn't installed, every action
+- The org-level API key is resolved by composioService: encrypted vault
+  first (saved via Settings → Integrations), then \`COMPOSIO_API_KEY\`
+  env var as an operator fallback.
+- If neither source has a key OR the SDK isn't installed, every action
   returns a structured error so the agent can recover gracefully.
 - Composio holds OAuth tokens server-side. We never see raw tokens.
     `.trim();
@@ -153,45 +166,31 @@ NOTES:
           if (!a.toolkitSlug) errors.push(`${tag}toolkitSlug is required for execute`);
           if (!a.actionSlug)  errors.push(`${tag}actionSlug is required for execute`);
           break;
+        case 'find-action':
+          if (!a.query || typeof a.query !== 'string' || !a.query.trim()) {
+            errors.push(`${tag}query is required for find-action`);
+          }
+          if (a.topK != null && (!Number.isInteger(a.topK) || a.topK <= 0)) {
+            errors.push(`${tag}topK must be a positive integer when provided`);
+          }
+          break;
+        // backfill-actions-index needs no params (full rebuild of the catalog).
       }
     }
     return { valid: errors.length === 0, errors };
   }
 
   /**
-   * Lazy SDK loader. Catches missing-dependency / missing-API-key gracefully.
+   * Delegate SDK loading to composioService — single source of truth for
+   * key resolution (vault → env-var fallback) and client caching. The
+   * service's cache invalidates on key rotation (UI save / forget), so
+   * the next invocation here picks up the new key without a restart.
+   *
    * Returns { client } on success or { error } on failure — never throws.
-   * Override-friendly: tests can stub `_loadSdk` to inject a mock client.
+   * Tests can still stub this method to inject a mock client.
    */
   async _loadSdk() {
-    if (this._sdkClient) return { client: this._sdkClient };
-    if (this._sdkUnavailableReason) return { error: this._sdkUnavailableReason };
-
-    const apiKey = process.env.COMPOSIO_API_KEY;
-    if (!apiKey) {
-      const error = 'COMPOSIO_API_KEY env var is not set. Obtain at https://dashboard.composio.dev and set it in the Loxia process environment.';
-      this._sdkUnavailableReason = error;
-      return { error };
-    }
-
-    let Composio;
-    try {
-      const mod = await import('@composio/core');
-      Composio = mod.Composio || mod.default?.Composio || mod.default;
-    } catch (e) {
-      const error = `@composio/core SDK is not installed. Run \`npm install @composio/core\` (optionalDependencies). Detail: ${e.message}`;
-      this._sdkUnavailableReason = error;
-      return { error };
-    }
-
-    try {
-      this._sdkClient = new Composio({ apiKey });
-      return { client: this._sdkClient };
-    } catch (e) {
-      const error = `Composio SDK initialization failed: ${e.message}`;
-      this._sdkUnavailableReason = error;
-      return { error };
-    }
+    return composioService.getClient();
   }
 
   async execute(params, context = {}) {
@@ -204,7 +203,11 @@ NOTES:
     const userId = context.agentId || 'default-user';
     const { client, error: sdkErr } = await this._loadSdk();
     if (sdkErr) {
-      return { success: false, error: sdkErr, hint: 'Set COMPOSIO_API_KEY and `npm install @composio/core`.' };
+      return {
+        success: false,
+        error: sdkErr,
+        hint: 'Add a Composio API key on the Integrations page (Settings → Integrations) or set COMPOSIO_API_KEY in the environment. Also ensure `@composio/core` is installed.',
+      };
     }
 
     // Single-action: return its result directly (back-compat with other
@@ -241,43 +244,346 @@ NOTES:
     return [];
   }
 
+  /**
+   * Lazy ComposioIndexer accessor. Mirrors the memoryTool / reminisce
+   * pattern: cached on the instance after first use; null when no
+   * embedding service is attached to aiService.
+   *
+   * @private
+   * @returns {ComposioIndexer | null}
+   */
+  _getComposioIndexer() {
+    if (this._composioIndexer) return this._composioIndexer;
+    const svc = this.aiService?.getEmbeddingService?.();
+    if (!svc) return null;
+    this._composioIndexer = new ComposioIndexer({ embeddingService: svc, logger: this.logger });
+    return this._composioIndexer;
+  }
+
+  /**
+   * Execute find-action: semantic search across the indexed catalog.
+   * Returns ranked candidates with their toolkitSlug, actionSlug,
+   * displayName, description, and inputSchema — everything the agent
+   * needs to immediately call `execute` without a second round-trip.
+   *
+   * When the index is empty / stale, surfaces a clear hint to run
+   * `backfill-actions-index` first.
+   *
+   * @private
+   */
+  async _executeFindAction(action, context = {}) {
+    const query = String(action.query || '').trim();
+    if (!query) {
+      return { success: false, action: 'find-action', error: 'query is required' };
+    }
+    const topK = Number.isInteger(action.topK) && action.topK > 0 ? action.topK : 10;
+    const hybrid = action.hybrid !== false;
+    const toolkitSlug = typeof action.toolkitSlug === 'string' && action.toolkitSlug ? action.toolkitSlug : null;
+    const userId = context.agentId || 'default-user';
+
+    // Try the embedding path first. We DEGRADE GRACEFULLY: if embeddings
+    // are missing, disabled, the index is empty, or search throws, we
+    // fall back to a substring-match scan via the Composio SDK so the
+    // agent always gets *something* useful. The fallback comes with a
+    // warning so the agent + operator know quality is degraded and how
+    // to upgrade it.
+    const indexer = this._getComposioIndexer();
+    const embeddingsLive = !!(indexer && indexer.isEnabled);
+    // Track WHY we fell back so the warning is accurate. Three distinct
+    // causes — each routes the operator to a different remediation.
+    let fallbackCause = 'embeddings-unavailable';
+    let embeddingErrorDetail = null;
+
+    if (embeddingsLive) {
+      fallbackCause = 'index-empty';
+      let indexCount = 0;
+      try {
+        const stats = await indexer.stats();
+        indexCount = stats?.count || 0;
+      } catch { /* fall through to fallback */ }
+
+      if (indexCount > 0) {
+        try {
+          const { hits, error } = await indexer.search(query, { topK, hybrid, toolkitSlug });
+          if (!error) {
+            return {
+              success: true,
+              action: 'find-action',
+              mode: hybrid ? 'hybrid' : 'semantic',
+              query, topK, toolkitSlug, hits,
+              message: hits.length === 0
+                ? `No actions matched "${query}"`
+                : `Found ${hits.length} candidate actions ranked by similarity to "${query}".`,
+            };
+          }
+          // Embedding search returned an error — fall through to substring.
+          fallbackCause = 'embedding-search-failed';
+          embeddingErrorDetail = error;
+          this.logger?.warn?.('[composio] embedding find-action failed; falling back to substring', { error });
+        } catch (err) {
+          fallbackCause = 'embedding-search-failed';
+          embeddingErrorDetail = err?.message || String(err);
+          this.logger?.warn?.('[composio] embedding find-action threw; falling back to substring', { error: err?.message });
+        }
+      }
+      // else: index empty → fall through with a "build me" hint.
+    }
+
+    // ─── Substring fallback ────────────────────────────────────────
+    let reason, upgradeHint;
+    if (fallbackCause === 'embeddings-unavailable') {
+      reason = 'Embeddings unavailable.';
+      upgradeHint = 'Enable an embedding provider in Settings → Embeddings for semantic ranking.';
+    } else if (fallbackCause === 'index-empty') {
+      reason = 'Catalog index is empty.';
+      upgradeHint = 'Run `composio.backfill-actions-index` once to enable semantic ranking.';
+    } else {
+      // 'embedding-search-failed'
+      reason = `Embedding search failed${embeddingErrorDetail ? ` (${embeddingErrorDetail})` : ''}.`;
+      upgradeHint = 'Check the embedding provider status; using substring search in the meantime.';
+    }
+
+    const fallback = await this._substringFindAction(query, { topK, toolkitSlug, userId });
+    return {
+      ...fallback,
+      warning: `${reason} Used substring search (lower quality). ${upgradeHint}`,
+      fallbackCause,
+    };
+  }
+
+  /**
+   * Brute-force substring/keyword search across the Composio catalog.
+   * Used when the embedding index is unavailable or empty so the agent
+   * still gets ranked candidates.
+   *
+   * Strategy:
+   *   1. List toolkits (or use the supplied toolkitSlug as the only one).
+   *   2. Score each by token overlap with the query against name/slug/desc.
+   *   3. For the top-5 toolkits (by score), list their tools and score
+   *      each tool the same way. Top-5 cap keeps API fan-out bounded —
+   *      a query against 250 toolkits should not produce 250 list-tools
+   *      calls in the worst case.
+   *   4. Rank tools by score, return top-K with the same hit shape as
+   *      the embedding path (toolkitSlug, actionSlug, displayName,
+   *      description, inputSchema, score).
+   *
+   * @private
+   */
+  async _substringFindAction(query, { topK, toolkitSlug, userId }) {
+    const tokenize = (s) => String(s || '')
+      .toLowerCase()
+      .split(/[\s_,.\-/()\[\]{}]+/)
+      .filter(t => t.length > 1);
+    const qTokens = tokenize(query);
+    if (qTokens.length === 0) {
+      return { success: true, action: 'find-action', mode: 'substring-fallback', query, topK, toolkitSlug, hits: [], message: 'No searchable tokens in query.' };
+    }
+    const scoreText = (text) => {
+      const tTokens = tokenize(text);
+      let s = 0;
+      for (const q of qTokens) {
+        for (const t of tTokens) {
+          if (t === q) s += 3;        // exact token match
+          else if (t.includes(q) || q.includes(t)) s += 1; // substring either way
+        }
+      }
+      return s;
+    };
+
+    // Step 1: get the candidate toolkit list.
+    let toolkits;
+    if (toolkitSlug) {
+      toolkits = [{ slug: toolkitSlug, name: toolkitSlug }];
+    } else {
+      const tk = await composioService.listToolkits();
+      if (!tk.success) {
+        return { success: false, action: 'find-action', mode: 'substring-fallback', error: tk.error };
+      }
+      toolkits = tk.toolkits;
+    }
+
+    if (toolkits.length === 0) {
+      return {
+        success: true,
+        action: 'find-action',
+        mode: 'substring-fallback',
+        query, topK, toolkitSlug,
+        hits: [],
+        message: 'Composio returned no toolkits to search.',
+      };
+    }
+
+    // Step 2: rank toolkits, take top-5 (or all if toolkitSlug is pinned).
+    const toolkitMaxFanout = toolkitSlug ? 1 : 5;
+    const toolkitsScored = toolkits
+      .map(tk => ({ tk, score: scoreText(`${tk.slug || ''} ${tk.name || ''} ${tk.description || ''}`) }))
+      .sort((a, b) => b.score - a.score);
+    // If at least one toolkit scored, scan those. Otherwise scan the first
+    // N alphabetically as a last-ditch — the query may not match any
+    // toolkit name but might match action names within a toolkit.
+    const toolkitsToScan = (toolkitsScored[0]?.score > 0
+      ? toolkitsScored.filter(x => x.score > 0)
+      : toolkitsScored
+    ).slice(0, toolkitMaxFanout);
+
+    // Step 3: scan tools in those toolkits.
+    const allHits = [];
+    for (const { tk } of toolkitsToScan) {
+      const r = await composioService.listTools(tk.slug, userId);
+      if (!r.success) continue;
+      for (const tool of (r.tools || [])) {
+        const text = `${tool.slug || ''} ${tool.displayName || tool.name || ''} ${tool.description || ''}`;
+        const score = scoreText(text);
+        if (score > 0) {
+          allHits.push({
+            toolkitSlug:  tk.slug,
+            actionSlug:   tool.slug,
+            displayName:  tool.displayName || tool.name || tool.slug,
+            description:  tool.description || '',
+            inputSchema:  tool.input || tool.parameters || tool.inputParameters || null,
+            score,
+          });
+        }
+      }
+    }
+    allHits.sort((a, b) => b.score - a.score);
+    const hits = allHits.slice(0, topK);
+
+    return {
+      success: true,
+      action: 'find-action',
+      mode: 'substring-fallback',
+      query, topK, toolkitSlug,
+      hits,
+      message: hits.length === 0
+        ? `No actions matched "${query}" via substring search.`
+        : `Found ${hits.length} candidate actions via substring search.`,
+    };
+  }
+
+  /**
+   * Execute backfill-actions-index: walk every Composio toolkit, fetch
+   * its actions, embed them all in batched calls, write the global
+   * catalog index. Drops any prior index first (full rebuild semantics).
+   *
+   * @private
+   */
+  async _executeBackfillActionsIndex(client, userId) {
+    const indexer = this._getComposioIndexer();
+    if (!indexer || !indexer.isEnabled) {
+      return {
+        success: false,
+        action: 'backfill-actions-index',
+        message: 'Embeddings are not enabled — nothing to backfill. Enable a provider in Settings → Embeddings first.',
+      };
+    }
+
+    // Delegate to composioService — same code path everything else uses.
+    // The previous implementation called `client.toolkits.list()` and
+    // `client.tools.list()` directly, which silently returned undefined
+    // because those methods don't exist on the @composio/core SDK
+    // (it's `toolkits.get` / `tools.get`, and toolkits.get drops
+    // pagination so even that wouldn't be enough). The service now
+    // paginates via REST and returns the full ~1043-toolkit catalog.
+    const tkResult = await composioService.listToolkits();
+    if (!tkResult.success) {
+      return {
+        success: false,
+        action: 'backfill-actions-index',
+        error: `Could not list toolkits: ${tkResult.error}`,
+      };
+    }
+    const toolkits = tkResult.toolkits;
+    if (toolkits.length === 0) {
+      return {
+        success: false,
+        action: 'backfill-actions-index',
+        error: 'Composio returned no toolkits — check the Composio API key and that the org has at least one integration enabled.',
+      };
+    }
+
+    const fetcher = async (toolkitSlug) => {
+      const r = await composioService.listTools(toolkitSlug, userId);
+      return r.success ? (r.tools || []) : [];
+    };
+
+    const r = await indexer.indexCatalog(toolkits, fetcher);
+    return {
+      success: r.ok !== false,
+      action: 'backfill-actions-index',
+      indexedActions: r.indexedActions || 0,
+      indexedToolkits: r.indexedToolkits || 0,
+      failedToolkits: r.failedToolkits || 0,
+      ...(r.error ? { error: r.error } : {}),
+      message: r.ok === false
+        ? `Backfill failed: ${r.reason || r.error || 'unknown'}`
+        : `Indexed ${r.indexedActions || 0} actions across ${r.indexedToolkits || 0} toolkits` +
+          (r.failedToolkits ? ` (${r.failedToolkits} toolkits failed to load)` : '') + '.',
+    };
+  }
+
   /** @private */
   async _dispatch(client, userId, action) {
     const type = action.type || action.action;
     try {
       switch (type) {
         case 'list-toolkits': {
-          const toolkits = await client.toolkits?.list?.();
-          return { success: true, action: 'list-toolkits', toolkits: toolkits || [] };
+          // Delegate to composioService — single source of truth for
+          // the SDK's paginated/wrapped response shape (`{items:[]}` vs
+          // `{data:[]}` vs Array). Direct `await client.toolkits.list()`
+          // here used to return the wrapper object as "the list" or
+          // collapse to `[]` via the `|| []` fallback, which is how
+          // agents saw "0 toolkits" even when integrations existed.
+          const r = await composioService.listToolkits();
+          return r.success
+            ? { success: true, action: 'list-toolkits', toolkits: r.toolkits }
+            : { success: false, action: 'list-toolkits', error: r.error };
         }
         case 'list-tools': {
-          const tools = await client.tools?.list?.({ toolkit: action.toolkitSlug, userId });
-          return { success: true, action: 'list-tools', toolkitSlug: action.toolkitSlug, tools: tools || [] };
+          const r = await composioService.listTools(action.toolkitSlug, userId);
+          return r.success
+            ? { success: true, action: 'list-tools', toolkitSlug: action.toolkitSlug, tools: r.tools }
+            : { success: false, action: 'list-tools', toolkitSlug: action.toolkitSlug, error: r.error };
         }
         case 'connect': {
-          const result = await client.connectedAccounts?.initiate?.({
-            userId,
-            toolkit: action.toolkitSlug,
-            ...(action.authConfigId ? { authConfigId: action.authConfigId } : {}),
+          // composioService.connect maps the cryptic SDK error
+          // "authConfigIds.0 should be a string, but you provided
+          // undefined" into a human message that names the actual
+          // remediation (go to Composio dashboard → Integrations,
+          // add the toolkit's integration).
+          const r = await composioService.connect(userId, action.toolkitSlug, {
+            authConfigId: action.authConfigId,
           });
-          return {
-            success: true,
-            action: 'connect',
-            connectLink: result?.redirectUrl || result?.connectLink || null,
-            connectionId: result?.id || result?.connectionId || null,
-            raw: result,
-          };
+          return r.success
+            ? {
+                success: true,
+                action: 'connect',
+                connectLink: r.connectLink,
+                connectionId: r.connectionId,
+                raw: r.raw,
+              }
+            : {
+                success: false,
+                action: 'connect',
+                error: r.error,
+                ...(r.code ? { code: r.code } : {}),
+              };
         }
         case 'connection-status': {
-          const conn = await client.connectedAccounts?.get?.({ userId, toolkit: action.toolkitSlug });
-          const connected = !!(conn && (conn.status === 'ACTIVE' || conn.connected === true));
+          // composioService.connectionStatus uses `.list({userId,
+          // toolkit})` + client-side filter — the old `.get({…})`
+          // path passed an object as a connectionId and got back the
+          // "Path parameters result in path with invalid segments"
+          // SDK error.
+          const r = await composioService.connectionStatus(userId, action.toolkitSlug);
           return {
-            success: true,
+            success: r.success,
             action: 'connection-status',
             toolkitSlug: action.toolkitSlug,
-            connected,
-            status: conn?.status || 'NOT_CONNECTED',
-            connectionId: conn?.id || null,
+            connected: r.connected,
+            status: r.status,
+            connectionId: r.connectionId,
+            ...(r.error ? { error: r.error } : {}),
           };
         }
         case 'execute': {
@@ -294,6 +600,10 @@ NOTES:
             error: result?.error || null,
           };
         }
+        case 'find-action':
+          return await this._executeFindAction(action, { agentId: userId });
+        case 'backfill-actions-index':
+          return await this._executeBackfillActionsIndex(client, userId);
         default:
           return { success: false, error: `Unknown composio action: ${type}` };
       }