denchclaw 2.5.2 → 2.5.4

package/apps/web/.next/standalone/extensions/dench-identity/composio-cheat-sheet.test.ts

@@ -1,11 +1,5 @@
-import { mkdirSync, writeFileSync, rmSync } from "node:fs";
-import path from "node:path";
-import os from "node:os";
-import { describe, expect, it, afterEach } from "vitest";
-import {
-  formatComposioToolCheatSheetFromIndex,
-  loadComposioToolCheatSheetMarkdown,
-} from "./composio-cheat-sheet.js";
+import { describe, expect, it } from "vitest";
+import { formatComposioToolCheatSheetFromIndex } from "./composio-cheat-sheet.js";
 
 describe("formatComposioToolCheatSheetFromIndex", () => {
   it("renders markdown with tool table and gotchas", () => {
@@ -16,6 +10,18 @@ describe("formatComposioToolCheatSheetFromIndex", () => {
           toolkit_slug: "gmail",
           toolkit_name: "Gmail",
           account_count: 2,
+          accounts: [
+            {
+              connected_account_id: "conn_gmail_1",
+              account_identity: "gmail:work",
+              account_identity_source: "gateway_stable_id",
+              identity_confidence: "high",
+              display_label: "Work Gmail",
+              account_email: "work@example.com",
+              related_connection_ids: [],
+              is_same_account_reconnect: false,
+            },
+          ],
           tools: [
             {
               name: "GMAIL_FETCH_EMAILS",
@@ -38,109 +44,11 @@ describe("formatComposioToolCheatSheetFromIndex", () => {
     expect(md).toContain("GMAIL_FETCH_EMAILS");
     expect(md).toContain("Read recent emails");
     expect(md).toContain("label_ids");
-    expect(md).toContain("Composio MCP");
-  });
-});
-
-describe("loadComposioToolCheatSheetMarkdown", () => {
-  let tmp: string;
-
-  afterEach(() => {
-    if (tmp) {
-      rmSync(tmp, { recursive: true, force: true });
-    }
-  });
-
-  it("reads composio-tool-index.json from workspace dir", () => {
-    tmp = path.join(
-      os.tmpdir(),
-      `dench-composio-index-${Date.now()}-${Math.random().toString(36).slice(2)}`,
-    );
-    mkdirSync(tmp, { recursive: true });
-    writeFileSync(
-      path.join(tmp, "composio-tool-index.json"),
-      JSON.stringify({
-        generated_at: "2025-01-02T00:00:00.000Z",
-        connected_apps: [
-          {
-            toolkit_slug: "slack",
-            toolkit_name: "Slack",
-            account_count: 1,
-            tools: [
-              {
-                name: "SLACK_SEND_MESSAGE",
-                title: "Send",
-                description_short: "Send a message.",
-                required_args: ["channel", "text"],
-                arg_hints: {},
-              },
-            ],
-            recipes: {},
-          },
-        ],
-      }),
-      "utf-8",
-    );
-
-    const md = loadComposioToolCheatSheetMarkdown(tmp);
-    expect(md).toContain("Slack (1 account connected)");
-    expect(md).toContain("SLACK_SEND_MESSAGE");
-    expect(md).toContain("composio_resolve_tool");
-  });
-
-  it("only claims verified MCP availability when the status file says so", () => {
-    tmp = path.join(
-      os.tmpdir(),
-      `dench-composio-status-${Date.now()}-${Math.random().toString(36).slice(2)}`,
-    );
-    mkdirSync(tmp, { recursive: true });
-    writeFileSync(
-      path.join(tmp, "composio-tool-index.json"),
-      JSON.stringify({
-        generated_at: "2025-01-02T00:00:00.000Z",
-        connected_apps: [
-          {
-            toolkit_slug: "gmail",
-            toolkit_name: "Gmail",
-            account_count: 1,
-            tools: [
-              {
-                name: "GMAIL_FETCH_EMAILS",
-                title: "Fetch emails",
-                description_short: "List inbox messages.",
-                required_args: [],
-                arg_hints: {},
-                default_args: { label_ids: ["INBOX"], max_results: 10 },
-              },
-            ],
-            recipes: { "Read recent emails": "GMAIL_FETCH_EMAILS" },
-          },
-        ],
-      }),
-      "utf-8",
-    );
-    writeFileSync(
-      path.join(tmp, "composio-mcp-status.json"),
-      JSON.stringify({
-        summary: {
-          verified: false,
-          message: "Composio MCP is configured and the gateway is reachable, but live agent visibility has not been verified yet.",
-        },
-      }),
-      "utf-8",
-    );
-
-    const md = loadComposioToolCheatSheetMarkdown(tmp);
+    expect(md).toContain("Dench Integrations");
+    expect(md).not.toContain("Composio MCP");
+    expect(md).toContain("composio_search_tools");
+    expect(md).toContain("composio_call_tool");
+    expect(md).toContain("Work Gmail");
     expect(md).toContain("configured integration layer");
-    expect(md).not.toContain("verified MCP tools available");
-  });
-
-  it("returns null when file is missing", () => {
-    tmp = path.join(
-      os.tmpdir(),
-      `dench-composio-empty-${Date.now()}-${Math.random().toString(36).slice(2)}`,
-    );
-    mkdirSync(tmp, { recursive: true });
-    expect(loadComposioToolCheatSheetMarkdown(tmp)).toBeNull();
   });
 });

package/apps/web/.next/standalone/extensions/dench-identity/composio-cheat-sheet.ts

@@ -1,94 +1,97 @@
-import { existsSync, readFileSync } from "node:fs";
-import path from "node:path";
-
 /**
- * Shape written by `apps/web/lib/composio-tool-index.ts` to
- * `<workspace>/composio-tool-index.json`. Kept in the extension package so the
- * agent runtime can format the cheat sheet without importing the Next app.
+ * Shared Composio response shapes kept in the extension package so the agent
+ * runtime can format human-readable guidance without importing the web app.
  */
+export type ComposioManagedAccount = {
+  connected_account_id: string;
+  account_identity: string;
+  account_identity_source: "gateway_stable_id" | "legacy_heuristic" | "connection_id";
+  identity_confidence: "high" | "low" | "unknown";
+  display_label: string;
+  account_label?: string | null;
+  account_name?: string | null;
+  account_email?: string | null;
+  external_account_id?: string | null;
+  related_connection_ids: string[];
+  is_same_account_reconnect: boolean;
+};
+
+export type ComposioToolSummary = {
+  name: string;
+  title: string;
+  description_short: string;
+  required_args: string[];
+  arg_hints: Record<string, string>;
+  default_args?: Record<string, unknown>;
+  example_args?: Record<string, unknown>;
+  example_prompts?: string[];
+  input_schema?: Record<string, unknown>;
+};
+
 export type ComposioToolIndexFile = {
   generated_at: string;
+  managed_tools?: string[];
   connected_apps: Array<{
     toolkit_slug: string;
     toolkit_name: string;
     account_count: number;
-    tools: Array<{
-      name: string;
-      title: string;
-      description_short: string;
-      required_args: string[];
-      arg_hints: Record<string, string>;
-      default_args?: Record<string, unknown>;
-      example_args?: Record<string, unknown>;
-      example_prompts?: string[];
-      input_schema?: Record<string, unknown>;
-    }>;
+    accounts?: ComposioManagedAccount[];
+    tools: ComposioToolSummary[];
     recipes: Record<string, string>;
   }>;
 };
 
-type ComposioMcpStatusFile = {
-  summary?: {
-    verified?: boolean;
-    message?: string;
-  };
-  config?: {
-    status?: "pass" | "fail" | "unknown";
-  };
-  gatewayTools?: {
-    status?: "pass" | "fail" | "unknown";
-  };
-  liveAgent?: {
-    status?: "pass" | "fail" | "unknown";
-  };
+export type ComposioToolCatalogFile = {
+  generated_at: string;
+  connected_apps: Array<{
+    toolkit_slug: string;
+    toolkit_name: string;
+    tools: ComposioToolSummary[];
+  }>;
 };
 
-function isComposioToolIndexFile(value: unknown): value is ComposioToolIndexFile {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    return false;
-  }
-  const rec = value as Record<string, unknown>;
-  if (typeof rec.generated_at !== "string" || !Array.isArray(rec.connected_apps)) {
-    return false;
-  }
-  return true;
-}
-
 /**
- * Build markdown for the identity system prompt from a parsed index file.
+ * Build markdown from a parsed app/tool summary.
  */
 export function formatComposioToolCheatSheetFromIndex(index: ComposioToolIndexFile): string {
-  return formatComposioToolCheatSheet(index, null);
+  return formatComposioToolCheatSheet(index);
 }
 
-function formatComposioToolCheatSheet(
-  index: ComposioToolIndexFile,
-  status: ComposioMcpStatusFile | null,
-): string {
-  const verified = status?.summary?.verified === true;
-  const summaryMessage = typeof status?.summary?.message === "string" ? status.summary.message : null;
+function formatComposioToolCheatSheet(index: ComposioToolIndexFile): string {
   const lines: string[] = [
-    "## Connected App Tools (via Composio MCP)",
+    "## Connected App Tools (Dench Integrations)",
     "",
-    verified
-      ? "You have verified MCP tools available for these connected apps. Call them directly."
-      : "Composio MCP is the configured integration layer for these connected apps. If the MCP tools are missing in this session, stop and report the Composio MCP repair status instead of bypassing it.",
+    "Dench Integrations is the configured integration layer for these connected apps. Always search first, inspect the returned full schemas and plan guidance, then execute the selected tool via `composio_call_tool`.",
     "",
-    "- Use `composio_resolve_tool` first when the exact Composio tool name or argument shape is not already obvious.",
-    "- Never use `gog`, shell CLIs, curl, or raw `/v1/composio/*` HTTP as a fallback for these connected apps.",
-    "- If a Composio tool fails because of argument shape, fix the JSON arguments and retry once.",
+    "- Use `composio_search_tools` first for every connected-app task unless you are intentionally consuming a compatibility response from `composio_resolve_tool`.",
+    "- Inspect the returned full `input_schema`, `recommended_plan_steps`, `known_pitfalls`, and any pagination hints before executing anything.",
+    "- Use `composio_resolve_tool` only when you specifically want a single best-match compatibility result instead of ranked search results.",
+    "- After searching or resolving, execute the returned tool via `composio_call_tool` with the returned `search_context_token`, optional `search_session_id`, and final `arguments` object.",
+    "- If the returned tool supports cursor fields like `starting_after`, `next_cursor`, or `page_token`, keep paginating until complete when the user asked for the full result.",
+    "- Never use `gog`, shell CLIs, curl, or raw gateway HTTP as a fallback for these connected apps.",
+    "- If an integration tool fails because of argument shape, fix the JSON arguments and retry once.",
     "",
   ];
-  if (summaryMessage) {
-    lines.push(`Current verification status: ${summaryMessage}`, "");
-  }
 
   for (const app of index.connected_apps) {
+    const accounts = app.accounts ?? [];
     const title =
       app.account_count > 1
         ? `### ${app.toolkit_name} (${app.account_count} accounts connected)`
         : `### ${app.toolkit_name} (1 account connected)`;
     lines.push(title, "");
+    if (accounts.length > 0) {
+      lines.push("**Connected accounts:**");
+      for (const account of accounts.slice(0, 5)) {
+        const bits = [
+          account.display_label,
+          account.account_email ? `email: ${account.account_email}` : null,
+          `id: \`${account.connected_account_id}\``,
+        ].filter(Boolean);
+        lines.push(`- ${bits.join(" · ")}`);
+      }
+      lines.push("");
+    }
     lines.push("| Intent | Tool | Key args |");
     lines.push("|--------|------|----------|");
 
@@ -147,51 +150,3 @@ function formatComposioToolCheatSheet(
 
   return lines.join("\n").trimEnd() + "\n";
 }
-
-function readComposioToolIndex(workspaceDir: string): ComposioToolIndexFile | null {
-  const filePath = path.join(workspaceDir, "composio-tool-index.json");
-  if (!existsSync(filePath)) {
-    return null;
-  }
-  try {
-    const raw = JSON.parse(readFileSync(filePath, "utf-8")) as unknown;
-    return isComposioToolIndexFile(raw) ? raw : null;
-  } catch {
-    return null;
-  }
-}
-
-export function readComposioToolIndexFile(workspaceDir: string): ComposioToolIndexFile | null {
-  return readComposioToolIndex(workspaceDir);
-}
-
-function readComposioMcpStatus(workspaceDir: string): ComposioMcpStatusFile | null {
-  const filePath = path.join(workspaceDir, "composio-mcp-status.json");
-  if (!existsSync(filePath)) {
-    return null;
-  }
-  try {
-    const raw = JSON.parse(readFileSync(filePath, "utf-8")) as unknown;
-    if (raw && typeof raw === "object" && !Array.isArray(raw)) {
-      return raw as ComposioMcpStatusFile;
-    }
-    return null;
-  } catch {
-    return null;
-  }
-}
-
-export function readComposioMcpStatusFile(workspaceDir: string): ComposioMcpStatusFile | null {
-  return readComposioMcpStatus(workspaceDir);
-}
-
-/**
- * Loads and formats the cheat sheet, or returns null if no index file / invalid JSON.
- */
-export function loadComposioToolCheatSheetMarkdown(workspaceDir: string): string | null {
-  const index = readComposioToolIndex(workspaceDir);
-  if (!index || index.connected_apps.length === 0) {
-    return null;
-  }
-  return formatComposioToolCheatSheet(index, readComposioMcpStatus(workspaceDir));
-}

package/apps/web/.next/standalone/extensions/dench-identity/composio-tool-search.ts

@@ -0,0 +1,322 @@
+import type {
+  ComposioToolCatalogFile,
+  ComposioToolIndexFile,
+  ComposioToolSummary,
+} from "./composio-cheat-sheet.js";
+
+export type ComposioToolSearchResult = {
+  toolkit_slug: string;
+  toolkit_name: string;
+  tool: ComposioToolSummary;
+  source: "featured" | "catalog" | "recipe";
+  recipe_intents: string[];
+  score: number;
+  why_matched: string[];
+};
+
+export type ComposioToolSearchResponse = {
+  results: ComposioToolSearchResult[];
+  top_confidence: "high" | "medium" | "low";
+};
+
+type SearchDocument = {
+  toolkit_slug: string;
+  toolkit_name: string;
+  tool: ComposioToolSummary;
+  source: "featured" | "catalog" | "recipe";
+  recipe_intents: string[];
+  weightedTokens: string[];
+  fieldTokens: Record<string, string[]>;
+};
+
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+const FIELD_WEIGHTS: Record<string, number> = {
+  tool_name: 6,
+  title: 4,
+  recipe: 5,
+  example: 3,
+  app: 2,
+  description: 2,
+  required_args: 2,
+  arg_hints: 1,
+};
+
+function tokenize(value: string): string[] {
+  return value
+    .toLowerCase()
+    .split(/[^a-z0-9]+/u)
+    .filter((token) => token.length > 1);
+}
+
+function normalizeAppFilter(value: string | undefined): string | null {
+  if (typeof value !== "string") {
+    return null;
+  }
+  const trimmed = value.trim().toLowerCase();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function repeatTokens(tokens: string[], weight: number): string[] {
+  if (tokens.length === 0 || weight <= 1) {
+    return [...tokens];
+  }
+  const out: string[] = [];
+  for (const token of tokens) {
+    for (let i = 0; i < weight; i += 1) {
+      out.push(token);
+    }
+  }
+  return out;
+}
+
+function mergeToolSummary(
+  catalogTool: ComposioToolSummary | undefined,
+  featuredTool: ComposioToolSummary | undefined,
+): { tool: ComposioToolSummary; source: "featured" | "catalog" } | null {
+  if (!catalogTool && !featuredTool) {
+    return null;
+  }
+  if (catalogTool && featuredTool) {
+    return {
+      source: "featured",
+      tool: {
+        ...catalogTool,
+        ...featuredTool,
+        required_args: featuredTool.required_args.length > 0
+          ? featuredTool.required_args
+          : catalogTool.required_args,
+        arg_hints: Object.keys(featuredTool.arg_hints).length > 0
+          ? featuredTool.arg_hints
+          : catalogTool.arg_hints,
+        default_args: featuredTool.default_args ?? catalogTool.default_args,
+        example_args: featuredTool.example_args ?? catalogTool.example_args,
+        example_prompts: featuredTool.example_prompts?.length
+          ? featuredTool.example_prompts
+          : catalogTool.example_prompts,
+        input_schema: featuredTool.input_schema ?? catalogTool.input_schema,
+      },
+    };
+  }
+  if (featuredTool) {
+    return { tool: featuredTool, source: "featured" };
+  }
+  return { tool: catalogTool as ComposioToolSummary, source: "catalog" };
+}
+
+function buildSyntheticRecipeTool(
+  toolkitName: string,
+  toolName: string,
+  recipeIntents: string[],
+): ComposioToolSummary {
+  const primaryIntent = recipeIntents[0] ?? toolName;
+  return {
+    name: toolName,
+    title: primaryIntent,
+    description_short: `Recommended ${toolkitName} tool for ${primaryIntent}.`,
+    required_args: [],
+    arg_hints: {},
+    example_prompts: recipeIntents,
+  };
+}
+
+function buildSearchDocuments(
+  index: ComposioToolIndexFile,
+  catalog: ComposioToolCatalogFile | null,
+  appFilter: string | null,
+): SearchDocument[] {
+  const catalogBySlug = new Map(
+    (catalog?.connected_apps ?? []).map((app) => [app.toolkit_slug.toLowerCase(), app]),
+  );
+  const docs: SearchDocument[] = [];
+
+  for (const app of index.connected_apps) {
+    const normalizedApp = app.toolkit_slug.toLowerCase();
+    if (appFilter && normalizedApp !== appFilter && app.toolkit_name.toLowerCase() !== appFilter) {
+      continue;
+    }
+
+    const featuredByName = new Map(app.tools.map((tool) => [tool.name, tool]));
+    const recipeIntentsByTool = new Map<string, string[]>();
+    for (const [intent, toolName] of Object.entries(app.recipes)) {
+      const bucket = recipeIntentsByTool.get(toolName);
+      if (bucket) {
+        bucket.push(intent);
+      } else {
+        recipeIntentsByTool.set(toolName, [intent]);
+      }
+    }
+
+    const catalogApp = catalogBySlug.get(normalizedApp);
+    const catalogByName = new Map((catalogApp?.tools ?? []).map((tool) => [tool.name, tool]));
+    const allToolNames = new Set([
+      ...featuredByName.keys(),
+      ...catalogByName.keys(),
+      ...recipeIntentsByTool.keys(),
+    ]);
+
+    for (const toolName of allToolNames) {
+      const recipeIntents = recipeIntentsByTool.get(toolName) ?? [];
+      const merged = mergeToolSummary(catalogByName.get(toolName), featuredByName.get(toolName));
+      const source = merged?.source ?? "recipe";
+      const tool = merged?.tool ?? buildSyntheticRecipeTool(app.toolkit_name, toolName, recipeIntents);
+
+      const fieldTokens: Record<string, string[]> = {
+        tool_name: tokenize(tool.name),
+        title: tokenize(tool.title),
+        recipe: tokenize(recipeIntents.join(" ")),
+        example: tokenize((tool.example_prompts ?? []).join(" ")),
+        app: tokenize(`${app.toolkit_slug} ${app.toolkit_name}`),
+        description: tokenize(tool.description_short),
+        required_args: tokenize(tool.required_args.join(" ")),
+        arg_hints: tokenize([
+          ...Object.keys(tool.arg_hints),
+          ...Object.values(tool.arg_hints),
+        ].join(" ")),
+      };
+
+      const weightedTokens = Object.entries(fieldTokens).flatMap(([field, tokens]) =>
+        repeatTokens(tokens, FIELD_WEIGHTS[field] ?? 1)
+      );
+
+      docs.push({
+        toolkit_slug: app.toolkit_slug,
+        toolkit_name: app.toolkit_name,
+        tool,
+        source,
+        recipe_intents: recipeIntents,
+        weightedTokens,
+        fieldTokens,
+      });
+    }
+  }
+
+  return docs;
+}
+
+function bm25Score(
+  queryTokens: string[],
+  documentTokens: string[],
+  avgDocLength: number,
+  totalDocs: number,
+  documentFrequency: Map<string, number>,
+): number {
+  if (queryTokens.length === 0 || documentTokens.length === 0 || totalDocs === 0) {
+    return 0;
+  }
+
+  const tf = new Map<string, number>();
+  for (const token of documentTokens) {
+    tf.set(token, (tf.get(token) ?? 0) + 1);
+  }
+
+  const docLength = documentTokens.length;
+  let score = 0;
+  for (const token of queryTokens) {
+    const frequency = tf.get(token) ?? 0;
+    if (frequency === 0) {
+      continue;
+    }
+    const df = documentFrequency.get(token) ?? 0;
+    const idf = Math.log(1 + ((totalDocs - df + 0.5) / (df + 0.5)));
+    const numerator = frequency * (BM25_K1 + 1);
+    const denominator = frequency + BM25_K1 * (1 - BM25_B + BM25_B * (docLength / avgDocLength));
+    score += idf * (numerator / denominator);
+  }
+
+  return score;
+}
+
+function matchedFields(queryTokens: string[], fieldTokens: Record<string, string[]>): string[] {
+  const querySet = new Set(queryTokens);
+  const matched = Object.entries(fieldTokens)
+    .filter(([, tokens]) => tokens.some((token) => querySet.has(token)))
+    .map(([field]) => field.replace(/_/g, " "));
+  return matched.length > 0 ? matched : ["catalog"];
+}
+
+function topConfidence(scores: number[]): "high" | "medium" | "low" {
+  const first = scores[0] ?? 0;
+  const second = scores[1] ?? 0;
+  if (first <= 0) {
+    return "low";
+  }
+  if (second <= 0) {
+    return "high";
+  }
+  const ratio = first / Math.max(second, 0.0001);
+  if (ratio >= 1.5) {
+    return "high";
+  }
+  if (ratio >= 1.15) {
+    return "medium";
+  }
+  return "low";
+}
+
+export function searchComposioTools(params: {
+  index: ComposioToolIndexFile;
+  catalog: ComposioToolCatalogFile | null;
+  query: string;
+  app?: string;
+  topK?: number;
+}): ComposioToolSearchResponse {
+  const queryTokens = tokenize(params.query);
+  if (queryTokens.length === 0) {
+    return { results: [], top_confidence: "low" };
+  }
+
+  const documents = buildSearchDocuments(
+    params.index,
+    params.catalog,
+    normalizeAppFilter(params.app),
+  );
+  if (documents.length === 0) {
+    return { results: [], top_confidence: "low" };
+  }
+
+  const avgDocLength = documents.reduce((sum, doc) => sum + doc.weightedTokens.length, 0)
+    / Math.max(documents.length, 1);
+  const documentFrequency = new Map<string, number>();
+  for (const doc of documents) {
+    const seen = new Set(doc.weightedTokens);
+    for (const token of seen) {
+      documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
+    }
+  }
+
+  const scored = documents
+    .map((doc) => ({
+      doc,
+      score: bm25Score(
+        queryTokens,
+        doc.weightedTokens,
+        avgDocLength || 1,
+        documents.length,
+        documentFrequency,
+      ),
+    }))
+    .filter((entry) => entry.score > 0)
+    .sort((left, right) => {
+      if (right.score !== left.score) {
+        return right.score - left.score;
+      }
+      return left.doc.tool.name.localeCompare(right.doc.tool.name);
+    });
+
+  const topK = Math.max(1, Math.min(params.topK ?? 5, 10));
+  const results = scored.slice(0, topK).map(({ doc, score }) => ({
+    toolkit_slug: doc.toolkit_slug,
+    toolkit_name: doc.toolkit_name,
+    tool: doc.tool,
+    source: doc.source,
+    recipe_intents: doc.recipe_intents,
+    score,
+    why_matched: matchedFields(queryTokens, doc.fieldTokens),
+  }));
+
+  return {
+    results,
+    top_confidence: topConfidence(scored.slice(0, 2).map((entry) => entry.score)),
+  };
+}