brownian-code 2026.2.10

package/src/tools/fetch/web-fetch.ts

@@ -0,0 +1,371 @@
+/**
+ * web_fetch tool — lightweight one-shot page reader with caching.
+ *
+ * Core extraction logic ported from OpenClaw's src/agents/tools/web-fetch.ts (MIT license).
+ * Adapted for Brownian's LangChain DynamicStructuredTool + Zod framework.
+ *
+ * Differences from OpenClaw:
+ * - fetchWithSsrFGuard replaced with plain fetch + manual redirect handling
+ * - Firecrawl fallback removed (falls back to htmlToMarkdown instead)
+ * - Config resolution replaced with hardcoded defaults
+ * - Tool wrapper uses LangChain DynamicStructuredTool + Zod (not AnyAgentTool + TypeBox)
+ */
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { formatToolResult } from '../types.js';
+import { wrapExternalContent, wrapWebContent } from './external-content.js';
+import {
+  extractReadableContent,
+  htmlToMarkdown,
+  markdownToText,
+  truncateText,
+  type ExtractMode,
+} from './web-fetch-utils.js';
+import {
+  type CacheEntry,
+  DEFAULT_CACHE_TTL_MINUTES,
+  DEFAULT_TIMEOUT_SECONDS,
+  normalizeCacheKey,
+  readCache,
+  readResponseText,
+  resolveCacheTtlMs,
+  resolveTimeoutSeconds,
+  withTimeout,
+  writeCache,
+} from './cache.js';
+
+// ============================================================================
+// Constants (identical to OpenClaw)
+// ============================================================================
+
+const DEFAULT_FETCH_MAX_CHARS = 50_000;
+const DEFAULT_FETCH_MAX_REDIRECTS = 3;
+const DEFAULT_ERROR_MAX_CHARS = 4_000;
+const DEFAULT_FETCH_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+
+// ============================================================================
+// Cache (identical to OpenClaw)
+// ============================================================================
+
+const FETCH_CACHE = new Map<string, CacheEntry<Record<string, unknown>>>();
+
+// ============================================================================
+// Content wrapping (identical to OpenClaw)
+// ============================================================================
+
+const WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD = wrapWebContent("", "web_fetch").length;
+const WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD = wrapExternalContent("", {
+  source: "web_fetch",
+  includeWarning: false,
+}).length;
+
+function wrapWebFetchContent(
+  value: string,
+  maxChars: number,
+): {
+  text: string;
+  truncated: boolean;
+  rawLength: number;
+  wrappedLength: number;
+} {
+  if (maxChars <= 0) {
+    return { text: "", truncated: true, rawLength: 0, wrappedLength: 0 };
+  }
+  const includeWarning = maxChars >= WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD;
+  const wrapperOverhead = includeWarning
+    ? WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD
+    : WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD;
+  if (wrapperOverhead > maxChars) {
+    const minimal = includeWarning
+      ? wrapWebContent("", "web_fetch")
+      : wrapExternalContent("", { source: "web_fetch", includeWarning: false });
+    const truncatedWrapper = truncateText(minimal, maxChars);
+    return {
+      text: truncatedWrapper.text,
+      truncated: true,
+      rawLength: 0,
+      wrappedLength: truncatedWrapper.text.length,
+    };
+  }
+  const maxInner = Math.max(0, maxChars - wrapperOverhead);
+  let truncated = truncateText(value, maxInner);
+  let wrappedText = includeWarning
+    ? wrapWebContent(truncated.text, "web_fetch")
+    : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+
+  if (wrappedText.length > maxChars) {
+    const excess = wrappedText.length - maxChars;
+    const adjustedMaxInner = Math.max(0, maxInner - excess);
+    truncated = truncateText(value, adjustedMaxInner);
+    wrappedText = includeWarning
+      ? wrapWebContent(truncated.text, "web_fetch")
+      : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+  }
+
+  return {
+    text: wrappedText,
+    truncated: truncated.truncated,
+    rawLength: truncated.text.length,
+    wrappedLength: wrappedText.length,
+  };
+}
+
+function wrapWebFetchField(value: string | undefined): string | undefined {
+  if (!value) {
+    return value;
+  }
+  return wrapExternalContent(value, { source: "web_fetch", includeWarning: false });
+}
+
+// ============================================================================
+// Helpers (identical to OpenClaw)
+// ============================================================================
+
+function normalizeContentType(value: string | null | undefined): string | undefined {
+  if (!value) {
+    return undefined;
+  }
+  const [raw] = value.split(";");
+  const trimmed = raw?.trim();
+  return trimmed || undefined;
+}
+
+function looksLikeHtml(value: string): boolean {
+  const trimmed = value.trimStart();
+  if (!trimmed) {
+    return false;
+  }
+  const head = trimmed.slice(0, 256).toLowerCase();
+  return head.startsWith("<!doctype html") || head.startsWith("<html");
+}
+
+function formatWebFetchErrorDetail(params: {
+  detail: string;
+  contentType?: string | null;
+  maxChars: number;
+}): string {
+  const { detail, contentType, maxChars } = params;
+  if (!detail) {
+    return "";
+  }
+  let text = detail;
+  const contentTypeLower = contentType?.toLowerCase();
+  if (contentTypeLower?.includes("text/html") || looksLikeHtml(detail)) {
+    const rendered = htmlToMarkdown(detail);
+    const withTitle = rendered.title ? `${rendered.title}\n${rendered.text}` : rendered.text;
+    text = markdownToText(withTitle);
+  }
+  const truncatedResult = truncateText(text.trim(), maxChars);
+  return truncatedResult.text;
+}
+
+function resolveMaxChars(value: unknown, fallback: number, cap: number): number {
+  const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
+  const clamped = Math.max(100, Math.floor(parsed));
+  return Math.min(clamped, cap);
+}
+
+// ============================================================================
+// HTTP fetch with manual redirect handling (replaces OpenClaw's fetchWithSsrFGuard)
+// ============================================================================
+
+function isRedirectStatus(status: number): boolean {
+  return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
+}
+
+async function fetchWithRedirects(params: {
+  url: string;
+  maxRedirects: number;
+  timeoutMs: number;
+  headers: Record<string, string>;
+}): Promise<{ response: Response; finalUrl: string }> {
+  const signal = withTimeout(undefined, params.timeoutMs);
+  const visited = new Set<string>();
+  let currentUrl = params.url;
+  let redirectCount = 0;
+
+  while (true) {
+    let parsedUrl: URL;
+    try {
+      parsedUrl = new URL(currentUrl);
+    } catch {
+      throw new Error("Invalid URL: must be http or https");
+    }
+    if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+      throw new Error("Invalid URL: must be http or https");
+    }
+
+    const response = await fetch(parsedUrl.toString(), {
+      redirect: "manual",
+      headers: params.headers,
+      signal,
+    });
+
+    if (isRedirectStatus(response.status)) {
+      const location = response.headers.get("location");
+      if (!location) {
+        throw new Error(`Redirect missing location header (${response.status})`);
+      }
+      redirectCount += 1;
+      if (redirectCount > params.maxRedirects) {
+        throw new Error(`Too many redirects (limit: ${params.maxRedirects})`);
+      }
+      const nextUrl = new URL(location, parsedUrl).toString();
+      if (visited.has(nextUrl)) {
+        throw new Error("Redirect loop detected");
+      }
+      visited.add(nextUrl);
+      currentUrl = nextUrl;
+      continue;
+    }
+
+    return { response, finalUrl: currentUrl };
+  }
+}
+
+// ============================================================================
+// Core fetch logic (ported from OpenClaw's runWebFetch, Firecrawl branches removed)
+// ============================================================================
+
+async function runWebFetch(params: {
+  url: string;
+  extractMode: ExtractMode;
+  maxChars: number;
+  maxRedirects: number;
+  timeoutSeconds: number;
+  cacheTtlMs: number;
+  userAgent: string;
+}): Promise<Record<string, unknown>> {
+  const cacheKey = normalizeCacheKey(
+    `fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
+  );
+  const cached = readCache(FETCH_CACHE, cacheKey);
+  if (cached) {
+    return { ...cached.value, cached: true };
+  }
+
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(params.url);
+  } catch {
+    throw new Error("Invalid URL: must be http or https");
+  }
+  if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+    throw new Error("Invalid URL: must be http or https");
+  }
+
+  const start = Date.now();
+  const { response: res, finalUrl } = await fetchWithRedirects({
+    url: params.url,
+    maxRedirects: params.maxRedirects,
+    timeoutMs: params.timeoutSeconds * 1000,
+    headers: {
+      Accept: "*/*",
+      "User-Agent": params.userAgent,
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+
+  if (!res.ok) {
+    const rawDetail = await readResponseText(res);
+    const detail = formatWebFetchErrorDetail({
+      detail: rawDetail,
+      contentType: res.headers.get("content-type"),
+      maxChars: DEFAULT_ERROR_MAX_CHARS,
+    });
+    const wrappedDetail = wrapWebFetchContent(detail || res.statusText, DEFAULT_ERROR_MAX_CHARS);
+    throw new Error(`Web fetch failed (${res.status}): ${wrappedDetail.text}`);
+  }
+
+  const contentType = res.headers.get("content-type") ?? "application/octet-stream";
+  const normalizedContentType = normalizeContentType(contentType) ?? "application/octet-stream";
+  const body = await readResponseText(res);
+
+  let title: string | undefined;
+  let extractor = "raw";
+  let text = body;
+  if (contentType.includes("text/html")) {
+    const readable = await extractReadableContent({
+      html: body,
+      url: finalUrl,
+      extractMode: params.extractMode,
+    });
+    if (readable?.text) {
+      text = readable.text;
+      title = readable.title;
+      extractor = "readability";
+    } else {
+      // Fallback to htmlToMarkdown (OpenClaw falls to Firecrawl here)
+      const rendered = htmlToMarkdown(body);
+      text = params.extractMode === "text" ? markdownToText(rendered.text) : rendered.text;
+      title = rendered.title;
+      extractor = "htmlToMarkdown";
+    }
+  } else if (contentType.includes("application/json")) {
+    try {
+      text = JSON.stringify(JSON.parse(body), null, 2);
+      extractor = "json";
+    } catch {
+      text = body;
+      extractor = "raw";
+    }
+  }
+
+  const wrapped = wrapWebFetchContent(text, params.maxChars);
+  const wrappedTitle = title ? wrapWebFetchField(title) : undefined;
+  const payload = {
+    url: params.url,
+    finalUrl,
+    status: res.status,
+    contentType: normalizedContentType,
+    title: wrappedTitle,
+    extractMode: params.extractMode,
+    extractor,
+    truncated: wrapped.truncated,
+    length: wrapped.wrappedLength,
+    rawLength: wrapped.rawLength,
+    wrappedLength: wrapped.wrappedLength,
+    fetchedAt: new Date().toISOString(),
+    tookMs: Date.now() - start,
+    text: wrapped.text,
+  };
+  writeCache(FETCH_CACHE, cacheKey, payload, params.cacheTtlMs);
+  return payload;
+}
+
+// ============================================================================
+// Tool definition (adapted for Brownian's LangChain + Zod framework)
+// ============================================================================
+
+export const webFetchTool = new DynamicStructuredTool({
+  name: 'web_fetch',
+  description:
+    'Fetch and extract readable content from a URL (HTML → markdown/text). Use for lightweight page access without browser automation.',
+  schema: z.object({
+    url: z.string().describe('HTTP or HTTPS URL to fetch.'),
+    extractMode: z
+      .enum(['markdown', 'text'])
+      .optional()
+      .describe('Extraction mode ("markdown" or "text"). Defaults to "markdown".'),
+    maxChars: z
+      .number()
+      .min(100)
+      .optional()
+      .describe('Maximum characters to return (truncates when exceeded).'),
+  }),
+  func: async (input) => {
+    const extractMode: ExtractMode = input.extractMode === 'text' ? 'text' : 'markdown';
+    const maxChars = resolveMaxChars(input.maxChars, DEFAULT_FETCH_MAX_CHARS, DEFAULT_FETCH_MAX_CHARS);
+    const result = await runWebFetch({
+      url: input.url,
+      extractMode,
+      maxChars,
+      maxRedirects: DEFAULT_FETCH_MAX_REDIRECTS,
+      timeoutSeconds: resolveTimeoutSeconds(undefined, DEFAULT_TIMEOUT_SECONDS),
+      cacheTtlMs: resolveCacheTtlMs(undefined, DEFAULT_CACHE_TTL_MINUTES),
+      userAgent: DEFAULT_FETCH_USER_AGENT,
+    });
+    return formatToolResult(result, [input.url]);
+  },
+});

package/src/tools/index.ts

@@ -0,0 +1,12 @@
+// Tool registry - the primary way to access tools and their descriptions
+export { getToolRegistry, getTools, buildToolDescriptions } from './registry.js';
+export type { RegisteredTool } from './registry.js';
+
+// Individual tool exports (for backward compatibility and direct access)
+export { tavilySearch } from './search/index.js';
+
+// Tool descriptions
+export {
+  WEB_SEARCH_DESCRIPTION,
+  CRYPTO_TOOLS_DESCRIPTION,
+} from './descriptions/index.js';

package/src/tools/registry.ts

@@ -0,0 +1,130 @@
+import { StructuredToolInterface } from '@langchain/core/tools';
+import { createCryptoTools } from './crypto/index.js';
+import { exaSearch, tavilySearch } from './search/index.js';
+import { skillTool, SKILL_TOOL_DESCRIPTION } from './skill.js';
+import { webFetchTool } from './fetch/index.js';
+import { browserTool } from './browser/index.js';
+import { CRYPTO_TOOLS_DESCRIPTION, WEB_SEARCH_DESCRIPTION, WEB_FETCH_DESCRIPTION, BROWSER_DESCRIPTION } from './descriptions/index.js';
+import { discoverSkills } from '../skills/index.js';
+
+/**
+ * A registered tool with its rich description for system prompt injection.
+ */
+export interface RegisteredTool {
+  /** Tool name (must match the tool's name property) */
+  name: string;
+  /** The actual tool instance */
+  tool: StructuredToolInterface;
+  /** Rich description for system prompt (includes when to use, when not to use, etc.) */
+  description: string;
+}
+
+/**
+ * Get all registered tools with their descriptions.
+ * Conditionally includes tools based on environment configuration.
+ *
+ * @param _model - The model name (kept for API compatibility)
+ * @returns Array of registered tools
+ */
+export function getToolRegistry(_model: string): RegisteredTool[] {
+  const tools: RegisteredTool[] = [];
+
+  // Add all crypto tools (12 tools: schema + invoke + 10 categories)
+  // The rich description is attached to invoke_api_endpoint as the primary data tool
+  const cryptoTools = createCryptoTools();
+  for (const tool of cryptoTools) {
+    tools.push({
+      name: tool.name,
+      tool: tool as StructuredToolInterface,
+      description: tool.name === 'invoke_api_endpoint' ? CRYPTO_TOOLS_DESCRIPTION : tool.description,
+    });
+  }
+
+  // Web fetch
+  tools.push({
+    name: 'web_fetch',
+    tool: webFetchTool,
+    description: WEB_FETCH_DESCRIPTION,
+  });
+
+  // Browser
+  tools.push({
+    name: 'browser',
+    tool: browserTool,
+    description: BROWSER_DESCRIPTION,
+  });
+
+  // Include web_search if Exa or Tavily API key is configured (Exa preferred)
+  if (process.env.EXASEARCH_API_KEY) {
+    tools.push({
+      name: 'web_search',
+      tool: exaSearch,
+      description: WEB_SEARCH_DESCRIPTION,
+    });
+  } else if (process.env.TAVILY_API_KEY) {
+    tools.push({
+      name: 'web_search',
+      tool: tavilySearch,
+      description: WEB_SEARCH_DESCRIPTION,
+    });
+  }
+
+  // Include skill tool if any skills are available
+  const availableSkills = discoverSkills();
+  if (availableSkills.length > 0) {
+    tools.push({
+      name: 'skill',
+      tool: skillTool,
+      description: SKILL_TOOL_DESCRIPTION,
+    });
+  }
+
+  return tools;
+}
+
+/**
+ * Get just the tool instances for binding to the LLM.
+ *
+ * @param model - The model name
+ * @returns Array of tool instances
+ */
+export function getTools(model: string): StructuredToolInterface[] {
+  return getToolRegistry(model).map((t) => t.tool);
+}
+
+/**
+ * Build the tool descriptions section for the system prompt.
+ * Formats each tool's rich description with a header.
+ * Groups crypto tools together under a single section.
+ *
+ * @param model - The model name
+ * @returns Formatted string with all tool descriptions
+ */
+export function buildToolDescriptions(model: string): string {
+  const registry = getToolRegistry(model);
+
+  // Separate invoke_api_endpoint (carries the rich crypto description) from other tools
+  const cryptoMain = registry.find((t) => t.name === 'invoke_api_endpoint');
+  const otherTools = registry.filter((t) => t.name !== 'invoke_api_endpoint');
+
+  const sections: string[] = [];
+
+  // Crypto tools section (with the rich description from invoke_api_endpoint)
+  if (cryptoMain) {
+    sections.push(`### Crypto Research Tools\n\n${cryptoMain.description}`);
+  }
+
+  // Other tools get individual sections
+  for (const t of otherTools) {
+    // Skip individual crypto tools from getting their own header — they're covered by the crypto section
+    const isCryptoTool = t.name.startsWith('get_') && t.name.endsWith('_endpoints')
+      || t.name === 'get_api_endpoint_schema'
+      || t.name === 'invoke_api_endpoint';
+
+    if (!isCryptoTool) {
+      sections.push(`### ${t.name}\n\n${t.description}`);
+    }
+  }
+
+  return sections.join('\n\n');
+}

package/src/tools/search/exa.ts

@@ -0,0 +1,43 @@
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { ExaSearchResults } from '@langchain/exa';
+import Exa from 'exa-js';
+import { z } from 'zod';
+import { formatToolResult, parseSearchResults } from '../types.js';
+import { logger } from '@/utils';
+
+// Lazily initialized to avoid errors when API key is not set
+let exaTool: ExaSearchResults | null = null;
+
+function getExaTool(): ExaSearchResults {
+  if (!exaTool) {
+    const client = new Exa(process.env.EXASEARCH_API_KEY);
+    // exa-js@2.x (root) vs exa-js@1.x (inside @langchain/exa) have
+    // incompatible private fields but are compatible at runtime.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    exaTool = new ExaSearchResults({
+      client: client as any,
+      searchArgs: { numResults: 5, text: true },
+    });
+  }
+  return exaTool!;
+}
+
+export const exaSearch = new DynamicStructuredTool({
+  name: 'web_search',
+  description:
+    'Search the web for current information on any topic. Returns relevant search results with URLs and content snippets.',
+  schema: z.object({
+    query: z.string().describe('The search query to look up on the web'),
+  }),
+  func: async (input) => {
+    try {
+      const result = await getExaTool().invoke(input.query);
+      const { parsed, urls } = parseSearchResults(result);
+      return formatToolResult(parsed, urls);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      logger.error(`[Exa API] error: ${message}`);
+      throw new Error(`[Exa API] ${message}`);
+    }
+  },
+});

package/src/tools/search/index.ts

@@ -0,0 +1,2 @@
+export { tavilySearch } from './tavily.js';
+export { exaSearch } from './exa.js';

package/src/tools/search/tavily.ts

@@ -0,0 +1,35 @@
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { TavilySearch } from '@langchain/tavily';
+import { z } from 'zod';
+import { formatToolResult, parseSearchResults } from '../types.js';
+import { logger } from '../../utils/logger.js';
+
+// Lazily initialized to avoid errors when API key is not set
+let tavilyClient: TavilySearch | null = null;
+
+function getTavilyClient(): TavilySearch {
+  if (!tavilyClient) {
+    tavilyClient = new TavilySearch({ maxResults: 5 });
+  }
+  return tavilyClient;
+}
+
+export const tavilySearch = new DynamicStructuredTool({
+  name: 'web_search',
+  description:
+    'Search the web for current information on any topic. Returns relevant search results with URLs and content snippets.',
+  schema: z.object({
+    query: z.string().describe('The search query to look up on the web'),
+  }),
+  func: async (input) => {
+    try {
+      const result = await getTavilyClient().invoke({ query: input.query });
+      const { parsed, urls } = parseSearchResults(result);
+      return formatToolResult(parsed, urls);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      logger.error(`[Tavily API] error: ${message}`);
+      throw new Error(`[Tavily API] ${message}`);
+    }
+  },
+});

package/src/tools/skill.ts

@@ -0,0 +1,62 @@
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { getSkill, discoverSkills } from '../skills/index.js';
+
+/**
+ * Rich description for the skill tool.
+ * Used in the system prompt to guide the LLM on when and how to use this tool.
+ */
+export const SKILL_TOOL_DESCRIPTION = `
+Execute a skill to get specialized instructions for complex tasks.
+
+## When to Use
+
+- When the user's query matches an available skill's description
+- For complex workflows that benefit from structured guidance (e.g., token analysis, protocol research)
+- When you need step-by-step instructions for a specialized task
+
+## When NOT to Use
+
+- For simple queries that don't require specialized workflows
+- When no available skill matches the task
+- If you've already invoked the skill for this query (don't invoke twice)
+
+## Usage Notes
+
+- Invoke the skill IMMEDIATELY when relevant, as your first action
+- The skill returns instructions that you should follow to complete the task
+- Use the skill name exactly as listed in Available Skills
+- Pass any relevant arguments (like token symbols or protocol names) via the args parameter
+`.trim();
+
+/**
+ * Skill invocation tool.
+ * Loads and returns skill instructions for the agent to follow.
+ */
+export const skillTool = new DynamicStructuredTool({
+  name: 'skill',
+  description: 'Execute a skill to get specialized instructions for a task. Returns instructions to follow.',
+  schema: z.object({
+    skill: z.string().describe('Name of the skill to invoke'),
+    args: z.string().optional().describe('Optional arguments for the skill (e.g., protocol name, token symbol)'),
+  }),
+  func: async ({ skill, args }) => {
+    const skillDef = getSkill(skill);
+
+    if (!skillDef) {
+      const available = discoverSkills().map((s) => s.name).join(', ');
+      return `Error: Skill "${skill}" not found. Available skills: ${available || 'none'}`;
+    }
+
+    // Return instructions with optional args context
+    let result = `## Skill: ${skillDef.name}\n\n`;
+    
+    if (args) {
+      result += `**Arguments provided:** ${args}\n\n`;
+    }
+    
+    result += skillDef.instructions;
+
+    return result;
+  },
+});