@librechat/agents 2.4.22 → 2.4.31

package/src/llm/anthropic/utils/output_parsers.ts

@@ -0,0 +1,114 @@
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-empty-object-type */
+import { z } from 'zod';
+import {
+  BaseLLMOutputParser,
+  OutputParserException,
+} from '@langchain/core/output_parsers';
+import { JsonOutputKeyToolsParserParams } from '@langchain/core/output_parsers/openai_tools';
+import { ChatGeneration } from '@langchain/core/outputs';
+import { ToolCall } from '@langchain/core/messages/tool';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+interface AnthropicToolsOutputParserParams<T extends Record<string, any>>
+  extends JsonOutputKeyToolsParserParams<T> {}
+
+export class AnthropicToolsOutputParser<
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  T extends Record<string, any> = Record<string, any>,
+> extends BaseLLMOutputParser<T> {
+  static lc_name() {
+    return 'AnthropicToolsOutputParser';
+  }
+
+  lc_namespace = ['langchain', 'anthropic', 'output_parsers'];
+
+  returnId = false;
+
+  /** The type of tool calls to return. */
+  keyName: string;
+
+  /** Whether to return only the first tool call. */
+  returnSingle = false;
+
+  zodSchema?: z.ZodType<T>;
+
+  constructor(params: AnthropicToolsOutputParserParams<T>) {
+    super(params);
+    this.keyName = params.keyName;
+    this.returnSingle = params.returnSingle ?? this.returnSingle;
+    this.zodSchema = params.zodSchema;
+  }
+
+  protected async _validateResult(result: unknown): Promise<T> {
+    let parsedResult = result;
+    if (typeof result === 'string') {
+      try {
+        parsedResult = JSON.parse(result);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } catch (e: any) {
+        throw new OutputParserException(
+          `Failed to parse. Text: "${JSON.stringify(
+            result,
+            null,
+            2
+          )}". Error: ${JSON.stringify(e.message)}`,
+          result
+        );
+      }
+    } else {
+      parsedResult = result;
+    }
+    if (this.zodSchema === undefined) {
+      return parsedResult as T;
+    }
+    const zodParsedResult = await this.zodSchema.safeParseAsync(parsedResult);
+    if (zodParsedResult.success) {
+      return zodParsedResult.data;
+    } else {
+      throw new OutputParserException(
+        `Failed to parse. Text: "${JSON.stringify(
+          result,
+          null,
+          2
+        )}". Error: ${JSON.stringify(zodParsedResult.error.errors)}`,
+        JSON.stringify(parsedResult, null, 2)
+      );
+    }
+  }
+
+  async parseResult(generations: ChatGeneration[]): Promise<T> {
+    const tools = generations.flatMap((generation) => {
+      const { message } = generation;
+      if (!Array.isArray(message.content)) {
+        return [];
+      }
+      const tool = extractToolCalls(message.content)[0];
+      return tool;
+    });
+    if (tools[0] === undefined) {
+      throw new Error(
+        'No parseable tool calls provided to AnthropicToolsOutputParser.'
+      );
+    }
+    const [tool] = tools;
+    const validatedResult = await this._validateResult(tool.args);
+    return validatedResult;
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function extractToolCalls(content: Record<string, any>[]) {
+  const toolCalls: ToolCall[] = [];
+  for (const block of content) {
+    if (block.type === 'tool_use') {
+      toolCalls.push({
+        name: block.name,
+        args: block.input,
+        id: block.id,
+        type: 'tool_call',
+      });
+    }
+  }
+  return toolCalls;
+}

package/src/llm/openai/index.ts

@@ -7,6 +7,7 @@ import {
   ChatOpenAI as OriginalChatOpenAI,
   AzureChatOpenAI as OriginalAzureChatOpenAI,
 } from '@langchain/openai';
+import type { OpenAICoreRequestOptions } from 'node_modules/@langchain/deepseek/node_modules/@langchain/openai';
 import type * as t from '@langchain/openai';
 
 function createAbortHandler(controller: AbortController): () => void {
@@ -191,8 +192,8 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
     return this.client;
   }
   protected _getClientOptions(
-    options?: t.OpenAICoreRequestOptions
-  ): t.OpenAICoreRequestOptions {
+    options?: OpenAICoreRequestOptions
+  ): OpenAICoreRequestOptions {
     if (!(this.client as OpenAIClient | undefined)) {
       const openAIEndpointConfig: t.OpenAIEndpointConfig = {
         baseURL: this.clientConfig.baseURL,
@@ -214,7 +215,7 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
     const requestOptions = {
       ...this.clientConfig,
       ...options,
-    } as t.OpenAICoreRequestOptions;
+    } as OpenAICoreRequestOptions;
     return requestOptions;
   }
 }
@@ -224,8 +225,8 @@ export class ChatXAI extends OriginalChatXAI {
     return this.client;
   }
   protected _getClientOptions(
-    options?: t.OpenAICoreRequestOptions
-  ): t.OpenAICoreRequestOptions {
+    options?: OpenAICoreRequestOptions
+  ): OpenAICoreRequestOptions {
     if (!(this.client as OpenAIClient | undefined)) {
       const openAIEndpointConfig: t.OpenAIEndpointConfig = {
         baseURL: this.clientConfig.baseURL,
@@ -247,7 +248,7 @@ export class ChatXAI extends OriginalChatXAI {
     const requestOptions = {
       ...this.clientConfig,
       ...options,
-    } as t.OpenAICoreRequestOptions;
+    } as OpenAICoreRequestOptions;
     return requestOptions;
   }
 }

package/src/run.ts

@@ -141,7 +141,7 @@ export class Run<T extends t.BaseGraphState> {
         }
 
         const jsonSchema = zodToJsonSchema(
-          tool.schema.describe(tool.description ?? ''),
+          (tool.schema as t.ZodObjectAny).describe(tool.description ?? ''),
           tool.name
         );
         return (

package/src/scripts/search.ts

@@ -0,0 +1,141 @@
+/* eslint-disable no-console */
+// src/scripts/cli.ts
+import { config } from 'dotenv';
+config();
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { createSearchTool } from '@/tools/search';
+
+import { getArgs } from '@/scripts/args';
+import { Run } from '@/run';
+import { GraphEvents, Callback } from '@/common';
+import { getLLMConfig } from '@/utils/llmConfig';
+
+const conversationHistory: BaseMessage[] = [];
+async function testStandardStreaming(): Promise<void> {
+  const { userName, location, provider, currentDate } = await getArgs();
+  const { contentParts, aggregateContent } = createContentAggregator();
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_RUN_STEP_COMPLETED ======');
+        // console.dir(data, { depth: null });
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_RUN_STEP ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_RUN_STEP_DELTA ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        console.log('====== ON_MESSAGE_DELTA ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+    [GraphEvents.TOOL_START]: {
+      handle: (
+        _event: string,
+        data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
+        console.log('====== TOOL_START ======');
+        // console.dir(data, { depth: null });
+      },
+    },
+  };
+
+  const llmConfig = getLLMConfig(provider);
+
+  const run = await Run.create<t.IState>({
+    runId: 'test-run-id',
+    graphConfig: {
+      type: 'standard',
+      llmConfig,
+      tools: [createSearchTool()],
+      instructions:
+        'You are a friendly AI assistant. Always address the user by their name.',
+      additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+    },
+    returnContent: true,
+    customHandlers,
+  });
+
+  const config = {
+    configurable: {
+      provider,
+      thread_id: 'conversation-num-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  console.log('Test 1: Weather query (content parts test)');
+
+  // const userMessage = `
+  // Make a search for the weather in ${location} today, which is ${currentDate}.
+  // Before making the search, please let me know what you're about to do, then immediately start searching without hesitation.
+  // Make sure to always refer to me by name, which is ${userName}.
+  // After giving me a thorough summary, tell me a joke about the weather forecast we went over.
+  // `;
+  const userMessage = 'Are massage guns good?';
+
+  conversationHistory.push(new HumanMessage(userMessage));
+
+  const inputs = {
+    messages: conversationHistory,
+  };
+  const finalContentParts = await run.processStream(inputs, config);
+  const finalMessages = run.getRunMessages();
+  if (finalMessages) {
+    conversationHistory.push(...finalMessages);
+    console.dir(conversationHistory, { depth: null });
+  }
+  // console.dir(finalContentParts, { depth: null });
+  console.log('\n\n====================\n\n');
+  // console.dir(contentParts, { depth: null });
+}
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  console.log('Conversation history:');
+  process.exit(1);
+});
+
+testStandardStreaming().catch((err) => {
+  console.error(err);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  process.exit(1);
+});

package/src/tools/search/firecrawl.ts

@@ -0,0 +1,270 @@
+/* eslint-disable no-console */
+import axios from 'axios';
+
+export interface FirecrawlScrapeOptions {
+  formats?: string[];
+  includeTags?: string[];
+  excludeTags?: string[];
+  headers?: Record<string, string>;
+  waitFor?: number;
+  timeout?: number;
+}
+
+interface ScrapeMetadata {
+  // Core source information
+  sourceURL?: string;
+  url?: string;
+  scrapeId?: string;
+  statusCode?: number;
+  // Basic metadata
+  title?: string;
+  description?: string;
+  language?: string;
+  favicon?: string;
+  viewport?: string;
+  robots?: string;
+  'theme-color'?: string;
+  // Open Graph metadata
+  'og:url'?: string;
+  'og:title'?: string;
+  'og:description'?: string;
+  'og:type'?: string;
+  'og:image'?: string;
+  'og:image:width'?: string;
+  'og:image:height'?: string;
+  'og:site_name'?: string;
+  ogUrl?: string;
+  ogTitle?: string;
+  ogDescription?: string;
+  ogImage?: string;
+  ogSiteName?: string;
+  // Article metadata
+  'article:author'?: string;
+  'article:published_time'?: string;
+  'article:modified_time'?: string;
+  'article:section'?: string;
+  'article:tag'?: string;
+  'article:publisher'?: string;
+  publishedTime?: string;
+  modifiedTime?: string;
+  // Twitter metadata
+  'twitter:site'?: string;
+  'twitter:creator'?: string;
+  'twitter:card'?: string;
+  'twitter:image'?: string;
+  'twitter:dnt'?: string;
+  'twitter:app:name:iphone'?: string;
+  'twitter:app:id:iphone'?: string;
+  'twitter:app:url:iphone'?: string;
+  'twitter:app:name:ipad'?: string;
+  'twitter:app:id:ipad'?: string;
+  'twitter:app:url:ipad'?: string;
+  'twitter:app:name:googleplay'?: string;
+  'twitter:app:id:googleplay'?: string;
+  'twitter:app:url:googleplay'?: string;
+  // Facebook metadata
+  'fb:app_id'?: string;
+  // App links
+  'al:ios:url'?: string;
+  'al:ios:app_name'?: string;
+  'al:ios:app_store_id'?: string;
+  // Allow for additional properties that might be present
+  [key: string]: string | number | boolean | null | undefined;
+}
+
+export interface FirecrawlScrapeResponse {
+  success: boolean;
+  data?: {
+    markdown?: string;
+    html?: string;
+    rawHtml?: string;
+    screenshot?: string;
+    links?: string[];
+    metadata?: ScrapeMetadata;
+  };
+  error?: string;
+}
+
+export interface FirecrawlScraperConfig {
+  apiKey?: string;
+  apiUrl?: string;
+  formats?: string[];
+  timeout?: number;
+}
+const getDomainName = (
+  link: string,
+  metadata?: ScrapeMetadata
+): string | undefined => {
+  try {
+    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
+    const domain = new URL(url).hostname.replace(/^www\./, '');
+    if (domain) {
+      return domain;
+    }
+  } catch (e) {
+    // URL parsing failed
+    console.error('Error parsing URL:', e);
+  }
+
+  return;
+};
+
+export function getAttribution(
+  link: string,
+  metadata?: ScrapeMetadata
+): string | undefined {
+  if (!metadata) return getDomainName(link, metadata);
+
+  const possibleAttributions = [
+    metadata.ogSiteName,
+    metadata['og:site_name'],
+    metadata.title?.split('|').pop()?.trim(),
+    metadata['twitter:site']?.replace(/^@/, ''),
+  ];
+
+  const attribution = possibleAttributions.find(
+    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''
+  );
+  if (attribution != null) {
+    return attribution;
+  }
+
+  return getDomainName(link, metadata);
+}
+
+/**
+ * Firecrawl scraper implementation
+ * Uses the Firecrawl API to scrape web pages
+ */
+export class FirecrawlScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private defaultFormats: string[];
+  private timeout: number;
+
+  constructor(config: FirecrawlScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
+
+    const baseUrl =
+      config.apiUrl ??
+      process.env.FIRECRAWL_BASE_URL ??
+      'https://api.firecrawl.dev';
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
+
+    this.defaultFormats = config.formats ?? ['markdown', 'html'];
+    this.timeout = config.timeout ?? 30000;
+
+    if (!this.apiKey) {
+      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+    }
+
+    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
+  }
+
+  /**
+   * Scrape a single URL
+   * @param url URL to scrape
+   * @param options Scrape options
+   * @returns Scrape response
+   */
+  async scrapeUrl(
+    url: string,
+    options: FirecrawlScrapeOptions = {}
+  ): Promise<[string, FirecrawlScrapeResponse]> {
+    if (!this.apiKey) {
+      return [
+        url,
+        {
+          success: false,
+          error: 'FIRECRAWL_API_KEY is not set',
+        },
+      ];
+    }
+
+    try {
+      const response = await axios.post(
+        this.apiUrl,
+        {
+          url,
+          formats: options.formats || this.defaultFormats,
+          includeTags: options.includeTags,
+          excludeTags: options.excludeTags,
+          headers: options.headers,
+          waitFor: options.waitFor,
+          timeout: options.timeout ?? this.timeout,
+        },
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          timeout: this.timeout,
+        }
+      );
+
+      return [url, response.data];
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return [
+        url,
+        {
+          success: false,
+          error: `Firecrawl API request failed: ${errorMessage}`,
+        },
+      ];
+    }
+  }
+
+  /**
+   * Extract content from scrape response
+   * @param response Scrape response
+   * @returns Extracted content or empty string if not available
+   */
+  extractContent(response: FirecrawlScrapeResponse): string {
+    if (!response.success || !response.data) {
+      return '';
+    }
+
+    // Prefer markdown content if available
+    if (response.data.markdown != null) {
+      return response.data.markdown;
+    }
+
+    // Fall back to HTML content
+    if (response.data.html != null) {
+      return response.data.html;
+    }
+
+    // Fall back to raw HTML content
+    if (response.data.rawHtml != null) {
+      return response.data.rawHtml;
+    }
+
+    return '';
+  }
+
+  /**
+   * Extract metadata from scrape response
+   * @param response Scrape response
+   * @returns Metadata object
+   */
+  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {
+    if (!response.success || !response.data || !response.data.metadata) {
+      return {};
+    }
+
+    return response.data.metadata;
+  }
+}
+
+/**
+ * Create a Firecrawl scraper instance
+ * @param config Scraper configuration
+ * @returns Firecrawl scraper instance
+ */
+export const createFirecrawlScraper = (
+  config: FirecrawlScraperConfig = {}
+): FirecrawlScraper => {
+  return new FirecrawlScraper(config);
+};

package/src/tools/search/format.ts

@@ -0,0 +1,121 @@
+import type * as t from './types';
+
+export function formatResultsForLLM(results: t.SearchResultData): string {
+  let output = '';
+
+  const addSection = (title: string): void => {
+    output += `\n=== ${title} ===\n`;
+  };
+
+  // Organic (web) results
+  const organic = results.organic ?? [];
+  if (organic.length) {
+    addSection('Web Results');
+    organic.forEach((r, i) => {
+      output += [
+        `Source ${i + 1}: ${r.title ?? '(no title)'}`,
+        `Citation Anchor: \\ue202turn0search${i + 1}`,
+        `URL: ${r.link}`,
+        r.snippet != null ? `Summary: ${r.snippet}` : '',
+        r.date != null ? `Date: ${r.date}` : '',
+        r.attribution != null ? `Source: ${r.attribution}` : '',
+        '',
+        '--- Content Highlights ---',
+        ...(r.highlights ?? [])
+          .filter((h) => h.text.trim().length > 0)
+          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\n${h.text.trim()}`),
+        '',
+      ]
+        .filter(Boolean)
+        .join('\n');
+    });
+  }
+
+  // Ignoring these sections for now
+  // // Top stories (news)
+  // const topStores = results.topStories ?? [];
+  // if (topStores.length) {
+  //   addSection('News Results');
+  //   topStores.forEach((r, i) => {
+  //     output += [
+  //       `Anchor: \ue202turn0news${i + 1}`,
+  //       `Title: ${r.title ?? '(no title)'}`,
+  //       `URL: ${r.link}`,
+  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
+  //       r.date != null ? `Date: ${r.date}` : '',
+  //       r.attribution != null ? `Source: ${r.attribution}` : '',
+  //       ''
+  //     ].filter(Boolean).join('\n');
+  //   });
+  // }
+
+  // // Images
+  // const images = results.images ?? [];
+  // if (images.length) {
+  //   addSection('Image Results');
+  //   images.forEach((img, i) => {
+  //     output += [
+  //       `Anchor: \ue202turn0image${i + 1}`,
+  //       `Title: ${img.title ?? '(no title)'}`,
+  //       `Image URL: ${img.imageUrl}`,
+  //       ''
+  //     ].join('\n');
+  //   });
+  // }
+
+  // Knowledge Graph
+  if (results.knowledgeGraph != null) {
+    addSection('Knowledge Graph');
+    output += [
+      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,
+      results.knowledgeGraph.description != null
+        ? `Description: ${results.knowledgeGraph.description}`
+        : '',
+      results.knowledgeGraph.type != null
+        ? `Type: ${results.knowledgeGraph.type}`
+        : '',
+      results.knowledgeGraph.imageUrl != null
+        ? `Image URL: ${results.knowledgeGraph.imageUrl}`
+        : '',
+      results.knowledgeGraph.attributes != null
+        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`
+        : '',
+      '',
+    ]
+      .filter(Boolean)
+      .join('\n');
+  }
+
+  // Answer Box
+  if (results.answerBox != null) {
+    addSection('Answer Box');
+    output += [
+      results.answerBox.title != null
+        ? `Title: ${results.answerBox.title}`
+        : '',
+      results.answerBox.answer != null
+        ? `Answer: ${results.answerBox.answer}`
+        : '',
+      results.answerBox.snippet != null
+        ? `Snippet: ${results.answerBox.snippet}`
+        : '',
+      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',
+      '',
+    ]
+      .filter(Boolean)
+      .join('\n');
+  }
+
+  // People also ask
+  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
+  if (peopleAlsoAsk.length) {
+    addSection('People Also Ask');
+    peopleAlsoAsk.forEach((p, _i) => {
+      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']
+        .filter(Boolean)
+        .join('\n');
+    });
+  }
+
+  return output.trim();
+}