@librechat/agents 3.2.63 → 3.2.65

package/src/tools/search/crw-scraper.ts

@@ -0,0 +1,244 @@
+import axios from 'axios';
+import type * as t from './types';
+import { createDefaultLogger } from './utils';
+import { processContent } from './content';
+
+/** HTTP headroom over the payload render budget: fastCRW's queue/verification
+ * overhead is not counted against `timeout`, so the client must wait longer. */
+const CRW_TIMEOUT_BUFFER = 5000;
+
+/**
+ * fastCRW scraper. Firecrawl-compatible web scraper; single binary;
+ * self-host or cloud. Posts to {base}/v1/scrape.
+ */
+export class CrwScraper implements t.BaseScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private defaultFormats: string[];
+  private timeout: number;
+  private logger: t.Logger;
+  private onlyMainContent?: boolean;
+  private includeTags?: string[];
+  private excludeTags?: string[];
+  private waitFor?: number;
+  private headers?: Record<string, string>;
+  private renderJs?: boolean | null;
+  private cssSelector?: string;
+  private xpath?: string;
+  private proxy?: string;
+  private stealth?: boolean;
+
+  constructor(config: t.CrwScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';
+
+    const baseUrl =
+      config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
+
+    this.defaultFormats = config.formats ?? ['markdown', 'html'];
+    this.timeout = config.timeout ?? 7500;
+    this.logger = config.logger || createDefaultLogger();
+
+    this.onlyMainContent = config.onlyMainContent;
+    this.includeTags = config.includeTags;
+    this.excludeTags = config.excludeTags;
+    this.waitFor = config.waitFor;
+    this.headers = config.headers;
+    this.renderJs = config.renderJs;
+    this.cssSelector = config.cssSelector;
+    this.xpath = config.xpath;
+    this.proxy = config.proxy;
+    this.stealth = config.stealth;
+
+    // Self-host fastCRW may run without auth, so a missing key is only a
+    // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.
+    if (!this.apiKey) {
+      this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');
+    }
+    this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);
+  }
+
+  async scrapeUrl(
+    url: string,
+    options: t.CrwScrapeOptions = {}
+  ): Promise<[string, t.CrwScrapeResponse]> {
+    try {
+      const payloadTimeout = options.timeout ?? this.timeout;
+      const payload = omitUndefined({
+        url,
+        formats: options.formats ?? this.defaultFormats,
+        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+        includeTags: options.includeTags ?? this.includeTags,
+        excludeTags: options.excludeTags ?? this.excludeTags,
+        waitFor: options.waitFor ?? this.waitFor,
+        headers: options.headers ?? this.headers,
+        renderJs: options.renderJs ?? this.renderJs,
+        cssSelector: options.cssSelector ?? this.cssSelector,
+        xpath: options.xpath ?? this.xpath,
+        proxy: options.proxy ?? this.proxy,
+        stealth: options.stealth ?? this.stealth,
+        // Cloud honors `timeout` (live-verified); the published OpenAPI
+        // documents `deadlineMs` (1..60000) instead. Send both.
+        timeout: payloadTimeout,
+        deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),
+      });
+
+      const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+      };
+      if (this.apiKey) {
+        headers.Authorization = `Bearer ${this.apiKey}`;
+      }
+
+      const response = await axios.post<t.CrwRawScrapeResponse>(
+        this.apiUrl,
+        payload,
+        {
+          headers,
+          timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,
+        }
+      );
+
+      return [url, normalizeCrwResponse(response.data)];
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return [
+        url,
+        {
+          success: false,
+          error: `fastCRW API request failed: ${errorMessage}`,
+        },
+      ];
+    }
+  }
+
+  /**
+   * Extract content from scrape response. Mirrors FirecrawlScraper — reads
+   * response.data.*, which normalizeCrwResponse guarantees, preserving the
+   * processContent ref-markers used by the reranker. Parameter is typed as the
+   * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:
+   * TS class-method bivariance accepts this against BaseScraper's
+   * AnyScraperResponse, AND it lets us read response.data.plainText (a
+   * CrwScrapeResponse-only field) with no cast.
+   */
+  extractContent(
+    response: t.CrwScrapeResponse
+  ): [string, undefined | t.References] {
+    if (!response.success || !response.data) {
+      return ['', undefined];
+    }
+
+    const htmlSource = response.data.html ?? response.data.rawHtml;
+    if (response.data.markdown != null && htmlSource != null) {
+      try {
+        const { markdown, ...rest } = processContent(
+          htmlSource,
+          response.data.markdown
+        );
+        return [markdown, rest];
+      } catch (error) {
+        this.logger.error('Error processing content:', error);
+        return [response.data.markdown, undefined];
+      }
+    } else if (response.data.markdown != null) {
+      return [response.data.markdown, undefined];
+    }
+
+    // Fall back to HTML content
+    if (response.data.html != null) {
+      return [response.data.html, undefined];
+    }
+
+    // Fall back to raw HTML content
+    if (response.data.rawHtml != null) {
+      return [response.data.rawHtml, undefined];
+    }
+
+    // CRW-only fallback (no Firecrawl equivalent): plain-text body.
+    if (response.data.plainText != null) {
+      return [response.data.plainText, undefined];
+    }
+
+    return ['', undefined];
+  }
+
+  extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {
+    if (!response.success || !response.data || !response.data.metadata) {
+      return {};
+    }
+
+    return response.data.metadata;
+  }
+}
+
+/**
+ * Create a fastCRW scraper instance
+ * @param config Scraper configuration
+ * @returns fastCRW scraper instance
+ */
+export const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>
+  new CrwScraper(config);
+
+/**
+ * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,
+ * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested
+ * container and fall back to top-level fields for self-host/legacy responses.
+ */
+function normalizeCrwResponse(
+  raw: t.CrwRawScrapeResponse | null | undefined
+): t.CrwScrapeResponse {
+  if (raw == null) {
+    return { success: false, error: 'Empty fastCRW response' };
+  }
+  if (raw.success === false) {
+    return {
+      success: false,
+      error:
+        raw.error_code != null
+          ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`
+          : (raw.error ?? 'fastCRW scrape failed'),
+      error_code: raw.error_code,
+    };
+  }
+  const data = raw.data ?? raw;
+  return {
+    success: true,
+    data: {
+      // Strip inline base64 image payloads from text content (not `screenshot`,
+      // which is base64 by design and never reaches the content processor).
+      markdown: stripBase64DataUris(data.markdown),
+      html: stripBase64DataUris(data.html),
+      rawHtml: stripBase64DataUris(data.rawHtml),
+      plainText: stripBase64DataUris(data.plainText),
+      screenshot: data.screenshot,
+      links: data.links,
+      metadata: data.metadata,
+    },
+  };
+}
+
+/**
+ * Replace inline base64 data-URI payloads (typically images) in scraped text
+ * with a short placeholder. Such payloads can be hundreds of KB; the content
+ * processor builds a per-link RegExp from each URL, and one that large overflows
+ * the engine's pattern-size limit ("Invalid regular expression"). Firecrawl
+ * drops them via removeBase64Images — mirror that so image-heavy pages stay
+ * processable (and don't bloat the LLM context with base64 noise).
+ */
+function stripBase64DataUris(text: string | undefined): string | undefined {
+  if (text == null) {
+    return text;
+  }
+  return text.replace(
+    /data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]+/g,
+    'data:base64-content-removed'
+  );
+}
+
+// Helper function to clean up payload for fastCRW
+function omitUndefined<T extends object>(obj: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([, v]) => v !== undefined)
+  ) as Partial<T>;
+}

package/src/tools/search/crw-search.ts

@@ -0,0 +1,167 @@
+import axios from 'axios';
+import type * as t from './types';
+
+const DEFAULT_CRW_TIMEOUT = 15000;
+
+const getHostname = (link: string): string => {
+  try {
+    return new URL(link).hostname;
+  } catch {
+    return link;
+  }
+};
+
+/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat
+ * SearchResult[] with a `category` field; route rows into source groups. */
+const splitByCategory = (rows: t.CrwSearchResult[]): t.CrwSearchGroups => {
+  const groups: Required<t.CrwSearchGroups> = { web: [], images: [], news: [] };
+  for (const row of rows) {
+    if (row.category === 'news') {
+      groups.news.push(row);
+    } else if (row.category === 'images') {
+      groups.images.push({ ...row, imageUrl: row.url });
+    } else {
+      groups.web.push(row);
+    }
+  }
+  return groups;
+};
+
+export const createCrwAPI = (
+  apiKey?: string,
+  apiUrl?: string,
+  options?: t.CrwSearchOptions
+): {
+  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
+} => {
+  // NOTE: fastCRW /v1/search is cloud-only, but we still allow a base-URL
+  // override for parity with the scraper. Self-host may have no auth, so —
+  // unlike Tavily (createTavilyAPI throws on missing key) — we do NOT throw
+  // here; we only attach the Authorization header when a key is present.
+  const base = (
+    apiUrl ??
+    process.env.CRW_API_URL ??
+    'https://api.fastcrw.com'
+  ).replace(/\/+$/, '');
+  const config = {
+    apiKey: apiKey ?? process.env.CRW_API_KEY,
+    apiUrl: `${base}/v1/search`,
+    timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT,
+  };
+
+  const getSources = async ({
+    query,
+    date,
+    numResults = 8,
+    type,
+  }: t.GetSourcesParams): Promise<t.SearchResult> => {
+    if (!query.trim()) {
+      return { success: false, error: 'Query cannot be empty' };
+    }
+
+    try {
+      const limit = Math.min(
+        Math.max(1, options?.maxResults ?? numResults),
+        20
+      );
+      // Mirror Serper's verticals: image/news requests hit only their native
+      // fastCRW source; plain web optionally adds images via includeImages.
+      let sources: t.CrwSearchSource[];
+      if (type === 'images') {
+        sources = ['images'];
+      } else if (type === 'news') {
+        sources = ['news'];
+      } else {
+        sources = options?.includeImages === true ? ['web', 'images'] : ['web'];
+      }
+
+      const payload: t.CrwSearchPayload = { query, limit, sources };
+      if (date != null) {
+        // Serper-style qdr filter; live-verified to constrain results.
+        payload.tbs = `qdr:${date}`;
+      }
+
+      const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+      };
+      if (config.apiKey != null && config.apiKey !== '') {
+        headers.Authorization = `Bearer ${config.apiKey}`;
+      }
+
+      const response = await axios.post<t.CrwSearchResponse>(
+        config.apiUrl,
+        payload,
+        { headers, timeout: config.timeout }
+      );
+
+      const body = response.data;
+      if (body.success === false) {
+        return {
+          success: false,
+          error: `fastCRW search failed: ${
+            body.error_code != null ? `[${body.error_code}] ` : ''
+          }${body.error ?? 'Unknown error'}`,
+        };
+      }
+
+      // fastCRW cloud keys results by source: {data: {web: [...], images:
+      // [...], news: [...]}} (live-verified 2026-07-02); the published
+      // OpenAPI documents a flat SearchResult[] instead (self-host), and a
+      // {data: {results: {...groups}}} wrapper also exists. Accept all three.
+      // OrganicResult.link is a REQUIRED string (types.ts), so defend against
+      // null/empty urls to never feed a broken '' link into the scraper.
+      const container = body.data ?? {};
+      const data: t.CrwSearchGroups = Array.isArray(container)
+        ? splitByCategory(container)
+        : (container.results ?? container);
+      const organicResults: t.OrganicResult[] = (data.web ?? [])
+        .filter((r) => r.url != null && r.url !== '')
+        .map((r) => ({
+          title: r.title ?? '',
+          link: r.url as string,
+          snippet: r.description ?? r.snippet ?? '',
+          position: r.position,
+        }));
+
+      const imageResults: t.ImageResult[] = (data.images ?? [])
+        .filter((r) => r.imageUrl != null && r.imageUrl !== '')
+        .map((r) => ({
+          title: r.title,
+          imageUrl: r.imageUrl,
+          link: r.url,
+          position: r.position,
+        }));
+
+      const newsResults: t.NewsResult[] = (data.news ?? [])
+        .filter((r) => r.url != null && r.url !== '')
+        .map((r) => ({
+          title: r.title ?? '',
+          link: r.url as string,
+          snippet: r.description ?? r.snippet ?? '',
+          date: r.publishedDate,
+          source: getHostname(r.url as string),
+          position: r.position,
+        }));
+
+      const results: t.SearchResultData = {
+        organic: organicResults,
+        images: imageResults,
+        topStories: [],
+        videos: [],
+        news: newsResults,
+        relatedSearches: [],
+      };
+
+      return { success: true, data: results };
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return {
+        success: false,
+        error: `fastCRW search request failed: ${errorMessage}`,
+      };
+    }
+  };
+
+  return { getSources };
+};