@illuma-ai/agents 1.0.81

package/src/tools/search/content.ts

@@ -0,0 +1,147 @@
+import * as cheerio from 'cheerio';
+import type { References, MediaReference } from './types';
+
+export function processContent(
+  html: string,
+  markdown: string
+): {
+  markdown: string;
+} & References {
+  const linkMap = new Map<string, MediaReference>();
+  const imageMap = new Map<string, MediaReference>();
+  const videoMap = new Map<string, MediaReference>();
+  const iframeMap = new Map<string, MediaReference>();
+
+  const $ = cheerio.load(html, {
+    xmlMode: false,
+  });
+
+  // Extract all media references
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href != null && href) {
+      linkMap.set(href, {
+        originalUrl: href,
+        title: $(el).attr('title'),
+        text: $(el).text().trim(),
+      });
+    }
+  });
+
+  $('img[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (src != null && src) {
+      imageMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('alt') ?? $(el).attr('title'),
+      });
+    }
+  });
+
+  // Handle videos (dedicated video elements and video platforms in iframes)
+  $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each(
+    (_, el) => {
+      const src = $(el).attr('src');
+      if (src != null && src) {
+        videoMap.set(src, {
+          originalUrl: src,
+          title: $(el).attr('title'),
+        });
+      }
+    }
+  );
+
+  // Handle all other generic iframes that aren't already captured as videos
+  $('iframe').each((_, el) => {
+    const src = $(el).attr('src');
+    if (
+      src != null &&
+      src &&
+      !src.includes('youtube') &&
+      !src.includes('vimeo')
+    ) {
+      iframeMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('title'),
+      });
+    }
+  });
+
+  // Create lookup maps with indices
+  const linkIndexMap = new Map<string, number>();
+  const imageIndexMap = new Map<string, number>();
+  const videoIndexMap = new Map<string, number>();
+  const iframeIndexMap = new Map<string, number>();
+
+  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+  Array.from(imageMap.keys()).forEach((url, i) =>
+    imageIndexMap.set(url, i + 1)
+  );
+  Array.from(videoMap.keys()).forEach((url, i) =>
+    videoIndexMap.set(url, i + 1)
+  );
+  Array.from(iframeMap.keys()).forEach((url, i) =>
+    iframeIndexMap.set(url, i + 1)
+  );
+
+  // Process the markdown
+  let result = markdown;
+
+  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+  const allUrls = [
+    ...Array.from(imageMap.keys()).map((url) => ({
+      url,
+      type: 'image',
+      idx: imageIndexMap.get(url),
+    })),
+    ...Array.from(videoMap.keys()).map((url) => ({
+      url,
+      type: 'video',
+      idx: videoIndexMap.get(url),
+    })),
+    ...Array.from(iframeMap.keys()).map((url) => ({
+      url,
+      type: 'iframe',
+      idx: iframeIndexMap.get(url),
+    })),
+    ...Array.from(linkMap.keys()).map((url) => ({
+      url,
+      type: 'link',
+      idx: linkIndexMap.get(url),
+    })),
+  ].sort((a, b) => b.url.length - a.url.length);
+
+  // Create a function to escape special characters in URLs for regex
+  function escapeRegex(string: string): string {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
+  // Replace each URL in the markdown
+  for (const { url, type, idx } of allUrls) {
+    // Create a regex that captures URLs in markdown links
+    const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+
+    result = result.replace(regex, (match) => {
+      // Keep any title attribute that might exist
+      const titleMatch = match.match(/\s+"([^"]*)"/);
+      const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+
+      return `(${type}#${idx}${titlePart})`;
+    });
+  }
+
+  iframeMap.clear();
+  const links = Array.from(linkMap.values());
+  linkMap.clear();
+  const images = Array.from(imageMap.values());
+  imageMap.clear();
+  const videos = Array.from(videoMap.values());
+  videoMap.clear();
+
+  return {
+    markdown: result,
+    links,
+    images,
+    videos,
+  };
+}

package/src/tools/search/firecrawl.ts

@@ -0,0 +1,210 @@
+import axios from 'axios';
+import { processContent } from './content';
+import type * as t from './types';
+import { createDefaultLogger } from './utils';
+
+/**
+ * Firecrawl scraper implementation
+ * Uses the Firecrawl API to scrape web pages
+ */
+export class FirecrawlScraper implements t.BaseScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private version: string;
+  private defaultFormats: string[];
+  private timeout: number;
+  private logger: t.Logger;
+  private includeTags?: string[];
+  private excludeTags?: string[];
+  private waitFor?: number;
+  private maxAge?: number;
+  private mobile?: boolean;
+  private skipTlsVerification?: boolean;
+  private blockAds?: boolean;
+  private removeBase64Images?: boolean;
+  private parsePDF?: boolean;
+  private storeInCache?: boolean;
+  private zeroDataRetention?: boolean;
+  private headers?: Record<string, string>;
+  private location?: { country?: string; languages?: string[] };
+  private onlyMainContent?: boolean;
+  private changeTrackingOptions?: object;
+
+  constructor(config: t.FirecrawlScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
+
+    this.version = config.version ?? 'v2';
+
+    const baseUrl =
+      config.apiUrl ??
+      process.env.FIRECRAWL_BASE_URL ??
+      'https://api.firecrawl.dev';
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/${this.version}/scrape`;
+
+    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
+    this.timeout = config.timeout ?? 7500;
+
+    this.logger = config.logger || createDefaultLogger();
+
+    this.includeTags = config.includeTags;
+    this.excludeTags = config.excludeTags;
+    this.waitFor = config.waitFor;
+    this.maxAge = config.maxAge;
+    this.mobile = config.mobile;
+    this.skipTlsVerification = config.skipTlsVerification;
+    this.blockAds = config.blockAds;
+    this.removeBase64Images = config.removeBase64Images;
+    this.parsePDF = config.parsePDF;
+    this.storeInCache = config.storeInCache;
+    this.zeroDataRetention = config.zeroDataRetention;
+    this.headers = config.headers;
+    this.location = config.location;
+    this.onlyMainContent = config.onlyMainContent;
+    this.changeTrackingOptions = config.changeTrackingOptions;
+
+    if (!this.apiKey) {
+      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+    }
+
+    this.logger.debug(
+      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`
+    );
+  }
+
+  /**
+   * Scrape a single URL
+   * @param url URL to scrape
+   * @param options Scrape options
+   * @returns Scrape response
+   */
+  async scrapeUrl(
+    url: string,
+    options: t.FirecrawlScrapeOptions = {}
+  ): Promise<[string, t.FirecrawlScrapeResponse]> {
+    if (!this.apiKey) {
+      return [
+        url,
+        {
+          success: false,
+          error: 'FIRECRAWL_API_KEY is not set',
+        },
+      ];
+    }
+
+    try {
+      const payload = omitUndefined({
+        url,
+        formats: options.formats ?? this.defaultFormats,
+        includeTags: options.includeTags ?? this.includeTags,
+        excludeTags: options.excludeTags ?? this.excludeTags,
+        headers: options.headers ?? this.headers,
+        waitFor: options.waitFor ?? this.waitFor,
+        timeout: options.timeout ?? this.timeout,
+        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+        maxAge: options.maxAge ?? this.maxAge,
+        mobile: options.mobile ?? this.mobile,
+        skipTlsVerification:
+          options.skipTlsVerification ?? this.skipTlsVerification,
+        parsePDF: options.parsePDF ?? this.parsePDF,
+        location: options.location ?? this.location,
+        removeBase64Images:
+          options.removeBase64Images ?? this.removeBase64Images,
+        blockAds: options.blockAds ?? this.blockAds,
+        storeInCache: options.storeInCache ?? this.storeInCache,
+        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+        changeTrackingOptions:
+          options.changeTrackingOptions ?? this.changeTrackingOptions,
+      });
+      const response = await axios.post(this.apiUrl, payload, {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+        timeout: this.timeout,
+      });
+
+      return [url, response.data];
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return [
+        url,
+        {
+          success: false,
+          error: `Firecrawl API request failed: ${errorMessage}`,
+        },
+      ];
+    }
+  }
+
+  /**
+   * Extract content from scrape response
+   * @param response Scrape response
+   * @returns Extracted content or empty string if not available
+   */
+  extractContent(
+    response: t.FirecrawlScrapeResponse
+  ): [string, undefined | t.References] {
+    if (!response.success || !response.data) {
+      return ['', undefined];
+    }
+
+    if (response.data.markdown != null && response.data.html != null) {
+      try {
+        const { markdown, ...rest } = processContent(
+          response.data.html,
+          response.data.markdown
+        );
+        return [markdown, rest];
+      } catch (error) {
+        this.logger.error('Error processing content:', error);
+        return [response.data.markdown, undefined];
+      }
+    } else if (response.data.markdown != null) {
+      return [response.data.markdown, undefined];
+    }
+
+    // Fall back to HTML content
+    if (response.data.html != null) {
+      return [response.data.html, undefined];
+    }
+
+    // Fall back to raw HTML content
+    if (response.data.rawHtml != null) {
+      return [response.data.rawHtml, undefined];
+    }
+
+    return ['', undefined];
+  }
+
+  /**
+   * Extract metadata from scrape response
+   * @param response Scrape response
+   * @returns Metadata object
+   */
+  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {
+    if (!response.success || !response.data || !response.data.metadata) {
+      return {};
+    }
+
+    return response.data.metadata;
+  }
+}
+
+/**
+ * Create a Firecrawl scraper instance
+ * @param config Scraper configuration
+ * @returns Firecrawl scraper instance
+ */
+export const createFirecrawlScraper = (
+  config: t.FirecrawlScraperConfig = {}
+): FirecrawlScraper => {
+  return new FirecrawlScraper(config);
+};
+
+// Helper function to clean up payload for firecrawl
+function omitUndefined<T extends object>(obj: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([, v]) => v !== undefined)
+  ) as Partial<T>;
+}

package/src/tools/search/format.ts

@@ -0,0 +1,250 @@
+import type * as t from './types';
+import { getDomainName, fileExtRegex } from './utils';
+
+function addHighlightSection(): string[] {
+  return ['\n## Highlights', ''];
+}
+
+// Helper function to format a source (organic or top story)
+function formatSource(
+  source: t.ValidSource,
+  index: number,
+  turn: number,
+  sourceType: 'search' | 'news',
+  references: t.ResultReference[]
+): string {
+  /** Array of all lines to include in the output */
+  const outputLines: string[] = [];
+
+  // Add the title
+  outputLines.push(
+    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`
+  );
+  outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+  outputLines.push(`URL: ${source.link}`);
+
+  // Add optional fields
+  if ('snippet' in source && source.snippet != null) {
+    outputLines.push(`Summary: ${source.snippet}`);
+  }
+
+  if (source.date != null) {
+    outputLines.push(`Date: ${source.date}`);
+  }
+
+  if (source.attribution != null) {
+    outputLines.push(`Source: ${source.attribution}`);
+  }
+
+  // Add highlight section or empty line
+  if ((source.highlights?.length ?? 0) > 0) {
+    outputLines.push(...addHighlightSection());
+  } else {
+    outputLines.push('');
+  }
+
+  // Process highlights if they exist
+  (source.highlights ?? [])
+    .filter((h) => h.text.trim().length > 0)
+    .forEach((h, hIndex) => {
+      outputLines.push(
+        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`
+      );
+      outputLines.push('');
+      outputLines.push('```text');
+      outputLines.push(h.text.trim());
+      outputLines.push('```');
+      outputLines.push('');
+
+      if (h.references != null && h.references.length) {
+        let hasHeader = false;
+        const refLines: string[] = [];
+
+        for (let j = 0; j < h.references.length; j++) {
+          const ref = h.references[j];
+          if (ref.reference.originalUrl.includes('mailto:')) {
+            continue;
+          }
+          if (ref.type !== 'link') {
+            continue;
+          }
+          if (fileExtRegex.test(ref.reference.originalUrl)) {
+            continue;
+          }
+          references.push({
+            type: ref.type,
+            link: ref.reference.originalUrl,
+            attribution: getDomainName(ref.reference.originalUrl),
+            title: (
+              ((ref.reference.title ?? '') || ref.reference.text) ??
+              ''
+            ).split('\n')[0],
+          });
+
+          if (!hasHeader) {
+            refLines.push('Core References:');
+            hasHeader = true;
+          }
+
+          refLines.push(
+            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`
+          );
+          refLines.push(
+            `\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`
+          );
+        }
+
+        if (hasHeader) {
+          outputLines.push(...refLines);
+          outputLines.push('');
+        }
+      }
+
+      if (hIndex < (source.highlights?.length ?? 0) - 1) {
+        outputLines.push('---');
+        outputLines.push('');
+      }
+    });
+
+  outputLines.push('');
+  return outputLines.join('\n');
+}
+
+export function formatResultsForLLM(
+  turn: number,
+  results: t.SearchResultData
+): { output: string; references: t.ResultReference[] } {
+  /** Array to collect all output lines */
+  const outputLines: string[] = [];
+
+  const addSection = (title: string): void => {
+    outputLines.push('');
+    outputLines.push(`=== ${title} ===`);
+    outputLines.push('');
+  };
+
+  const references: t.ResultReference[] = [];
+
+  // Organic (web) results
+  if (results.organic?.length != null && results.organic.length > 0) {
+    addSection(`Web Results, Turn ${turn}`);
+    for (let i = 0; i < results.organic.length; i++) {
+      const r = results.organic[i];
+      outputLines.push(formatSource(r, i, turn, 'search', references));
+      delete results.organic[i].highlights;
+    }
+  }
+
+  // Top stories (news)
+  const topStories = results.topStories ?? [];
+  if (topStories.length) {
+    addSection('News Results');
+    for (let i = 0; i < topStories.length; i++) {
+      const r = topStories[i];
+      outputLines.push(formatSource(r, i, turn, 'news', references));
+      if (results.topStories?.[i]?.highlights) {
+        delete results.topStories[i].highlights;
+      }
+    }
+  }
+
+  // // Images
+  // const images = results.images ?? [];
+  // if (images.length) {
+  //   addSection('Image Results');
+  //   const imageLines = images.map((img, i) => [
+  //     `Anchor: \ue202turn0image${i}`,
+  //     `Title: ${img.title ?? '(no title)'}`,
+  //     `Image URL: ${img.imageUrl}`,
+  //     ''
+  //   ].join('\n'));
+  //   outputLines.push(imageLines.join('\n'));
+  // }
+
+  // Knowledge Graph
+  if (results.knowledgeGraph != null) {
+    addSection('Knowledge Graph');
+    const kgLines = [
+      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+      results.knowledgeGraph.type != null
+        ? `**Type:** ${results.knowledgeGraph.type}`
+        : '',
+      results.knowledgeGraph.description != null
+        ? `**Description:** ${results.knowledgeGraph.description}`
+        : '',
+      results.knowledgeGraph.descriptionSource != null
+        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+        : '',
+      results.knowledgeGraph.descriptionLink != null
+        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
+        : '',
+      results.knowledgeGraph.imageUrl != null
+        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+        : '',
+      results.knowledgeGraph.website != null
+        ? `**Website:** ${results.knowledgeGraph.website}`
+        : '',
+      results.knowledgeGraph.attributes != null
+        ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(
+          results.knowledgeGraph.attributes,
+          null,
+          2
+        )}\n\`\`\``
+        : '',
+      '',
+    ].filter(Boolean);
+
+    outputLines.push(kgLines.join('\n\n'));
+  }
+
+  // Answer Box
+  if (results.answerBox != null) {
+    addSection('Answer Box');
+    const abLines = [
+      results.answerBox.title != null
+        ? `**Title:** ${results.answerBox.title}`
+        : '',
+      results.answerBox.snippet != null
+        ? `**Snippet:** ${results.answerBox.snippet}`
+        : '',
+      results.answerBox.snippetHighlighted != null
+        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+          .map((s) => `\`${s}\``)
+          .join(' ')}`
+        : '',
+      results.answerBox.link != null
+        ? `**Link:** ${results.answerBox.link}`
+        : '',
+      '',
+    ].filter(Boolean);
+
+    outputLines.push(abLines.join('\n\n'));
+  }
+
+  // People also ask
+  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
+  if (peopleAlsoAsk.length) {
+    addSection('People Also Ask');
+
+    const paaLines: string[] = [];
+    peopleAlsoAsk.forEach((p, i) => {
+      const questionLines = [
+        `### Question ${i + 1}:`,
+        `"${p.question}"`,
+        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
+        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+        '',
+      ].filter(Boolean);
+
+      paaLines.push(questionLines.join('\n\n'));
+    });
+
+    outputLines.push(paaLines.join(''));
+  }
+
+  return {
+    output: outputLines.join('\n').trim(),
+    references,
+  };
+}