@illuma-ai/agents 1.5.1 → 2.1.1

package/src/tools/search/tavily-scraper.ts

@@ -1,235 +0,0 @@
-import axios from 'axios';
-import type * as t from './types';
-import { createDefaultLogger } from './utils';
-
-const DEFAULT_BASIC_TIMEOUT = 15000;
-const DEFAULT_ADVANCED_TIMEOUT = 30000;
-const MAX_BATCH_SIZE = 20;
-
-const getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>
-  extractDepth === 'advanced'
-    ? DEFAULT_ADVANCED_TIMEOUT
-    : DEFAULT_BASIC_TIMEOUT;
-
-const normalizeUrlKey = (url: string): string => {
-  try {
-    const parsedUrl = new URL(url);
-    parsedUrl.hash = '';
-    if (parsedUrl.pathname.length > 1) {
-      parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '');
-    }
-    return parsedUrl.toString();
-  } catch {
-    return url;
-  }
-};
-
-const setUrlResult = (
-  map: Map<string, t.TavilyExtractResult>,
-  result: t.TavilyExtractResult
-): void => {
-  map.set(result.url, result);
-  const normalizedUrl = normalizeUrlKey(result.url);
-  if (!map.has(normalizedUrl)) {
-    map.set(normalizedUrl, result);
-  }
-};
-
-export class TavilyScraper implements t.BaseScraper {
-  private apiKey: string;
-  private apiUrl: string;
-  private timeout: number;
-  private payloadTimeout: number | undefined;
-  private logger: t.Logger;
-  private extractDepth: 'basic' | 'advanced';
-  private includeImages: boolean;
-  private includeFavicon: boolean;
-  private format: 'markdown' | 'text' | undefined;
-
-  constructor(config: t.TavilyScraperConfig = {}) {
-    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';
-    this.apiUrl =
-      config.apiUrl ??
-      process.env.TAVILY_EXTRACT_URL ??
-      'https://api.tavily.com/extract';
-    this.payloadTimeout = config.timeout;
-    this.extractDepth = config.extractDepth ?? 'basic';
-    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
-    this.includeImages = config.includeImages ?? false;
-    this.includeFavicon = config.includeFavicon ?? false;
-    this.format = config.format;
-    this.logger = config.logger || createDefaultLogger();
-
-    if (!this.apiKey) {
-      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');
-    }
-  }
-
-  async scrapeUrl(
-    url: string,
-    options: t.TavilyScrapeOptions = {}
-  ): Promise<[string, t.TavilyScrapeResponse]> {
-    const results = await this.scrapeUrls([url], options);
-    return results[0];
-  }
-
-  async scrapeUrls(
-    urls: string[],
-    options: t.TavilyScrapeOptions = {}
-  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {
-    if (!this.apiKey) {
-      return urls.map((url) => [
-        url,
-        { success: false, error: 'TAVILY_API_KEY is not set' },
-      ]);
-    }
-
-    const batches: string[][] = [];
-    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {
-      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
-    }
-
-    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];
-
-    for (const batch of batches) {
-      const batchResults = await this.extractBatch(batch, options);
-      allResults.push(...batchResults);
-    }
-
-    return allResults;
-  }
-
-  private async extractBatch(
-    urls: string[],
-    options: t.TavilyScrapeOptions = {}
-  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {
-    try {
-      const includeFavicon = options.includeFavicon ?? this.includeFavicon;
-      const format = options.format ?? this.format;
-      const extractDepth = options.extractDepth ?? this.extractDepth;
-      const payload: t.TavilyExtractPayload = {
-        urls,
-        extract_depth: extractDepth,
-        include_images: options.includeImages ?? this.includeImages,
-      };
-
-      if (includeFavicon) {
-        payload.include_favicon = true;
-      }
-      if (format != null) {
-        payload.format = format;
-      }
-
-      const effectiveTimeout =
-        options.timeout ??
-        this.payloadTimeout ??
-        (options.extractDepth != null
-          ? getDefaultTimeout(extractDepth)
-          : this.timeout);
-      const payloadTimeout = options.timeout ?? this.payloadTimeout;
-      if (payloadTimeout != null) {
-        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);
-      }
-
-      const response = await axios.post<{
-        results?: t.TavilyExtractResult[];
-        failed_results?: t.TavilyExtractResult[];
-      }>(this.apiUrl, payload, {
-        headers: {
-          Authorization: `Bearer ${this.apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        timeout: effectiveTimeout,
-      });
-
-      const data = response.data;
-      const successMap = new Map<string, t.TavilyExtractResult>();
-      const failedMap = new Map<string, t.TavilyExtractResult>();
-
-      for (const result of data.results ?? []) {
-        setUrlResult(successMap, result);
-      }
-      for (const result of data.failed_results ?? []) {
-        setUrlResult(failedMap, result);
-      }
-
-      return urls.map((url): [string, t.TavilyScrapeResponse] => {
-        const success =
-          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
-        if (success && success.error == null) {
-          return [
-            url,
-            {
-              success: true,
-              data: {
-                rawContent: success.raw_content ?? '',
-                images: success.images ?? [],
-                favicon: success.favicon,
-              },
-            },
-          ];
-        }
-
-        const failed =
-          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
-        const error =
-          success?.error ??
-          failed?.error ??
-          'URL not found in Tavily Extract response';
-        return [url, { success: false, error }];
-      });
-    } catch (error) {
-      const errorMessage =
-        error instanceof Error ? error.message : String(error);
-      return urls.map((url) => [
-        url,
-        {
-          success: false,
-          error: `Tavily Extract API request failed: ${errorMessage}`,
-        },
-      ]);
-    }
-  }
-
-  extractContent(
-    response: t.TavilyScrapeResponse
-  ): [string, undefined | t.References] {
-    if (!response.success || !response.data) {
-      return ['', undefined];
-    }
-
-    const content = response.data.rawContent ?? '';
-    const images = response.data.images ?? [];
-
-    const references: t.References | undefined =
-      images.length > 0
-        ? {
-            links: [],
-            images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
-            videos: [],
-          }
-        : undefined;
-
-    return [content, references];
-  }
-
-  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {
-    if (!response.success || !response.data) {
-      return {};
-    }
-
-    const metadata: t.GenericScrapeMetadata = {
-      images_count: response.data.images?.length ?? 0,
-    };
-    if (response.data.favicon != null) {
-      metadata.favicon = response.data.favicon;
-    }
-    return metadata;
-  }
-}
-
-export const createTavilyScraper = (
-  config: t.TavilyScraperConfig = {}
-): TavilyScraper => {
-  return new TavilyScraper(config);
-};

package/src/tools/search/tavily-search.ts

@@ -1,424 +0,0 @@
-import axios from 'axios';
-import type * as t from './types';
-
-const DEFAULT_TAVILY_TIMEOUT = 15000;
-
-const TAVILY_COUNTRY_ALIASES: Record<string, string> = {
-  ba: 'bosnia and herzegovina',
-  cg: 'congo',
-  cz: 'czech republic',
-  gb: 'united kingdom',
-  mm: 'myanmar',
-  tr: 'turkey',
-  tt: 'trinidad and tobago',
-  uk: 'united kingdom',
-};
-
-const TAVILY_SUPPORTED_COUNTRIES = new Set([
-  'afghanistan',
-  'albania',
-  'algeria',
-  'andorra',
-  'angola',
-  'argentina',
-  'armenia',
-  'australia',
-  'austria',
-  'azerbaijan',
-  'bahamas',
-  'bahrain',
-  'bangladesh',
-  'barbados',
-  'belarus',
-  'belgium',
-  'belize',
-  'benin',
-  'bhutan',
-  'bolivia',
-  'bosnia and herzegovina',
-  'botswana',
-  'brazil',
-  'brunei',
-  'bulgaria',
-  'burkina faso',
-  'burundi',
-  'cambodia',
-  'cameroon',
-  'canada',
-  'cape verde',
-  'central african republic',
-  'chad',
-  'chile',
-  'china',
-  'colombia',
-  'comoros',
-  'congo',
-  'costa rica',
-  'croatia',
-  'cuba',
-  'cyprus',
-  'czech republic',
-  'denmark',
-  'djibouti',
-  'dominican republic',
-  'ecuador',
-  'egypt',
-  'el salvador',
-  'equatorial guinea',
-  'eritrea',
-  'estonia',
-  'ethiopia',
-  'fiji',
-  'finland',
-  'france',
-  'gabon',
-  'gambia',
-  'georgia',
-  'germany',
-  'ghana',
-  'greece',
-  'guatemala',
-  'guinea',
-  'haiti',
-  'honduras',
-  'hungary',
-  'iceland',
-  'india',
-  'indonesia',
-  'iran',
-  'iraq',
-  'ireland',
-  'israel',
-  'italy',
-  'jamaica',
-  'japan',
-  'jordan',
-  'kazakhstan',
-  'kenya',
-  'kuwait',
-  'kyrgyzstan',
-  'latvia',
-  'lebanon',
-  'lesotho',
-  'liberia',
-  'libya',
-  'liechtenstein',
-  'lithuania',
-  'luxembourg',
-  'madagascar',
-  'malawi',
-  'malaysia',
-  'maldives',
-  'mali',
-  'malta',
-  'mauritania',
-  'mauritius',
-  'mexico',
-  'moldova',
-  'monaco',
-  'mongolia',
-  'montenegro',
-  'morocco',
-  'mozambique',
-  'myanmar',
-  'namibia',
-  'nepal',
-  'netherlands',
-  'new zealand',
-  'nicaragua',
-  'niger',
-  'nigeria',
-  'north korea',
-  'north macedonia',
-  'norway',
-  'oman',
-  'pakistan',
-  'panama',
-  'papua new guinea',
-  'paraguay',
-  'peru',
-  'philippines',
-  'poland',
-  'portugal',
-  'qatar',
-  'romania',
-  'russia',
-  'rwanda',
-  'saudi arabia',
-  'senegal',
-  'serbia',
-  'singapore',
-  'slovakia',
-  'slovenia',
-  'somalia',
-  'south africa',
-  'south korea',
-  'south sudan',
-  'spain',
-  'sri lanka',
-  'sudan',
-  'sweden',
-  'switzerland',
-  'syria',
-  'taiwan',
-  'tajikistan',
-  'tanzania',
-  'thailand',
-  'togo',
-  'trinidad and tobago',
-  'tunisia',
-  'turkey',
-  'turkmenistan',
-  'uganda',
-  'ukraine',
-  'united arab emirates',
-  'united kingdom',
-  'united states',
-  'uruguay',
-  'uzbekistan',
-  'venezuela',
-  'vietnam',
-  'yemen',
-  'zambia',
-  'zimbabwe',
-]);
-
-const TAVILY_REGION_NAMES = new Intl.DisplayNames(['en'], {
-  type: 'region',
-});
-
-const normalizeTavilyCountryName = (country: string): string =>
-  country
-    .toLowerCase()
-    .replace(/\s*\([^)]*\)/g, '')
-    .replace(/\s*&\s*/g, ' and ')
-    .normalize('NFD')
-    .replace(/\p{Diacritic}/gu, '');
-
-const normalizeTavilyCountry = (country?: string): string | undefined => {
-  const normalizedCountry = country?.trim().toLowerCase();
-  if (normalizedCountry == null || normalizedCountry === '') {
-    return undefined;
-  }
-
-  const countryAlias = TAVILY_COUNTRY_ALIASES[normalizedCountry];
-  if (countryAlias != null) {
-    return TAVILY_SUPPORTED_COUNTRIES.has(countryAlias)
-      ? countryAlias
-      : undefined;
-  }
-
-  if (/^[a-z]{2}$/.test(normalizedCountry)) {
-    const regionName = TAVILY_REGION_NAMES.of(normalizedCountry.toUpperCase());
-    if (regionName == null) {
-      return undefined;
-    }
-    const countryName = normalizeTavilyCountryName(regionName);
-    return TAVILY_SUPPORTED_COUNTRIES.has(countryName)
-      ? countryName
-      : undefined;
-  }
-
-  const countryName = normalizeTavilyCountryName(normalizedCountry);
-  return TAVILY_SUPPORTED_COUNTRIES.has(countryName) ? countryName : undefined;
-};
-
-const normalizeTavilyTimeRange = (
-  timeRange?: t.TavilyTimeRangeInput
-): t.TavilyTimeRange | undefined => {
-  switch (timeRange) {
-    case 'h':
-    case 'd':
-      return 'day';
-    case 'w':
-      return 'week';
-    case 'm':
-      return 'month';
-    case 'y':
-      return 'year';
-    case 'day':
-    case 'week':
-    case 'month':
-    case 'year':
-      return timeRange;
-    default:
-      return undefined;
-  }
-};
-
-const getHostname = (link: string): string => {
-  try {
-    return new URL(link).hostname;
-  } catch {
-    return link;
-  }
-};
-
-export const createTavilyAPI = (
-  apiKey?: string,
-  apiUrl?: string,
-  options?: t.TavilySearchOptions
-): {
-  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
-} => {
-  const config = {
-    apiKey: apiKey ?? process.env.TAVILY_API_KEY,
-    apiUrl:
-      apiUrl ??
-      process.env.TAVILY_SEARCH_URL ??
-      'https://api.tavily.com/search',
-    timeout: options?.timeout ?? DEFAULT_TAVILY_TIMEOUT,
-  };
-
-  if (config.apiKey == null || config.apiKey === '') {
-    throw new Error('TAVILY_API_KEY is required for Tavily API');
-  }
-
-  const getSources = async ({
-    query,
-    date,
-    country,
-    numResults = 8,
-    type,
-    news,
-  }: t.GetSourcesParams): Promise<t.SearchResult> => {
-    if (!query.trim()) {
-      return { success: false, error: 'Query cannot be empty' };
-    }
-
-    try {
-      const timeRange =
-        normalizeTavilyTimeRange(options?.timeRange) ??
-        (date != null ? (normalizeTavilyTimeRange(date) ?? 'day') : undefined);
-      const topic =
-        news === true || type === 'news'
-          ? 'news'
-          : (options?.topic ?? 'general');
-      const maxResults = options?.maxResults ?? numResults;
-      const searchDepth = options?.searchDepth ?? 'basic';
-
-      const payload: t.TavilySearchPayload = {
-        query,
-        search_depth: searchDepth,
-        topic,
-        max_results: Math.min(Math.max(1, maxResults), 20),
-      };
-
-      if (
-        options?.safeSearch != null &&
-        searchDepth !== 'fast' &&
-        searchDepth !== 'ultra-fast'
-      ) {
-        payload.safe_search = options.safeSearch;
-      }
-      if (timeRange != null) {
-        payload.time_range = timeRange;
-      }
-      const tavilyCountry =
-        topic === 'general' ? normalizeTavilyCountry(country) : undefined;
-      if (tavilyCountry != null) {
-        payload.country = tavilyCountry;
-      }
-      if (type === 'images' || options?.includeImages) {
-        payload.include_images = true;
-      }
-      if (options?.includeAnswer != null) {
-        payload.include_answer = options.includeAnswer;
-      }
-      if (options?.includeRawContent != null) {
-        payload.include_raw_content = options.includeRawContent;
-      }
-      if (
-        options?.includeDomains != null &&
-        options.includeDomains.length > 0
-      ) {
-        payload.include_domains = options.includeDomains;
-      }
-      if (
-        options?.excludeDomains != null &&
-        options.excludeDomains.length > 0
-      ) {
-        payload.exclude_domains = options.excludeDomains;
-      }
-      if (options?.includeImageDescriptions != null) {
-        payload.include_image_descriptions = options.includeImageDescriptions;
-      }
-      if (options?.includeFavicon != null) {
-        payload.include_favicon = options.includeFavicon;
-      }
-      if (options?.chunksPerSource != null && searchDepth === 'advanced') {
-        payload.chunks_per_source = options.chunksPerSource;
-      }
-
-      const response = await axios.post<t.TavilySearchResponse>(
-        config.apiUrl,
-        payload,
-        {
-          headers: {
-            Authorization: `Bearer ${config.apiKey}`,
-            'Content-Type': 'application/json',
-          },
-          timeout: config.timeout,
-        }
-      );
-
-      const data = response.data;
-
-      const organicResults: t.OrganicResult[] = (data.results ?? []).map(
-        (result: t.TavilySearchResult) => ({
-          title: result.title ?? '',
-          link: result.url ?? '',
-          snippet: result.content ?? '',
-          date: result.published_date,
-        })
-      );
-
-      const imageResults: t.ImageResult[] = Array.isArray(data.images)
-        ? data.images.slice(0, 6).reduce<t.ImageResult[]>((acc, image) => {
-            const imageUrl = typeof image === 'string' ? image : image.url;
-            if (imageUrl == null || imageUrl === '') {
-              return acc;
-            }
-            acc.push({
-              imageUrl,
-              title: typeof image === 'string' ? undefined : image.description,
-              position: acc.length + 1,
-            });
-            return acc;
-          }, [])
-        : [];
-
-      const newsResults: t.NewsResult[] =
-        topic === 'news'
-          ? organicResults.map((r) => ({
-              title: r.title,
-              link: r.link,
-              snippet: r.snippet,
-              date: r.date,
-              source: getHostname(r.link),
-            }))
-          : [];
-
-      const results: t.SearchResultData = {
-        organic: organicResults,
-        images: imageResults,
-        topStories: [],
-        videos: [],
-        news: newsResults,
-        answerBox: data.answer != null ? { snippet: data.answer } : undefined,
-        relatedSearches: [],
-      };
-
-      return { success: true, data: results };
-    } catch (error) {
-      const errorMessage =
-        error instanceof Error ? error.message : String(error);
-      return {
-        success: false,
-        error: `Tavily API request failed: ${errorMessage}`,
-      };
-    }
-  };
-
-  return { getSources };
-};