@elizaos/plugin-research 0.1.0

package/src/integrations/search-providers/npm.ts

@@ -0,0 +1,330 @@
+import axios, { AxiosError } from 'axios';
+import { SearchResult } from '../../types';
+import { elizaLogger } from '@elizaos/core';
+import { z } from 'zod';
+
+// NPM Registry API response schema validation
+const NPMPackageSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  version: z.string(),
+  keywords: z.array(z.string()).optional(),
+  author: z.union([
+    z.string(),
+    z.object({
+      name: z.string(),
+      email: z.string().optional(),
+    })
+  ]).optional(),
+  maintainers: z.array(z.object({
+    name: z.string(),
+    email: z.string().optional(),
+  })).optional(),
+  repository: z.union([
+    z.string(),
+    z.object({
+      type: z.string().optional(),
+      url: z.string(),
+    })
+  ]).optional(),
+  homepage: z.string().optional(),
+  license: z.string().optional(),
+  readme: z.string().optional(),
+  'dist-tags': z.object({
+    latest: z.string(),
+  }).optional(),
+  time: z.record(z.string()).optional(),
+});
+
+const NPMSearchSchema = z.object({
+  objects: z.array(z.object({
+    package: z.object({
+      name: z.string(),
+      version: z.string(),
+      description: z.string().optional(),
+      keywords: z.array(z.string()).optional(),
+      author: z.union([
+        z.string(),
+        z.object({
+          name: z.string(),
+          email: z.string().optional(),
+        })
+      ]).optional(),
+      maintainers: z.array(z.object({
+        username: z.string(),
+        email: z.string().optional(),
+      })).optional(),
+      repository: z.union([
+        z.string(),
+        z.object({
+          type: z.string().optional(),
+          url: z.string(),
+        })
+      ]).optional(),
+      links: z.object({
+        npm: z.string().optional(),
+        homepage: z.string().optional(),
+        repository: z.string().optional(),
+        bugs: z.string().optional(),
+      }).optional(),
+    }),
+    score: z.object({
+      final: z.number(),
+      detail: z.object({
+        quality: z.number(),
+        popularity: z.number(),
+        maintenance: z.number(),
+      }),
+    }),
+    searchScore: z.number().optional(),
+  })),
+  total: z.number(),
+  time: z.string(),
+});
+
+export interface NPMConfig {
+  maxResults?: number;
+  includeDetails?: boolean;
+  quality?: number; // Minimum quality score (0-1)
+  popularity?: number; // Minimum popularity score (0-1)
+}
+
+export class NPMSearchProvider {
+  public readonly name = 'npm';
+  private readonly registryUrl = 'https://registry.npmjs.org';
+  private readonly searchUrl = 'https://registry.npmjs.com/-/v1/search';
+  private readonly config: NPMConfig;
+
+  constructor(config: NPMConfig = {}) {
+    this.config = {
+      maxResults: 20,
+      includeDetails: true,
+      quality: 0.3,
+      popularity: 0.1,
+      ...config,
+    };
+  }
+
+  async search(query: string, maxResults?: number): Promise<SearchResult[]> {
+    const startTime = Date.now();
+    const limit = Math.min(maxResults || this.config.maxResults || 20, 100); // NPM search API limit
+
+    // Handle empty query
+    if (!query || query.trim().length === 0) {
+      elizaLogger.debug('[NPM] Empty query, returning no results');
+      return [];
+    }
+
+    try {
+      elizaLogger.info(`[NPM] Searching for: ${query}`);
+
+      const searchResponse = await axios.get(this.searchUrl, {
+        params: {
+          text: query,
+          size: limit,
+          quality: this.config.quality,
+          popularity: this.config.popularity,
+        },
+        headers: {
+          'User-Agent': 'ElizaOS-Research-Agent/1.0',
+        },
+        timeout: 10000,
+      });
+
+      const searchData = NPMSearchSchema.parse(searchResponse.data);
+      
+      // Get detailed information if requested
+      let results: SearchResult[] = [];
+      
+      if (this.config.includeDetails) {
+        // Get detailed package info for top results
+        const topPackages = searchData.objects.slice(0, Math.min(limit, 10));
+        const detailedResults = await Promise.all(
+          topPackages.map(async (item) => {
+            const details = await this.getPackageDetails(item.package.name);
+            return this.convertToSearchResult(item, details);
+          })
+        );
+        results = detailedResults.filter(Boolean);
+      } else {
+        // Use search results directly
+        results = searchData.objects
+          .slice(0, limit)
+          .map((item) => this.convertToSearchResult(item, null));
+      }
+
+      const duration = Date.now() - startTime;
+      elizaLogger.info(`[NPM] Found ${results.length} results in ${duration}ms`);
+
+      return results;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      elizaLogger.error(`[NPM] Search failed after ${duration}ms:`, error);
+      
+      // Wrap the error to prevent serialization issues
+      if (axios.isAxiosError(error)) {
+        throw new Error(`NPM search failed: ${error.message} (${error.response?.status || 'no status'})`);
+      }
+      throw new Error(`NPM search failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  private async getPackageDetails(packageName: string): Promise<any | null> {
+    try {
+      const response = await axios.get(`${this.registryUrl}/${encodeURIComponent(packageName)}`, {
+        headers: {
+          'User-Agent': 'ElizaOS-Research-Agent/1.0',
+        },
+        timeout: 5000,
+      });
+
+      return NPMPackageSchema.parse(response.data);
+    } catch (error) {
+      if (axios.isAxiosError(error) && error.response?.status === 404) {
+        elizaLogger.debug(`[NPM] Package ${packageName} not found`);
+      } else {
+        elizaLogger.warn(`[NPM] Failed to get details for ${packageName}:`, 
+          error instanceof Error ? error.message : String(error));
+      }
+      return null;
+    }
+  }
+
+  private convertToSearchResult(item: any, details?: any): SearchResult {
+    const pkg = item.package;
+    const score = item.score || { final: 0.5, detail: { quality: 0.5, popularity: 0.5, maintenance: 0.5 } };
+    
+    // Use detailed info if available, fallback to search result data
+    const packageData = details || pkg;
+    const packageUrl = `https://www.npmjs.com/package/${pkg.name}`;
+    
+    // Extract author information
+    let author = '';
+    if (packageData.author) {
+      if (typeof packageData.author === 'string') {
+        author = packageData.author;
+      } else if (packageData.author.name) {
+        author = packageData.author.name;
+      }
+    }
+
+    // Extract repository URL
+    let repositoryUrl = '';
+    if (packageData.repository) {
+      if (typeof packageData.repository === 'string') {
+        repositoryUrl = packageData.repository;
+      } else if (packageData.repository.url) {
+        repositoryUrl = packageData.repository.url
+          .replace(/^git\+/, '')
+          .replace(/\.git$/, '')
+          .replace(/^git:\/\//, 'https://');
+      }
+    }
+
+    // Create comprehensive description
+    let description = pkg.description || packageData.description || '';
+    if (pkg.keywords && pkg.keywords.length > 0) {
+      description += `\nKeywords: ${pkg.keywords.join(', ')}`;
+    }
+    if (packageData.license) {
+      description += `\nLicense: ${packageData.license}`;
+    }
+
+    // Add quality metrics
+    let qualityInfo = '';
+    if (score.detail) {
+      qualityInfo = `\nQuality: ${(score.detail.quality * 100).toFixed(0)}% | ` +
+                   `Popularity: ${(score.detail.popularity * 100).toFixed(0)}% | ` +
+                   `Maintenance: ${(score.detail.maintenance * 100).toFixed(0)}%`;
+    }
+
+    return {
+      title: `${pkg.name} v${pkg.version}`,
+      url: packageUrl,
+      snippet: description.substring(0, 300) + (description.length > 300 ? '...' : ''),
+      content: description + qualityInfo,
+      score: score.final,
+      provider: 'npm',
+      metadata: {
+        language: 'javascript',
+        author: author ? [author] : [],
+        type: 'package',
+        domain: 'npmjs.com',
+      },
+    };
+  }
+
+  /**
+   * Get specific package information by name
+   */
+  async getPackage(packageName: string): Promise<SearchResult | null> {
+    try {
+      const pkg = await this.getPackageDetails(packageName);
+      if (!pkg) return null;
+
+      // Create a mock search item for conversion
+      const mockSearchItem = {
+        package: {
+          name: pkg.name,
+          version: pkg['dist-tags']?.latest || pkg.version,
+          description: pkg.description,
+          keywords: pkg.keywords,
+          author: pkg.author,
+          links: {
+            npm: `https://www.npmjs.com/package/${pkg.name}`,
+            homepage: pkg.homepage,
+          },
+        },
+        score: {
+          final: 1.0, // Assume high relevance for direct lookup
+          detail: {
+            quality: 0.8,
+            popularity: 0.7,
+            maintenance: 0.8,
+          },
+        },
+      };
+
+      return this.convertToSearchResult(mockSearchItem, pkg);
+    } catch (error) {
+      elizaLogger.error(`[NPM] Failed to get package ${packageName}:`, error);
+      return null;
+    }
+  }
+
+  /**
+   * Search for packages by scope (e.g., @types, @angular)
+   */
+  async searchByScope(scope: string, maxResults?: number): Promise<SearchResult[]> {
+    const query = `scope:${scope}`;
+    return this.search(query, maxResults);
+  }
+
+  /**
+   * Search for packages by maintainer
+   */
+  async searchByMaintainer(maintainer: string, maxResults?: number): Promise<SearchResult[]> {
+    const query = `maintainer:${maintainer}`;
+    return this.search(query, maxResults);
+  }
+
+  /**
+   * Search for packages with specific keywords
+   */
+  async searchByKeywords(keywords: string[], maxResults?: number): Promise<SearchResult[]> {
+    const query = `keywords:${keywords.join(',')}`;
+    return this.search(query, maxResults);
+  }
+
+  /**
+   * Get trending/popular packages in a category
+   */
+  async getTrendingPackages(category?: string, maxResults?: number): Promise<SearchResult[]> {
+    let query = 'boost-exact:false';
+    if (category) {
+      query += ` ${category}`;
+    }
+    
+    return this.search(query, maxResults);
+  }
+}

package/src/integrations/search-providers/pypi.ts

@@ -0,0 +1,211 @@
+import axios, { AxiosError } from 'axios';
+import { SearchResult } from '../../types';
+import { elizaLogger } from '@elizaos/core';
+import { z } from 'zod';
+
+// PyPI API response schema validation
+const PyPIPackageSchema = z.object({
+  info: z.object({
+    name: z.string(),
+    summary: z.string().optional(),
+    description: z.string().optional(),
+    home_page: z.string().optional(),
+    author: z.string().optional(),
+    author_email: z.string().optional(),
+    maintainer: z.string().optional(),
+    version: z.string(),
+    keywords: z.string().optional(),
+    license: z.string().optional(),
+    classifiers: z.array(z.string()).optional(),
+    project_urls: z.record(z.string()).optional(),
+  }),
+  urls: z.array(z.object({
+    filename: z.string(),
+    url: z.string(),
+    upload_time: z.string().optional(),
+  })).optional(),
+});
+
+const PyPISearchSchema = z.object({
+  projects: z.array(z.object({
+    name: z.string(),
+    version: z.string(),
+    description: z.string().optional(),
+  })),
+});
+
+export interface PyPIConfig {
+  maxResults?: number;
+  includeClassifiers?: boolean;
+}
+
+export class PyPISearchProvider {
+  public readonly name = 'pypi';
+  private readonly baseUrl = 'https://pypi.org/pypi';
+  private readonly searchUrl = 'https://pypi.org/search/';
+  private readonly config: PyPIConfig;
+
+  constructor(config: PyPIConfig = {}) {
+    this.config = {
+      maxResults: 20,
+      includeClassifiers: true,
+      ...config,
+    };
+  }
+
+  async search(query: string, maxResults?: number): Promise<SearchResult[]> {
+    const startTime = Date.now();
+    const limit = maxResults || this.config.maxResults || 20;
+
+    // Handle empty query
+    if (!query || query.trim().length === 0) {
+      elizaLogger.debug('[PyPI] Empty query, returning no results');
+      return [];
+    }
+
+    try {
+      elizaLogger.info(`[PyPI] Searching for: ${query}`);
+
+      // Search PyPI (no official search API, using HTML scraping as fallback)
+      // For production use, consider using a proper PyPI API client
+      const searchUrl = `https://pypi.org/search/?q=${encodeURIComponent(query)}`;
+      const searchResponse = await axios.get(searchUrl, {
+        headers: {
+          'User-Agent': 'ElizaOS-Research-Agent/1.0',
+        },
+        timeout: 10000,
+      });
+
+      // Extract package names from search results
+      const packageMatches = searchResponse.data.match(
+        /href="\/project\/([^\/]+)\//g
+      ) || [];
+
+      const packages = packageMatches
+        .map((match: string) => {
+          const name = match.match(/href="\/project\/([^\/]+)\//)?.[1];
+          return name ? { name } : null;
+        })
+        .filter(Boolean)
+        .slice(0, limit);
+
+      // Get detailed information for each package
+      const results = await Promise.all(
+        packages.map(async (pkg: any, index: number) => {
+          const details = await this.getPackageDetails(pkg.name);
+          if (details) {
+            return this.convertToSearchResult(details, index);
+          }
+          return null;
+        })
+      );
+
+      const validResults = results.filter(Boolean) as SearchResult[];
+      const duration = Date.now() - startTime;
+      elizaLogger.info(`[PyPI] Found ${validResults.length} results in ${duration}ms`);
+
+      return validResults;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      elizaLogger.error(`[PyPI] Search failed after ${duration}ms:`, error);
+      
+      // Wrap the error to prevent serialization issues
+      if (axios.isAxiosError(error)) {
+        throw new Error(`PyPI search failed: ${error.message} (${error.response?.status || 'no status'})`);
+      }
+      throw new Error(`PyPI search failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  private async getPackageDetails(packageName: string): Promise<any | null> {
+    try {
+      const response = await axios.get(`${this.baseUrl}/${packageName}/json`, {
+        headers: {
+          'User-Agent': 'ElizaOS-Research-Agent/1.0',
+        },
+        timeout: 5000,
+      });
+
+      return PyPIPackageSchema.parse(response.data);
+    } catch (error) {
+      if (axios.isAxiosError(error) && error.response?.status === 404) {
+        elizaLogger.debug(`[PyPI] Package ${packageName} not found`);
+      } else {
+        elizaLogger.warn(`[PyPI] Failed to get details for ${packageName}:`, 
+          error instanceof Error ? error.message : String(error));
+      }
+      return null;
+    }
+  }
+
+  private convertToSearchResult(pkg: any, index: number): SearchResult {
+    const info = pkg.info;
+    const packageUrl = `https://pypi.org/project/${info.name}/`;
+    
+    // Create a comprehensive description
+    let description = info.summary || info.description || '';
+    if (info.keywords) {
+      description += `\nKeywords: ${info.keywords}`;
+    }
+    if (info.license) {
+      description += `\nLicense: ${info.license}`;
+    }
+
+    // Extract relevant classifiers for additional context
+    let classifiers = '';
+    if (this.config.includeClassifiers && info.classifiers) {
+      const relevantClassifiers = info.classifiers
+        .filter((c: string) => 
+          c.includes('Development Status') || 
+          c.includes('Intended Audience') ||
+          c.includes('Programming Language') ||
+          c.includes('Topic')
+        )
+        .slice(0, 5);
+      
+      if (relevantClassifiers.length > 0) {
+        classifiers = `\nClassifiers: ${relevantClassifiers.join(', ')}`;
+      }
+    }
+
+    return {
+      title: `${info.name} v${info.version}`,
+      url: packageUrl,
+      snippet: description.substring(0, 300) + (description.length > 300 ? '...' : ''),
+      content: description + classifiers,
+      score: Math.max(0.1, 1.0 - (index * 0.05)), // Decrease score based on result order
+      provider: 'pypi',
+      metadata: {
+        language: 'python',
+        author: info.author || info.maintainer ? [info.author || info.maintainer] : [],
+        type: 'package',
+        domain: 'pypi.org',
+      },
+    };
+  }
+
+  /**
+   * Get specific package information by name
+   */
+  async getPackage(packageName: string): Promise<SearchResult | null> {
+    try {
+      const pkg = await this.getPackageDetails(packageName);
+      if (!pkg) return null;
+
+      return this.convertToSearchResult(pkg, 0);
+    } catch (error) {
+      elizaLogger.error(`[PyPI] Failed to get package ${packageName}:`, error);
+      return null;
+    }
+  }
+
+  /**
+   * Search for packages by category/classifier
+   */
+  async searchByCategory(category: string, maxResults?: number): Promise<SearchResult[]> {
+    // This would require scraping PyPI's browse page or using alternative APIs
+    // For now, return empty array and log the limitation
+    elizaLogger.warn('[PyPI] Category search not implemented - use keyword search instead');
+    return [];
+  }
+}