@monostate/node-scraper 1.3.0 → 1.5.0

package/README.md

@@ -19,7 +19,9 @@ yarn add @monostate/node-scraper
 pnpm add @monostate/node-scraper
 ```
 
-**🎉 New in v1.3.0**: PDF parsing support added! Automatically extracts text, metadata, and page count from PDF documents.
+**🤖 New in v1.5.0**: AI-powered Q&A! Ask questions about any website using OpenRouter, OpenAI, or built-in AI. (Note: v1.4.0 was an internal release)
+
+**🎉 Also in v1.3.0**: PDF parsing support added! Automatically extracts text, metadata, and page count from PDF documents.
 
 **✨ Also in v1.2.0**: Lightpanda binary is now automatically downloaded and configured during installation! No manual setup required.
 
@@ -199,6 +201,73 @@ Clean up resources (close browser instances).
 await scraper.cleanup();
 ```
 
+### 🤖 AI-Powered Q&A
+
+Ask questions about any website and get AI-generated answers:
+
+```javascript
+// Method 1: Using your own OpenRouter API key
+const scraper = new BNCASmartScraper({
+  openRouterApiKey: 'your-openrouter-api-key'
+});
+const result = await scraper.askAI('https://example.com', 'What is this website about?');
+
+// Method 2: Using OpenAI API (or compatible endpoints)
+const scraper = new BNCASmartScraper({
+  openAIApiKey: 'your-openai-api-key',
+  // Optional: Use a compatible endpoint like Groq, Together AI, etc.
+  openAIBaseUrl: 'https://api.groq.com/openai'
+});
+const result = await scraper.askAI('https://example.com', 'What services do they offer?');
+
+// Method 3: One-liner with OpenRouter
+import { askWebsiteAI } from '@monostate/node-scraper';
+const answer = await askWebsiteAI('https://example.com', 'What is the main topic?', {
+  openRouterApiKey: process.env.OPENROUTER_API_KEY
+});
+
+// Method 4: Using BNCA backend API (requires BNCA API key)
+const scraper = new BNCASmartScraper({
+  apiKey: 'your-bnca-api-key'
+});
+const result = await scraper.askAI('https://example.com', 'What products are featured?');
+```
+
+**API Key Priority:**
+1. OpenRouter API key (`openRouterApiKey`)
+2. OpenAI API key (`openAIApiKey`)
+3. BNCA backend API (`apiKey`)
+4. Local fallback (pattern matching - no API key required)
+
+**Configuration Options:**
+```javascript
+const result = await scraper.askAI(url, question, {
+  // OpenRouter specific
+  openRouterApiKey: 'sk-or-...',
+  model: 'meta-llama/llama-4-scout:free', // Default model
+  
+  // OpenAI specific
+  openAIApiKey: 'sk-...',
+  openAIBaseUrl: 'https://api.openai.com', // Or compatible endpoint
+  model: 'gpt-3.5-turbo',
+  
+  // Shared options
+  temperature: 0.3,
+  maxTokens: 500
+});
+```
+
+**Response Format:**
+```javascript
+{
+  success: true,
+  answer: "This website is about...",
+  method: "direct-fetch",     // Scraping method used
+  scrapeTime: 1234,          // Time to scrape in ms
+  processing: "openrouter"   // AI processing method used
+}
+```
+
 ### 📄 PDF Support
 
 BNCA automatically detects and parses PDF documents:

package/index.d.ts

@@ -13,6 +13,24 @@ export interface ScrapingOptions {
   lightpandaPath?: string;
   /** Custom user agent string */
   userAgent?: string;
+  /** BNCA API key for backend services */
+  apiKey?: string;
+  /** BNCA API URL (defaults to https://bnca-api.fly.dev) */
+  apiUrl?: string;
+  /** OpenRouter API key for AI processing */
+  openRouterApiKey?: string;
+  /** OpenAI API key for AI processing */
+  openAIApiKey?: string;
+  /** OpenAI base URL (for compatible endpoints) */
+  openAIBaseUrl?: string;
+  /** AI model to use */
+  model?: string;
+  /** AI temperature setting */
+  temperature?: number;
+  /** Maximum tokens for AI response */
+  maxTokens?: number;
+  /** HTTP referer for OpenRouter */
+  referer?: string;
 }
 
 export interface ScrapingResult {
@@ -146,6 +164,22 @@ export class BNCASmartScraper {
    * @returns Promise resolving to screenshot result
    */
   quickshot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+  
+  /**
+   * Ask AI a question about a URL
+   * @param url The URL to analyze
+   * @param question The question to answer about the page
+   * @param options Optional configuration overrides
+   * @returns Promise resolving to AI answer
+   */
+  askAI(url: string, question: string, options?: ScrapingOptions): Promise<{
+    success: boolean;
+    answer?: string;
+    error?: string;
+    method?: string;
+    scrapeTime?: number;
+    processing?: 'openrouter' | 'openai' | 'backend' | 'local';
+  }>;
 
   /**
    * Get performance statistics for all methods
@@ -248,6 +282,22 @@ export function smartScreenshot(url: string, options?: ScrapingOptions): Promise
  */
 export function quickShot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
 
+/**
+ * Convenience function for asking AI questions about a webpage
+ * @param url The URL to analyze
+ * @param question The question to answer
+ * @param options Optional configuration
+ * @returns Promise resolving to AI answer
+ */
+export function askWebsiteAI(url: string, question: string, options?: ScrapingOptions): Promise<{
+  success: boolean;
+  answer?: string;
+  error?: string;
+  method?: string;
+  scrapeTime?: number;
+  processing?: 'openrouter' | 'openai' | 'backend' | 'local';
+}>;
+
 /**
  * Default export - same as BNCASmartScraper class
  */

package/index.js

@@ -48,6 +48,232 @@ export class BNCASmartScraper {
     };
   }
   
+  /**
+   * Ask AI a question about a URL
+   * Scrapes the URL and uses AI to answer the question
+   * 
+   * @param {string} url - URL to analyze
+   * @param {string} question - Question to answer
+   * @param {object} options - Additional options
+   * @returns {Promise<object>} AI response with answer
+   */
+  async askAI(url, question, options = {}) {
+    try {
+      // First scrape the content
+      const scrapeResult = await this.scrape(url, options);
+      
+      if (!scrapeResult.success) {
+        return {
+          success: false,
+          error: `Failed to scrape URL: ${scrapeResult.error}`,
+          method: scrapeResult.method
+        };
+      }
+      
+      // Check for OpenRouter/OpenAI API key
+      const openRouterKey = options.openRouterApiKey || this.options.openRouterApiKey || process.env.OPENROUTER_API_KEY;
+      const openAIKey = options.openAIApiKey || this.options.openAIApiKey || process.env.OPENAI_API_KEY;
+      
+      // Priority: OpenRouter > OpenAI > Backend API > Local
+      if (openRouterKey) {
+        try {
+          const answer = await this.processWithOpenRouter(question, scrapeResult.content, openRouterKey, options);
+          return {
+            success: true,
+            answer,
+            method: scrapeResult.method,
+            scrapeTime: scrapeResult.stats.totalTime,
+            processing: 'openrouter'
+          };
+        } catch (error) {
+          this.log('  ⚠️ OpenRouter API call failed, falling back...');
+        }
+      }
+      
+      if (openAIKey) {
+        try {
+          const answer = await this.processWithOpenAI(question, scrapeResult.content, openAIKey, options);
+          return {
+            success: true,
+            answer,
+            method: scrapeResult.method,
+            scrapeTime: scrapeResult.stats.totalTime,
+            processing: 'openai'
+          };
+        } catch (error) {
+          this.log('  ⚠️ OpenAI API call failed, falling back...');
+        }
+      }
+      
+      // If BNCA API key is provided, use the backend API
+      if (this.options.apiKey) {
+        try {
+          const response = await fetch(`${this.options.apiUrl || 'https://bnca-api.fly.dev'}/aireply`, {
+            method: 'POST',
+            headers: {
+              'x-api-key': this.options.apiKey,
+              'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ url, question })
+          });
+          
+          if (response.ok) {
+            const data = await response.json();
+            return {
+              success: true,
+              answer: data.answer,
+              method: scrapeResult.method,
+              scrapeTime: scrapeResult.stats.totalTime,
+              processing: 'backend'
+            };
+          }
+        } catch (error) {
+          this.log('  ⚠️ Backend API call failed, using local AI processing');
+        }
+      }
+      
+      // Local AI processing fallback
+      const answer = this.processLocally(question, scrapeResult.content);
+      
+      return {
+        success: true,
+        answer,
+        method: scrapeResult.method,
+        scrapeTime: scrapeResult.stats.totalTime,
+        processing: 'local'
+      };
+      
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message || 'AI processing failed'
+      };
+    }
+  }
+  
+  /**
+   * Process with OpenRouter API
+   * @private
+   */
+  async processWithOpenRouter(question, content, apiKey, options = {}) {
+    const parsedContent = typeof content === 'string' ? JSON.parse(content) : content;
+    
+    const contentText = `
+Title: ${parsedContent.title || 'Unknown'}
+Content: ${parsedContent.content || parsedContent.bodyText || 'No content available'}
+Meta Description: ${parsedContent.metaDescription || 'None'}
+${parsedContent.headings?.length ? `\nHeadings:\n${parsedContent.headings.map(h => `- ${h.text || h}`).join('\n')}` : ''}
+`.trim();
+
+    const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+        'HTTP-Referer': options.referer || 'https://github.com/monostate/node-scraper',
+        'X-Title': 'BNCA Node Scraper',
+      },
+      body: JSON.stringify({
+        model: options.model || 'meta-llama/llama-4-scout:free',
+        messages: [
+          {
+            role: 'system',
+            content: 'You are a helpful assistant that answers questions based on website content. Provide accurate, concise answers based only on the provided content.'
+          },
+          {
+            role: 'user',
+            content: `Based on the following website content, please answer this question: ${question}\n\nWebsite content:\n${contentText}`
+          }
+        ],
+        temperature: options.temperature || 0.3,
+        max_tokens: options.maxTokens || 500,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`OpenRouter API error: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data.choices[0]?.message?.content || 'No response from AI';
+  }
+
+  /**
+   * Process with OpenAI API
+   * @private
+   */
+  async processWithOpenAI(question, content, apiKey, options = {}) {
+    const parsedContent = typeof content === 'string' ? JSON.parse(content) : content;
+    
+    const contentText = `
+Title: ${parsedContent.title || 'Unknown'}
+Content: ${parsedContent.content || parsedContent.bodyText || 'No content available'}
+Meta Description: ${parsedContent.metaDescription || 'None'}
+${parsedContent.headings?.length ? `\nHeadings:\n${parsedContent.headings.map(h => `- ${h.text || h}`).join('\n')}` : ''}
+`.trim();
+
+    const baseUrl = options.openAIBaseUrl || 'https://api.openai.com';
+    const response = await fetch(`${baseUrl}/v1/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model: options.model || 'gpt-3.5-turbo',
+        messages: [
+          {
+            role: 'system',
+            content: 'You are a helpful assistant that answers questions based on website content. Provide accurate, concise answers based only on the provided content.'
+          },
+          {
+            role: 'user',
+            content: `Based on the following website content, please answer this question: ${question}\n\nWebsite content:\n${contentText}`
+          }
+        ],
+        temperature: options.temperature || 0.3,
+        max_tokens: options.maxTokens || 500,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`OpenAI API error: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data.choices[0]?.message?.content || 'No response from AI';
+  }
+
+  /**
+   * Local AI processing (simple pattern matching)
+   * @private
+   */
+  processLocally(question, content) {
+    const parsedContent = typeof content === 'string' ? 
+      JSON.parse(content) : content;
+      
+    const title = parsedContent.title || 'Unknown';
+    const text = parsedContent.content || parsedContent.bodyText || '';
+    const lowerQuestion = question.toLowerCase();
+    
+    if (lowerQuestion.includes('title')) {
+      return `The page title is "${title}".`;
+    }
+    
+    if (lowerQuestion.includes('about') || lowerQuestion.includes('what')) {
+      return `This page titled "${title}" contains: ${text.substring(0, 200)}...`;
+    }
+    
+    if (lowerQuestion.includes('contact') || lowerQuestion.includes('email')) {
+      const emailMatch = text.match(/[\w.-]+@[\w.-]+\.\w+/);
+      return emailMatch ? 
+        `Found contact: ${emailMatch[0]}` : 
+        'No contact information found.';
+    }
+    
+    return `Based on "${title}": ${text.substring(0, 150)}...`;
+  }
+  
   /**
    * Main scraping method with intelligent fallback
    */
@@ -1165,4 +1391,16 @@ export async function quickShot(url, options = {}) {
   }
 }
 
+export async function askWebsiteAI(url, question, options = {}) {
+  const scraper = new BNCASmartScraper(options);
+  try {
+    const result = await scraper.askAI(url, question, options);
+    return result;
+  } catch (error) {
+    throw error;
+  } finally {
+    await scraper.cleanup();
+  }
+}
+
 export default BNCASmartScraper;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@monostate/node-scraper",
-  "version": "1.3.0",
-  "description": "Intelligent web scraping with PDF support and multi-level fallback system - 11.35x faster than Firecrawl",
+  "version": "1.5.0",
+  "description": "Intelligent web scraping with AI Q&A, PDF support and multi-level fallback system - 11x faster than traditional scrapers",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
@@ -32,6 +32,12 @@
     "data-extraction",
     "automation",
     "browser",
+    "ai-powered",
+    "question-answering",
+    "pdf-parsing",
+    "openrouter",
+    "openai",
+    "llm",
     "nextjs",
     "react",
     "performance",