@monostate/node-scraper 1.0.3 → 1.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 BNCA Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -22,13 +22,20 @@ pnpm add @monostate/node-scraper
 ### Basic Usage
 
 ```javascript
-import { smartScrape, BNCASmartScraper } from '@monostate/node-scraper';
+import { smartScrape, smartScreenshot, quickShot } from '@monostate/node-scraper';
 
 // Simple one-line scraping
 const result = await smartScrape('https://example.com');
 console.log(result.content); // Extracted content
 console.log(result.method);  // Method used: direct-fetch, lightpanda, or puppeteer
-console.log(result.performance.totalTime); // Time taken in ms
+
+// Take a screenshot
+const screenshot = await smartScreenshot('https://example.com');
+console.log(screenshot.screenshot); // Base64 encoded image
+
+// Quick screenshot (optimized for speed)
+const quick = await quickShot('https://example.com');
+console.log(quick.screenshot); // Fast screenshot capture
 ```
 
 ### Advanced Usage
@@ -67,6 +74,10 @@ BNCA uses a sophisticated 3-tier fallback system:
 - **Performance**: Complete JavaScript execution
 - **Fallback triggers**: Complex interactions needed
 
+### 📸 Screenshot Methods
+- **Chrome CLI**: Direct Chrome screenshot capture
+- **Quickshot**: Optimized with retry logic and smart timeouts
+
 ## 📊 Performance Benchmark
 
 | Site Type | BNCA | Firecrawl | Speed Advantage |
@@ -79,12 +90,19 @@ BNCA uses a sophisticated 3-tier fallback system:
 
 ## 🎛️ API Reference
 
-### `smartScrape(url, options?)`
+### Convenience Functions
+
+#### `smartScrape(url, options?)`
+Quick scraping with intelligent fallback.
 
-Convenience function for quick scraping.
+#### `smartScreenshot(url, options?)`
+Take a screenshot of any webpage.
+
+#### `quickShot(url, options?)`
+Optimized screenshot capture for maximum speed.
 
 **Parameters:**
-- `url` (string): URL to scrape
+- `url` (string): URL to scrape/capture
 - `options` (object, optional): Configuration options
 
 **Returns:** Promise<ScrapingResult>
@@ -115,6 +133,24 @@ Scrape a URL with intelligent fallback.
 const result = await scraper.scrape('https://example.com');
 ```
 
+##### `scraper.screenshot(url, options?)`
+
+Take a screenshot of a webpage.
+
+```javascript
+const result = await scraper.screenshot('https://example.com');
+const img = result.screenshot; // data:image/png;base64,...
+```
+
+##### `scraper.quickshot(url, options?)`
+
+Quick screenshot capture - optimized for speed with retry logic.
+
+```javascript
+const result = await scraper.quickshot('https://example.com');
+// 2-3x faster than regular screenshot
+```
+
 ##### `scraper.getStats()`
 
 Get performance statistics.

package/index.d.ts

@@ -25,13 +25,15 @@ export interface ScrapingResult {
   /** Size of the content in bytes */
   size?: number;
   /** Method used for scraping */
-  method: 'direct-fetch' | 'lightpanda' | 'puppeteer' | 'failed' | 'error';
+  method: 'direct-fetch' | 'lightpanda' | 'puppeteer' | 'chrome-screenshot' | 'quickshot' | 'failed' | 'error';
   /** Whether browser rendering was needed */
   needsBrowser?: boolean;
   /** Content type from response headers */
   contentType?: string;
   /** Error message if scraping failed */
   error?: string;
+  /** Base64 encoded screenshot (if captured) */
+  screenshot?: string;
   /** Performance metrics */
   performance: {
     /** Total time taken in milliseconds */
@@ -129,6 +131,22 @@ export class BNCASmartScraper {
    */
   scrape(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
 
+  /**
+   * Take a screenshot of a webpage
+   * @param url The URL to capture
+   * @param options Optional configuration overrides
+   * @returns Promise resolving to screenshot result
+   */
+  screenshot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+
+  /**
+   * Quick screenshot capture - optimized for speed
+   * @param url The URL to capture
+   * @param options Optional configuration overrides
+   * @returns Promise resolving to screenshot result
+   */
+  quickshot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+
   /**
    * Get performance statistics for all methods
    * @returns Current statistics
@@ -214,6 +232,22 @@ export class BNCASmartScraper {
  */
 export function smartScrape(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
 
+/**
+ * Convenience function for taking screenshots
+ * @param url The URL to capture
+ * @param options Optional configuration
+ * @returns Promise resolving to screenshot result
+ */
+export function smartScreenshot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+
+/**
+ * Convenience function for quick screenshot capture
+ * @param url The URL to capture
+ * @param options Optional configuration
+ * @returns Promise resolving to screenshot result
+ */
+export function quickShot(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+
 /**
  * Default export - same as BNCASmartScraper class
  */

package/index.js

@@ -3,6 +3,7 @@ import { spawn } from 'child_process';
 import fs from 'fs/promises';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { promises as fsPromises } from 'fs';
 
 let puppeteer = null;
 try {
@@ -397,13 +398,17 @@ export class BNCASmartScraper {
       /__webpack_require__/i
     ];
     
-    // Check for protection systems
+    // Check for protection systems (more specific patterns)
     const protectionIndicators = [
-      /cloudflare/i,
+      /cloudflare.*challenge/i,
+      /cloudflare.*protection/i,
+      /ray id.*cloudflare/i,
       /please enable javascript/i,
       /you need to enable javascript/i,
       /this site requires javascript/i,
-      /jscript.*required/i
+      /jscript.*required/i,
+      /security check.*cloudflare/i,
+      /attention required.*cloudflare/i
     ];
     
     // Check for minimal content (likely SPA)
@@ -447,7 +452,10 @@ export class BNCASmartScraper {
     if (/window\.__NEXT_DATA__/i.test(html)) {
       indicators.push('Next.js data detected');
     }
-    if (/cloudflare/i.test(html)) {
+    if (/cloudflare.*challenge/i.test(html)) {
+      indicators.push('Cloudflare challenge detected');
+    }
+    if (/cloudflare.*protection/i.test(html)) {
       indicators.push('Cloudflare protection detected');
     }
     if (/please enable javascript/i.test(html)) {
@@ -590,6 +598,271 @@ export class BNCASmartScraper {
     }
   }
   
+  /**
+   * Take a screenshot of a webpage
+   */
+  async screenshot(url, options = {}) {
+    const startTime = Date.now();
+    const config = { ...this.options, ...options };
+    
+    this.log(`📸 Taking screenshot for: ${url}`);
+    
+    try {
+      const screenshot = await this.takeScreenshotWithChrome(url, config);
+      
+      return {
+        success: !!screenshot,
+        screenshot,
+        method: 'chrome-screenshot',
+        performance: {
+          totalTime: Date.now() - startTime
+        }
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+        method: 'chrome-screenshot',
+        performance: {
+          totalTime: Date.now() - startTime
+        }
+      };
+    }
+  }
+  
+  /**
+   * Quick screenshot capture - optimized for speed
+   */
+  async quickshot(url, options = {}) {
+    const startTime = Date.now();
+    const config = { 
+      ...this.options, 
+      ...options,
+      timeout: options.timeout || 15000 // Longer timeout for screenshots
+    };
+    
+    this.log(`⚡ Taking quick screenshot for: ${url}`);
+    
+    try {
+      const screenshot = await this.takeScreenshotOptimized(url, config);
+      
+      return {
+        success: !!screenshot,
+        screenshot,
+        method: 'quickshot',
+        performance: {
+          totalTime: Date.now() - startTime
+        }
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+        method: 'quickshot',
+        performance: {
+          totalTime: Date.now() - startTime
+        }
+      };
+    }
+  }
+  
+  /**
+   * Take screenshot using Chrome CLI
+   */
+  async takeScreenshotWithChrome(url, config) {
+    const tempFile = path.join('/tmp', `screenshot_${Date.now()}_${Math.random().toString(36).substring(7)}.png`);
+    
+    try {
+      const args = [
+        '--headless=new',
+        '--disable-gpu',
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-blink-features=AutomationControlled',
+        '--user-agent=' + config.userAgent,
+        '--screenshot=' + tempFile,
+        '--window-size=1280,800',
+        '--hide-scrollbars',
+        '--virtual-time-budget=10000',
+        url
+      ];
+      
+      const chromePath = await this.findChromePath();
+      if (!chromePath) {
+        throw new Error('Chrome/Chromium not found');
+      }
+      
+      return new Promise((resolve) => {
+        const chrome = spawn(chromePath, args, {
+          stdio: ['ignore', 'pipe', 'pipe'],
+          detached: false
+        });
+        
+        let processExited = false;
+        let stderr = '';
+        
+        chrome.stderr.on('data', (data) => {
+          stderr += data.toString();
+        });
+        
+        const killTimeout = setTimeout(() => {
+          if (!processExited) {
+            this.log('Chrome timeout, sending SIGTERM...');
+            chrome.kill('SIGTERM');
+            
+            setTimeout(() => {
+              if (!processExited) {
+                chrome.kill('SIGKILL');
+              }
+            }, 1000);
+          }
+        }, config.timeout || 15000);
+        
+        chrome.on('exit', async (code, signal) => {
+          processExited = true;
+          clearTimeout(killTimeout);
+          
+          try {
+            await new Promise(r => setTimeout(r, 500));
+            const screenshotBuffer = await fsPromises.readFile(tempFile);
+            const base64 = screenshotBuffer.toString('base64');
+            await fsPromises.unlink(tempFile).catch(() => {});
+            resolve(`data:image/png;base64,${base64}`);
+          } catch (error) {
+            resolve(null);
+          }
+        });
+        
+        chrome.on('error', (error) => {
+          clearTimeout(killTimeout);
+          resolve(null);
+        });
+      });
+    } catch (error) {
+      return null;
+    }
+  }
+  
+  /**
+   * Optimized screenshot for speed
+   */
+  async takeScreenshotOptimized(url, config, retryCount = 0) {
+    const tempFile = path.join('/tmp', `screenshot_${Date.now()}_${Math.random().toString(36).substring(7)}.png`);
+    
+    try {
+      const virtualTimeBudget = retryCount === 0 ? 5000 : 8000;
+      const processTimeout = retryCount === 0 ? 8000 : 12000;
+      
+      const args = [
+        '--headless=new',
+        '--disable-gpu',
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-blink-features=AutomationControlled',
+        '--disable-features=TranslateUI',
+        '--disable-extensions',
+        '--disable-default-apps',
+        '--disable-sync',
+        '--metrics-recording-only',
+        '--mute-audio',
+        '--no-first-run',
+        '--disable-background-timer-throttling',
+        '--disable-backgrounding-occluded-windows',
+        '--disable-renderer-backgrounding',
+        '--user-agent=' + config.userAgent,
+        '--screenshot=' + tempFile,
+        '--window-size=1280,800',
+        '--hide-scrollbars',
+        '--run-all-compositor-stages-before-draw',
+        `--virtual-time-budget=${virtualTimeBudget}`,
+        url
+      ];
+      
+      const chromePath = await this.findChromePath();
+      if (!chromePath) {
+        throw new Error('Chrome/Chromium not found');
+      }
+      
+      return new Promise((resolve) => {
+        const chrome = spawn(chromePath, args, {
+          stdio: ['ignore', 'pipe', 'pipe'],
+          detached: false
+        });
+        
+        let processExited = false;
+        
+        const killTimeout = setTimeout(() => {
+          if (!processExited) {
+            chrome.kill('SIGTERM');
+            setTimeout(() => {
+              if (!processExited) {
+                chrome.kill('SIGKILL');
+              }
+            }, 1000);
+          }
+        }, processTimeout);
+        
+        chrome.on('exit', async (code, signal) => {
+          processExited = true;
+          clearTimeout(killTimeout);
+          
+          try {
+            await new Promise(r => setTimeout(r, 500));
+            const screenshotBuffer = await fsPromises.readFile(tempFile);
+            const base64 = screenshotBuffer.toString('base64');
+            await fsPromises.unlink(tempFile).catch(() => {});
+            resolve(`data:image/png;base64,${base64}`);
+          } catch (error) {
+            if (retryCount === 0) {
+              const retryResult = await this.takeScreenshotOptimized(url, config, 1);
+              resolve(retryResult);
+            } else {
+              resolve(null);
+            }
+          }
+        });
+        
+        chrome.on('error', (error) => {
+          clearTimeout(killTimeout);
+          resolve(null);
+        });
+      });
+    } catch (error) {
+      if (retryCount === 0) {
+        return this.takeScreenshotOptimized(url, config, 1);
+      }
+      return null;
+    }
+  }
+  
+  /**
+   * Find Chrome/Chromium binary path
+   */
+  async findChromePath() {
+    const chromePaths = process.platform === 'darwin' ? [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Chromium.app/Contents/MacOS/Chromium',
+    ] : [
+      '/usr/bin/chromium-browser',
+      '/usr/bin/chromium',
+      '/usr/bin/google-chrome-stable',
+      '/usr/bin/google-chrome',
+    ];
+    
+    for (const path of chromePaths) {
+      try {
+        await fsPromises.access(path);
+        return path;
+      } catch (e) {
+        continue;
+      }
+    }
+    
+    return null;
+  }
+  
   /**
    * Health check for all scraping methods
    */
@@ -630,7 +903,7 @@ export class BNCASmartScraper {
   }
 }
 
-// Export convenience function
+// Export convenience functions
 export async function smartScrape(url, options = {}) {
   const scraper = new BNCASmartScraper(options);
   try {
@@ -643,4 +916,24 @@ export async function smartScrape(url, options = {}) {
   }
 }
 
+export async function smartScreenshot(url, options = {}) {
+  const scraper = new BNCASmartScraper(options);
+  try {
+    const result = await scraper.screenshot(url, options);
+    return result;
+  } catch (error) {
+    throw error;
+  }
+}
+
+export async function quickShot(url, options = {}) {
+  const scraper = new BNCASmartScraper(options);
+  try {
+    const result = await scraper.quickshot(url, options);
+    return result;
+  } catch (error) {
+    throw error;
+  }
+}
+
 export default BNCASmartScraper;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@monostate/node-scraper",
-  "version": "1.0.3",
+  "version": "1.1.1",
   "description": "Intelligent web scraping with multi-level fallback system - 11.35x faster than Firecrawl",
   "type": "module",
   "main": "index.js",