ag-webscrape 0.0.1

package/README.md

@@ -0,0 +1,207 @@
+# ag-webscrape
+
+A TypeScript web scraper with intelligent fallback strategy. Attempts direct HTTP fetching first, then falls back to Playwright for anti-scraping protection.
+
+## Features
+
+- **Dual Strategy**: Direct fetch first, Playwright fallback
+- **Anti-Scraping Detection**: Automatically detects and bypasses common anti-scraping measures
+- **Persistent Browser**: Maintains browser instance for faster subsequent scrapes
+- **Error Handling**: Comprehensive error detection for 4xx/5xx responses
+- **TypeScript Support**: Full type safety and IntelliSense
+- **Configurable**: Extensive customization options
+
+## Installation
+
+```bash
+npm install ag-webscrape
+```
+
+## Quick Start
+
+```typescript
+import { WebScraper } from 'ag-webscrape';
+
+const scraper = new WebScraper();
+
+// Scrape a single URL
+const result = await scraper.scrape('https://example.com');
+console.log(result.html);
+
+// Clean up when done
+await scraper.dispose();
+```
+
+## API Reference
+
+### WebScraper Class
+
+#### Constructor
+
+```typescript
+new WebScraper(options?: ScrapingOptions)
+```
+
+#### Options
+
+```typescript
+interface ScrapingOptions {
+  timeout?: number;           // Request timeout in ms (default: 30000)
+  userAgent?: string;         // Custom user agent
+  headers?: Record<string, string>; // Additional headers
+  retries?: number;           // Number of retries (default: 3)
+  waitForSelector?: string;   // CSS selector to wait for
+  waitForTimeout?: number;    // Time to wait in ms (default: 5000)
+}
+```
+
+#### Methods
+
+##### `scrape(url: string, options?: ScrapingOptions): Promise<ScrapingResult>`
+
+Scrapes a single URL with fallback strategy.
+
+```typescript
+const result = await scraper.scrape('https://example.com', {
+  timeout: 60000,
+  waitForSelector: '.main-content'
+});
+```
+
+##### `scrapeMultiple(urls: string[], options?: ScrapingOptions): Promise<ScrapingResult[]>`
+
+Scrapes multiple URLs efficiently.
+
+```typescript
+const results = await scraper.scrapeMultiple([
+  'https://example1.com',
+  'https://example2.com'
+]);
+```
+
+##### `dispose(): Promise<void>`
+
+Cleans up browser resources. Always call this when done.
+
+```typescript
+await scraper.dispose();
+```
+
+#### Result Object
+
+```typescript
+interface ScrapingResult {
+  url: string;              // Original URL
+  html: string;             // HTML content
+  status: number;           // HTTP status code
+  method: 'fetch' | 'playwright'; // Method used
+  error?: string;           // Error message if any
+  redirected?: boolean;     // Whether request was redirected
+  finalUrl?: string;        // Final URL after redirects
+}
+```
+
+## Advanced Usage
+
+### Custom Headers and User Agent
+
+```typescript
+const scraper = new WebScraper({
+  userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+  headers: {
+    'Accept': 'text/html,application/xhtml+xml',
+    'Accept-Language': 'en-US,en;q=0.9'
+  }
+});
+```
+
+### Waiting for Content
+
+```typescript
+// Wait for specific element
+const result = await scraper.scrape('https://spa-app.com', {
+  waitForSelector: '.dynamic-content'
+});
+
+// Wait for specific time
+const result = await scraper.scrape('https://slow-app.com', {
+  waitForTimeout: 10000
+});
+```
+
+### Error Handling
+
+```typescript
+const result = await scraper.scrape('https://example.com');
+
+if (result.error) {
+  console.error('Scraping failed:', result.error);
+} else {
+  console.log('Success:', result.html.length, 'characters');
+}
+```
+
+### Batch Scraping
+
+```typescript
+const urls = [
+  'https://news.site.com/article1',
+  'https://news.site.com/article2',
+  'https://news.site.com/article3'
+];
+
+const results = await scraper.scrapeMultiple(urls, {
+  waitForSelector: '.article-content'
+});
+
+results.forEach((result, index) => {
+  if (!result.error) {
+    console.log(`Article ${index + 1}: ${result.html.length} chars`);
+  }
+});
+```
+
+## How It Works
+
+1. **Direct Fetch**: First attempts HTTP request using `node-fetch`
+2. **Anti-Scraping Detection**: Checks response for common anti-scraping patterns
+3. **Playwright Fallback**: If direct fetch fails or anti-scraping detected, uses Playwright
+4. **Error Detection**: Monitors for 4xx/5xx responses in both methods
+5. **Resource Management**: Maintains browser instance for performance
+
+## Anti-Scraping Protection
+
+The scraper automatically detects and handles:
+
+- Cloudflare protection
+- DistilNetworks
+- PerimeterX
+- DataDome
+- Akamai Bot Manager
+- CAPTCHA challenges
+- JavaScript requirement checks
+- Rate limiting
+- Access denied pages
+
+## Performance
+
+- **Fast**: Direct fetch for simple pages
+- **Efficient**: Reuses browser instance
+- **Robust**: Fallback ensures high success rate
+- **Intelligent**: Only uses Playwright when necessary
+
+## Examples
+
+Check out the `src/example.ts` file for complete usage examples.
+
+## License
+
+MIT
+
+## Contributing
+
+Pull requests welcome! Please ensure TypeScript compilation and tests pass.
+
+## Support
+
+For issues and questions, please use the GitHub issue tracker. 

package/dist/WebScraper.d.ts

@@ -0,0 +1,34 @@
+export interface ScrapingOptions {
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    retries?: number;
+    waitForSelector?: string;
+    waitForTimeout?: number;
+}
+export interface ScrapingResult {
+    url: string;
+    html: string;
+    status: number;
+    method: 'fetch' | 'playwright';
+    error?: string;
+    redirected?: boolean;
+    finalUrl?: string;
+}
+export declare class WebScraper {
+    private browser;
+    private context;
+    private page;
+    private userAgent;
+    private defaultOptions;
+    constructor(options?: ScrapingOptions);
+    private initializeBrowser;
+    private detectAntiScraping;
+    private fetchDirectly;
+    private scrapeWithPlaywright;
+    private isErrorPage;
+    scrape(url: string, options?: ScrapingOptions): Promise<ScrapingResult>;
+    scrapeMultiple(urls: string[], options?: ScrapingOptions): Promise<ScrapingResult[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=WebScraper.d.ts.map

package/dist/WebScraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebScraper.d.ts","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,GAAG,YAAY,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAkB;gBAE5B,OAAO,GAAE,eAAoB;YAa3B,iBAAiB;IA2B/B,OAAO,CAAC,kBAAkB;YAwBZ,aAAa;YAiDb,oBAAoB;IA4ElC,OAAO,CAAC,WAAW;IAmBb,MAAM,CACV,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC;IAkDpB,cAAc,CAClB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB/B"}

package/dist/WebScraper.js

@@ -0,0 +1,228 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebScraper = void 0;
+const playwright_1 = require("playwright");
+class WebScraper {
+    constructor(options = {}) {
+        this.browser = null;
+        this.context = null;
+        this.page = null;
+        this.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
+        this.defaultOptions = {
+            timeout: 30000,
+            retries: 3,
+            waitForTimeout: 5000,
+            ...options,
+        };
+    }
+    async initializeBrowser() {
+        if (!this.browser) {
+            this.browser = await playwright_1.chromium.launch({
+                headless: true,
+                args: [
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-accelerated-2d-canvas',
+                    '--disable-gpu',
+                    '--window-size=1920,1080',
+                ],
+            });
+            this.context = await this.browser.newContext({
+                userAgent: this.defaultOptions.userAgent || this.userAgent.toString(),
+                viewport: { width: 1920, height: 1080 },
+                extraHTTPHeaders: this.defaultOptions.headers || {},
+            });
+            this.page = await this.context.newPage();
+        }
+    }
+    detectAntiScraping(html) {
+        const antiScrapingPatterns = [
+            /cloudflare/i,
+            /distil.networks/i,
+            /perimeterx/i,
+            /datadome/i,
+            /akamai/i,
+            /bot.protection/i,
+            /please.enable.javascript/i,
+            /access.denied/i,
+            /blocked/i,
+            /captcha/i,
+            /challenge/i,
+            /security.check/i,
+            /rate.limit/i,
+            /temporarily.unavailable/i,
+        ];
+        return antiScrapingPatterns.some((pattern) => pattern.test(html));
+    }
+    async fetchDirectly(url, options) {
+        const headers = {
+            'User-Agent': options.userAgent || this.userAgent.toString(),
+            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate',
+            Connection: 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            ...options.headers,
+        };
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), options.timeout || this.defaultOptions.timeout);
+        try {
+            const response = await fetch(url, {
+                headers,
+                signal: controller.signal,
+                redirect: 'follow',
+            });
+            clearTimeout(timeoutId);
+            const html = await response.text();
+            return {
+                url,
+                html,
+                status: response.status,
+                method: 'fetch',
+                redirected: response.redirected,
+                finalUrl: response.url,
+            };
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            throw error;
+        }
+    }
+    async scrapeWithPlaywright(url, options) {
+        await this.initializeBrowser();
+        if (!this.page) {
+            throw new Error('Failed to initialize Playwright page');
+        }
+        const page = this.page;
+        let html = '';
+        let status = 0;
+        let error;
+        try {
+            if (options.headers) {
+                await page.setExtraHTTPHeaders(options.headers);
+            }
+            const response = await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: options.timeout || this.defaultOptions.timeout,
+            });
+            if (response) {
+                status = response.status();
+                if (status >= 400) {
+                    error = `HTTP ${status} error`;
+                }
+            }
+            if (options.waitForSelector) {
+                await page.waitForSelector(options.waitForSelector, {
+                    timeout: options.waitForTimeout || this.defaultOptions.waitForTimeout,
+                });
+            }
+            else if (options.waitForTimeout) {
+                await page.waitForTimeout(options.waitForTimeout);
+            }
+            html = await page.content();
+            if (this.isErrorPage(html)) {
+                error = error || 'Error page detected in HTML content';
+            }
+            return {
+                url,
+                html,
+                status,
+                method: 'playwright',
+                error,
+                finalUrl: page.url(),
+            };
+        }
+        catch (err) {
+            return {
+                url,
+                html: '',
+                status: 0,
+                method: 'playwright',
+                error: err instanceof Error ? err.message : 'Unknown error',
+                finalUrl: page.url(),
+            };
+        }
+    }
+    isErrorPage(html) {
+        const errorPatterns = [
+            /<title>.*4\d\d.*<\/title>/i,
+            /<title>.*5\d\d.*<\/title>/i,
+            /<title>.*error.*<\/title>/i,
+            /<title>.*not found.*<\/title>/i,
+            /<title>.*forbidden.*<\/title>/i,
+            /<title>.*unauthorized.*<\/title>/i,
+            /<h1>.*4\d\d.*<\/h1>/i,
+            /<h1>.*5\d\d.*<\/h1>/i,
+            /<h1>.*error.*<\/h1>/i,
+        ];
+        return errorPatterns.some((pattern) => pattern.test(html));
+    }
+    async scrape(url, options = {}) {
+        const mergedOptions = { ...this.defaultOptions, ...options };
+        let lastError = null;
+        try {
+            const result = await this.fetchDirectly(url, mergedOptions);
+            if (result.status >= 200 &&
+                result.status < 300 &&
+                !this.detectAntiScraping(result.html)) {
+                return result;
+            }
+            console.log(`Direct fetch failed or anti-scraping detected for ${url}. Falling back to Playwright.`);
+        }
+        catch (error) {
+            lastError =
+                error instanceof Error ? error : new Error('Unknown fetch error');
+            console.log(`Direct fetch failed for ${url}: ${lastError.message}. Falling back to Playwright.`);
+        }
+        try {
+            const result = await this.scrapeWithPlaywright(url, mergedOptions);
+            return result;
+        }
+        catch (error) {
+            const playwrightError = error instanceof Error ? error : new Error('Unknown Playwright error');
+            return {
+                url,
+                html: '',
+                status: 0,
+                method: 'playwright',
+                error: `Both methods failed. Fetch: ${lastError?.message || 'Unknown'}. Playwright: ${playwrightError.message}`,
+            };
+        }
+    }
+    async scrapeMultiple(urls, options = {}) {
+        const results = [];
+        for (const url of urls) {
+            try {
+                const result = await this.scrape(url, options);
+                results.push(result);
+            }
+            catch (error) {
+                results.push({
+                    url,
+                    html: '',
+                    status: 0,
+                    method: 'fetch',
+                    error: error instanceof Error ? error.message : 'Unknown error',
+                });
+            }
+        }
+        return results;
+    }
+    async dispose() {
+        if (this.page) {
+            await this.page.close();
+            this.page = null;
+        }
+        if (this.context) {
+            await this.context.close();
+            this.context = null;
+        }
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+        }
+    }
+}
+exports.WebScraper = WebScraper;
+//# sourceMappingURL=WebScraper.js.map

package/dist/WebScraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebScraper.js","sourceRoot":"","sources":["../src/WebScraper.ts"],"names":[],"mappings":";;;AACA,2CAAsC;AAqBtC,MAAa,UAAU;IAOrB,YAAY,UAA2B,EAAE;QANjC,YAAO,GAAmB,IAAI,CAAC;QAC/B,YAAO,GAA0B,IAAI,CAAC;QACtC,SAAI,GAAgB,IAAI,CAAC;QAK/B,IAAI,CAAC,SAAS,GAAG,iHAAiH,CAAC;QACnI,IAAI,CAAC,cAAc,GAAG;YACpB,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,CAAC;YACV,cAAc,EAAE,IAAI;YACpB,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAKO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,GAAG,MAAM,qBAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ,EAAE,IAAI;gBACd,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,iCAAiC;oBACjC,eAAe;oBACf,yBAAyB;iBAC1B;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC3C,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;gBACrE,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;gBACvC,gBAAgB,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,IAAI,EAAE;aACpD,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;IAKO,kBAAkB,CAAC,IAAY;QACrC,MAAM,oBAAoB,GAAG;YAC3B,aAAa;YACb,kBAAkB;YAClB,aAAa;YACb,WAAW;YACX,SAAS;YACT,iBAAiB;YACjB,2BAA2B;YAC3B,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,0BAA0B;SAC3B,CAAC;QAEF,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,CAAC;IAKO,KAAK,CAAC,aAAa,CACzB,GAAW,EACX,OAAwB;QAExB,MAAM,OAAO,GAAG;YACd,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;YAC5D,MAAM,EACJ,4EAA4E;YAC9E,iBAAiB,EAAE,gBAAgB;YACnC,iBAAiB,EAAE,eAAe;YAClC,UAAU,EAAE,YAAY;YACxB,2BAA2B,EAAE,GAAG;YAChC,GAAG,OAAO,CAAC,OAAO;SACnB,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAC1B,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EACxB,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,OAAQ,CAChD,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,OAAO;gBACL,GAAG;gBACH,IAAI;gBACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,GAAG;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,oBAAoB,CAChC,GAAW,EACX,OAAwB;QAExB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,KAAyB,CAAC;QAE9B,IAAI,CAAC;YAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC;YAGD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpC,SAAS,EAAE,aAAa;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO;aACxD,CAAC,CAAC;YAEH,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAG3B,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBAClB,KAAK,GAAG,QAAQ,MAAM,QAAQ,CAAC;gBACjC,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE;oBAClD,OAAO,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc;iBACtE,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAClC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACpD,CAAC;YAGD,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAG5B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,GAAG,KAAK,IAAI,qCAAqC,CAAC;YACzD,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,IAAI;gBACJ,MAAM;gBACN,MAAM,EAAE,YAAY;gBACpB,KAAK;gBACL,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;aACrB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,GAAG;gBACH,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,YAAY;gBACpB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBAC3D,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAKO,WAAW,CAAC,IAAY;QAC9B,MAAM,aAAa,GAAG;YACpB,4BAA4B;YAC5B,4BAA4B;YAC5B,4BAA4B;YAC5B,gCAAgC;YAChC,gCAAgC;YAChC,mCAAmC;YACnC,sBAAsB;YACtB,sBAAsB;YACtB,sBAAsB;SACvB,CAAC;QAEF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IAKD,KAAK,CAAC,MAAM,CACV,GAAW,EACX,UAA2B,EAAE;QAE7B,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,OAAO,EAAE,CAAC;QAC7D,IAAI,SAAS,GAAiB,IAAI,CAAC;QAGnC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YAG5D,IACE,MAAM,CAAC,MAAM,IAAI,GAAG;gBACpB,MAAM,CAAC,MAAM,GAAG,GAAG;gBACnB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,EACrC,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;YAGD,OAAO,CAAC,GAAG,CACT,qDAAqD,GAAG,+BAA+B,CACxF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS;gBACP,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACpE,OAAO,CAAC,GAAG,CACT,2BAA2B,GAAG,KAAK,SAAS,CAAC,OAAO,+BAA+B,CACpF,CAAC;QACJ,CAAC;QAGD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,eAAe,GACnB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAEzE,OAAO;gBACL,GAAG;gBACH,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,YAAY;gBACpB,KAAK,EAAE,+BAA+B,SAAS,EAAE,OAAO,IAAI,SAAS,iBAAiB,eAAe,CAAC,OAAO,EAAE;aAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,cAAc,CAClB,IAAc,EACd,UAA2B,EAAE;QAE7B,MAAM,OAAO,GAAqB,EAAE,CAAC;QAErC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG;oBACH,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;iBAChE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAKD,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;CACF;AAvTD,gCAuTC"}

package/dist/example.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=example.d.ts.map

package/dist/example.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"example.d.ts","sourceRoot":"","sources":["../src/example.ts"],"names":[],"mappings":""}

package/dist/example.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+async function runExample() {
+    const scraper = new index_1.WebScraper({
+        timeout: 30000,
+        waitForTimeout: 3000,
+    });
+    try {
+        console.log('Starting web scraping example...');
+        const result = await scraper.scrape('https://httpbin.org/html');
+        console.log('Scraped successfully:', {
+            url: result.url,
+            method: result.method,
+            status: result.status,
+            htmlLength: result.html.length,
+            error: result.error,
+        });
+        const urls = [
+            'https://httpbin.org/html',
+            'https://httpbin.org/status/200',
+            'https://httpbin.org/status/404',
+        ];
+        console.log('\nScraping multiple URLs...');
+        const results = await scraper.scrapeMultiple(urls);
+        results.forEach((result, index) => {
+            console.log(`Result ${index + 1}:`, {
+                url: result.url,
+                method: result.method,
+                status: result.status,
+                htmlLength: result.html.length,
+                error: result.error,
+            });
+        });
+    }
+    catch (error) {
+        console.error('Error during scraping:', error);
+    }
+    finally {
+        await scraper.dispose();
+        console.log('Scraper disposed successfully');
+    }
+}
+if (require.main === module) {
+    runExample().catch(console.error);
+}
+//# sourceMappingURL=example.js.map

package/dist/example.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"example.js","sourceRoot":"","sources":["../src/example.ts"],"names":[],"mappings":";;AAAA,mCAAqC;AAErC,KAAK,UAAU,UAAU;IACvB,MAAM,OAAO,GAAG,IAAI,kBAAU,CAAC;QAC7B,OAAO,EAAE,KAAK;QACd,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAGhD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE;YACnC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;YAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;QAGH,MAAM,IAAI,GAAG;YACX,0BAA0B;YAC1B,gCAAgC;YAChC,gCAAgC;SACjC,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEnD,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YAChC,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,GAAG,EAAE;gBAClC,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;gBAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QAET,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAGD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;IAC5B,UAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import { WebScraper } from './WebScraper';
+export type { ScrapingOptions, ScrapingResult } from './WebScraper';
+export { WebScraper } from './WebScraper';
+export default WebScraper;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,eAAe,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebScraper = void 0;
+const WebScraper_1 = require("./WebScraper");
+var WebScraper_2 = require("./WebScraper");
+Object.defineProperty(exports, "WebScraper", { enumerable: true, get: function () { return WebScraper_2.WebScraper; } });
+exports.default = WebScraper_1.WebScraper;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,6CAA0C;AAG1C,2CAA0C;AAAjC,wGAAA,UAAU,OAAA;AAEnB,kBAAe,uBAAU,CAAC"}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "ag-webscrape",
+  "version": "0.0.1",
+  "author": "admin@gec.dev",
+  "description": "TypeScript web scraper with Playwright fallback for anti-scraping protection",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "keywords": [
+    "scraping",
+    "playwright",
+    "typescript",
+    "web-scraper",
+    "anti-scraping"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "playwright": "1.54.1"
+  },
+  "devDependencies": {
+    "@types/node": "24.0.13",
+    "eslint": "9.31.0",
+    "eslint-config-e7npm": "0.1.23",
+    "typescript": "5.8.3"
+  },
+  "files": [
+    "dist/**/*",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=20",
+    "yarn": "use pnpm",
+    "npm": "use pnpm",
+    "pnpm": ">=3"
+  },
+  "scripts": {
+    "preinstall": "npx only-allow pnpm",
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "next lint",
+    "format": "eslint src --fix"
+  }
+}