mcp-researchpowerpack 3.6.9

package/dist/clients/research.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"research.js","sourceRoot":"","sources":["../../src/clients/research.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAkC5C,wCAAwC;AACxC,iEAAiE;AACjE,MAAM,qBAAqB,GAAG;IAC5B,UAAU,EAAE,CAAC,EAAE,iDAAiD;IAChE,WAAW,EAAE,IAAI,EAAE,oBAAoB;IACvC,UAAU,EAAE,KAAK;CACT,CAAC;AAEX,0CAA0C;AAC1C,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEpE,+EAA+E;AAC/E,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,yBAAyB;IACzB,uBAAuB;IACvB,yBAAyB;IACzB,mBAAmB;CACpB,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IAEvB;QACE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,OAAO,EAAE,QAAQ,CAAC,QAAQ;YAC1B,MAAM,EAAE,QAAQ,CAAC,OAAO;YACxB,OAAO,EAAE,QAAQ,CAAC,UAAU;YAC5B,UAAU,EAAE,CAAC,EAAE,8BAA8B;SAC9C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAc;QACrC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,MAAM,GAAG,GAAG,KAIX,CAAC;QAEF,0BAA0B;QAC1B,IAAI,GAAG,CAAC,MAAM,IAAI,wBAAwB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,yBAAyB;QACzB,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAClD,IACE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC7B,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAC9B,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,gBAAgB,CAAC;QACtD,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,MAAM,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IAED;;;OAGG;IACK,mBAAmB,CACzB,KAAa,EACb,QAA6D,EAC7D,OAMC;QAED,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9F,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,uCAAuC;YACvC,MAAM,OAAO,GAA4B;gBACvC,KAAK;gBACL,QAAQ;gBACR,WAAW;gBACX,UAAU,EAAE,SAAS;gBACrB,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,eAAe;wBACrB,YAAY,EAAE,EAAE;qBACjB;iBACF;aACF,CAAC;YACF,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,CAAC,eAAe,GAAG,cAAc,CAAC;YAC3C,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,8DAA8D;QAC9D,MAAM,OAAO,GAA4B;YACvC,KAAK;YACL,QAAQ;YACR,WAAW;YACX,gBAAgB,EAAE,eAAe;YACjC,qBAAqB,EAAE,SAAS;YAChC,iBAAiB,EAAE;gBACjB,IAAI,EAAE,IAAI;gBACV,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAClD,gBAAgB,EAAE,IAAI;gBACtB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aAC3B;SACF,CAAC;QACF,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,eAAe,GAAG,cAAc,CAAC;QAC3C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,KAAa,EACb,QAA6D,EAC7D,OAMC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC1E,IAAI,SAAsC,CAAC;QAE3C,4BAA4B;QAC5B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,qBAAqB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,IAAI,CAAC;gBACH,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,MAAM,CAAC,SAAS,EAAE,iBAAiB,OAAO,IAAI,qBAAqB,CAAC,UAAU,QAAQ,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;gBAC7G,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,cAAqB,CAAC,CAAC;gBAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;gBACrC,MAAM,OAAO,GAAG,MAAM,EAAE,OAAc,CAAC;gBAEvC,oBAAoB;gBACpB,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;oBACjC,SAAS,GAAG;wBACV,IAAI,EAAE,SAAS,CAAC,cAAc;wBAC9B,OAAO,EAAE,sCAAsC;wBAC/C,SAAS,EAAE,IAAI;qBAChB,CAAC;oBAEF,IAAI,OAAO,GAAG,qBAAqB,CAAC,UAAU,EAAE,CAAC;wBAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,MAAM,CAAC,SAAS,EAAE,+BAA+B,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;wBAC7E,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,EAAE,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE;oBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,KAAK;oBAC9B,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE;oBACvC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;oBAC/B,YAAY,EAAE,MAAM,EAAE,aAAa;oBACnC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;wBACtB,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;wBAC1C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAC,iBAAiB;wBAClD,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;wBACxC,WAAW,EAAG,QAAQ,CAAC,KAAa,CAAC,gBAAgB;qBACtD,CAAC,CAAC,CAAC,SAAS;oBACb,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;wBAClD,IAAI,EAAE,cAAuB;wBAC7B,GAAG,EAAE,CAAC,CAAC,YAAY,EAAE,GAAG,IAAI,EAAE;wBAC9B,KAAK,EAAE,CAAC,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;wBAClC,UAAU,EAAE,CAAC,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC;wBAC5C,QAAQ,EAAE,CAAC,CAAC,YAAY,EAAE,SAAS,IAAI,CAAC;qBACzC,CAAC,CAAC;iBACJ,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,MAAM,GAAG,GAAG,KAA8C,CAAC;gBAC3D,MAAM,CAAC,OAAO,EAAE,cAAc,KAAK,aAAa,OAAO,GAAG,CAAC,MAAM,SAAS,CAAC,OAAO,aAAa,GAAG,CAAC,MAAM,GAAG,EAAE,UAAU,CAAC,CAAC;gBAE1H,2BAA2B;gBAC3B,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,OAAO,GAAG,qBAAqB,CAAC,UAAU,EAAE,CAAC;oBAC/E,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,eAAe,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;oBAC7D,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uCAAuC;gBACvC,MAAM;YACR,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,EAAE,EAAE,EAAE;YACN,KAAK;YACL,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;YACnB,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,SAAS,IAAI;gBAClB,IAAI,EAAE,SAAS,CAAC,aAAa;gBAC7B,OAAO,EAAE,wBAAwB;gBACjC,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAsB;QACnC,MAAM,EACJ,QAAQ,EACR,YAAY,EACZ,eAAe,GAAG,QAAQ,CAAC,gBAAgB,EAC3C,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,EACpC,SAAS,GAAG,KAAK,EACjB,WAAW,GAAG,GAAG,EACjB,cAAc,GACf,GAAG,MAAM,CAAC;QAEX,iBAAiB;QACjB,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YACtB,OAAO;gBACL,EAAE,EAAE,EAAE;gBACN,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;gBACnB,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS,CAAC,aAAa;oBAC7B,OAAO,EAAE,mCAAmC;oBAC5C,SAAS,EAAE,KAAK;iBACjB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAwD,EAAE,CAAC;QACzE,IAAI,YAAY,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEnD,MAAM,OAAO,GAAG,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QAE9F,0BAA0B;QAC1B,MAAM,CAAC,MAAM,EAAE,yBAAyB,QAAQ,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAEpF,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;YACzB,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,mDAAmD;QACnD,IAAI,QAAQ,CAAC,cAAc,IAAI,QAAQ,CAAC,cAAc,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC1E,MAAM,CAAC,SAAS,EAAE,0CAA0C,QAAQ,CAAC,cAAc,EAAE,EAAE,UAAU,CAAC,CAAC;YACnG,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE9F,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;gBAC1B,OAAO,cAAc,CAAC;YACxB,CAAC;YAED,wDAAwD;YACxD,MAAM,CAAC,OAAO,EAAE,gCAAgC,aAAa,CAAC,KAAK,EAAE,OAAO,eAAe,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;YACxI,OAAO;gBACL,GAAG,cAAc;gBACjB,OAAO,EAAE,8CAA8C,QAAQ,CAAC,KAAK,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,eAAe,QAAQ,CAAC,cAAc,MAAM,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE;aACnL,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,MAAM,CAAC,OAAO,EAAE,wBAAwB,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;QACpF,OAAO;YACL,GAAG,aAAa;YAChB,OAAO,EAAE,oBAAoB,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}

package/dist/clients/scraper.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Web Scraper Client
+ * Generic interface for URL scraping with automatic fallback modes
+ * Implements robust error handling that NEVER crashes
+ */
+import { type StructuredError } from '../utils/errors.js';
+interface ScrapeRequest {
+    url: string;
+    mode?: 'basic' | 'javascript';
+    timeout?: number;
+    country?: string;
+}
+interface ScrapeResponse {
+    content: string;
+    statusCode: number;
+    credits: number;
+    headers?: Record<string, string>;
+    error?: StructuredError;
+}
+interface BatchScrapeResult {
+    results: Array<ScrapeResponse & {
+        url: string;
+    }>;
+    batchesProcessed: number;
+    totalAttempted: number;
+    rateLimitHits: number;
+}
+export declare class ScraperClient {
+    private apiKey;
+    private baseURL;
+    constructor(apiKey?: string);
+    /**
+     * Scrape a single URL with retry logic
+     * NEVER throws - always returns a ScrapeResponse (possibly with error)
+     */
+    scrape(request: ScrapeRequest, maxRetries?: 3): Promise<ScrapeResponse>;
+    /**
+     * Calculate exponential backoff with jitter
+     */
+    private calculateBackoff;
+    /**
+     * Scrape with automatic fallback through different modes
+     * NEVER throws - always returns a ScrapeResponse
+     */
+    scrapeWithFallback(url: string, options?: {
+        timeout?: number;
+    }): Promise<ScrapeResponse>;
+    /**
+     * Scrape multiple URLs with batching
+     * NEVER throws - always returns results array
+     */
+    scrapeMultiple(urls: string[], options?: {
+        timeout?: number;
+    }): Promise<Array<ScrapeResponse & {
+        url: string;
+    }>>;
+    /**
+     * Batch scrape with progress callback
+     * NEVER throws - uses Promise.allSettled internally
+     */
+    batchScrape(urls: string[], options?: {
+        timeout?: number;
+    }, onBatchComplete?: (batchNum: number, totalBatches: number, processed: number) => void): Promise<BatchScrapeResult>;
+    /**
+     * Process a single batch of URLs
+     * NEVER throws
+     */
+    private processBatch;
+}
+export {};
+//# sourceMappingURL=scraper.d.ts.map

package/dist/clients/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAI5B,UAAU,aAAa;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,cAAc;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,UAAU,iBAAiB;IACzB,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAOD,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA2B;gBAE9B,MAAM,CAAC,EAAE,MAAM;IAS3B;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,IAAsB,GAAG,OAAO,CAAC,cAAc,CAAC;IA0J/F;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACG,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,cAAc,CAAC;IA2DlG;;;OAGG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAa1H;;;OAGG;IACG,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,EAClC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,GACpF,OAAO,CAAC,iBAAiB,CAAC;IAmE7B;;;OAGG;YACW,YAAY;CAqB3B"}

package/dist/clients/scraper.js

@@ -0,0 +1,321 @@
+/**
+ * Web Scraper Client
+ * Generic interface for URL scraping with automatic fallback modes
+ * Implements robust error handling that NEVER crashes
+ */
+import { parseEnv, SCRAPER } from '../config/index.js';
+import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
+import { pMapSettled } from '../utils/concurrency.js';
+import { mcpLog } from '../utils/logger.js';
+// Status codes that indicate we should retry (no credit consumed)
+const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);
+// Status codes that are permanent failures (don't retry)
+const PERMANENT_FAILURE_CODES = new Set([400, 401, 403]);
+export class ScraperClient {
+    apiKey;
+    baseURL = 'https://api.scrape.do';
+    constructor(apiKey) {
+        const env = parseEnv();
+        this.apiKey = apiKey || env.SCRAPER_API_KEY;
+        if (!this.apiKey) {
+            throw new Error('SCRAPEDO_API_KEY is required');
+        }
+    }
+    /**
+     * Scrape a single URL with retry logic
+     * NEVER throws - always returns a ScrapeResponse (possibly with error)
+     */
+    async scrape(request, maxRetries = SCRAPER.RETRY_COUNT) {
+        const { url, mode = 'basic', timeout = 30, country } = request;
+        const credits = mode === 'javascript' ? 5 : 1;
+        // Validate URL first
+        try {
+            new URL(url);
+        }
+        catch {
+            return {
+                content: `Invalid URL: ${url}`,
+                statusCode: 400,
+                credits: 0,
+                error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false },
+            };
+        }
+        const params = new URLSearchParams({
+            url: url,
+            token: this.apiKey,
+            timeout: String(timeout * 1000),
+        });
+        if (mode === 'javascript') {
+            params.append('render', 'true');
+        }
+        if (country) {
+            params.append('geoCode', country.toUpperCase());
+        }
+        const apiUrl = `${this.baseURL}?${params.toString()}`;
+        let lastError;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                // Use AbortController for timeout
+                const timeoutMs = (timeout + 10) * 1000; // Add 10s buffer over scrape timeout
+                const response = await fetchWithTimeout(apiUrl, {
+                    method: 'GET',
+                    headers: { Accept: 'text/html,application/json' },
+                    timeoutMs,
+                });
+                // Safely read response body
+                let content;
+                try {
+                    content = await response.text();
+                }
+                catch (readError) {
+                    content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;
+                }
+                // SUCCESS: 2xx - Successful API call
+                if (response.ok) {
+                    return {
+                        content,
+                        statusCode: response.status,
+                        credits,
+                        headers: Object.fromEntries(response.headers.entries()),
+                    };
+                }
+                // 404 - Target not found (permanent, but not an error for our purposes)
+                if (response.status === 404) {
+                    return {
+                        content: '404 - Page not found',
+                        statusCode: 404,
+                        credits,
+                    };
+                }
+                // Permanent failures - don't retry
+                if (PERMANENT_FAILURE_CODES.has(response.status)) {
+                    const errorMsg = response.status === 401
+                        ? 'No credits remaining or subscription suspended'
+                        : `Request failed with status ${response.status}`;
+                    return {
+                        content: `Error: ${errorMsg}`,
+                        statusCode: response.status,
+                        credits: 0,
+                        error: {
+                            code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,
+                            message: errorMsg,
+                            retryable: false,
+                            statusCode: response.status,
+                        },
+                    };
+                }
+                // Retryable status codes
+                if (RETRYABLE_STATUS_CODES.has(response.status)) {
+                    lastError = {
+                        code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,
+                        message: `Server returned ${response.status}`,
+                        retryable: true,
+                        statusCode: response.status,
+                    };
+                    if (attempt < maxRetries - 1) {
+                        const delayMs = this.calculateBackoff(attempt);
+                        mcpLog('warning', `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, 'scraper');
+                        await sleep(delayMs);
+                        continue;
+                    }
+                }
+                // Other non-success status - treat as retryable
+                lastError = classifyError({ status: response.status, message: content });
+                if (attempt < maxRetries - 1 && lastError.retryable) {
+                    const delayMs = this.calculateBackoff(attempt);
+                    mcpLog('warning', `Status ${response.status}. Retrying in ${delayMs}ms`, 'scraper');
+                    await sleep(delayMs);
+                    continue;
+                }
+                // Final attempt failed
+                return {
+                    content: `Error: ${lastError.message}`,
+                    statusCode: response.status,
+                    credits: 0,
+                    error: lastError,
+                };
+            }
+            catch (error) {
+                lastError = classifyError(error);
+                // Non-retryable errors - return immediately
+                if (!lastError.retryable) {
+                    return {
+                        content: `Error: ${lastError.message}`,
+                        statusCode: lastError.statusCode || 500,
+                        credits: 0,
+                        error: lastError,
+                    };
+                }
+                // Retryable error - continue if attempts remaining
+                if (attempt < maxRetries - 1) {
+                    const delayMs = this.calculateBackoff(attempt);
+                    mcpLog('warning', `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, 'scraper');
+                    await sleep(delayMs);
+                    continue;
+                }
+            }
+        }
+        // All retries exhausted
+        return {
+            content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || 'Unknown error'}`,
+            statusCode: lastError?.statusCode || 500,
+            credits: 0,
+            error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'All retries exhausted', retryable: false },
+        };
+    }
+    /**
+     * Calculate exponential backoff with jitter
+     */
+    calculateBackoff(attempt) {
+        const baseDelay = SCRAPER.RETRY_DELAYS[attempt] || 8000;
+        const jitter = Math.random() * 0.3 * baseDelay;
+        return Math.floor(baseDelay + jitter);
+    }
+    /**
+     * Scrape with automatic fallback through different modes
+     * NEVER throws - always returns a ScrapeResponse
+     */
+    async scrapeWithFallback(url, options = {}) {
+        const attempts = [
+            { mode: 'basic', description: 'basic mode' },
+            { mode: 'javascript', description: 'javascript rendering' },
+            { mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },
+        ];
+        const attemptResults = [];
+        let lastResult = null;
+        for (const attempt of attempts) {
+            // scrape() never throws, so no try-catch needed
+            const result = await this.scrape({
+                url,
+                mode: attempt.mode,
+                timeout: options.timeout,
+                country: attempt.country,
+            });
+            lastResult = result;
+            // Success
+            if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
+                if (attemptResults.length > 0) {
+                    mcpLog('info', `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, 'scraper');
+                }
+                return result;
+            }
+            // 404 is a valid response, not an error
+            if (result.statusCode === 404) {
+                return result;
+            }
+            // Non-retryable errors - don't try other modes
+            if (result.error && !result.error.retryable) {
+                mcpLog('error', `Non-retryable error with ${attempt.description}: ${result.error.message}`, 'scraper');
+                return result;
+            }
+            // Collect failure reason and try next mode
+            attemptResults.push(`${attempt.description}: ${result.error?.message || result.statusCode}`);
+            mcpLog('warning', `Failed with ${attempt.description} (${result.statusCode}), trying next fallback...`, 'scraper');
+        }
+        // All fallbacks exhausted - return last result with aggregated error info
+        const errorMessage = `Failed after ${attempts.length} fallback modes: ${attemptResults.join('; ')}`;
+        return {
+            content: `Error: ${errorMessage}`,
+            statusCode: lastResult?.statusCode || 500,
+            credits: 0,
+            error: {
+                code: ErrorCode.SERVICE_UNAVAILABLE,
+                message: errorMessage,
+                retryable: false,
+            },
+        };
+    }
+    /**
+     * Scrape multiple URLs with batching
+     * NEVER throws - always returns results array
+     */
+    async scrapeMultiple(urls, options = {}) {
+        if (urls.length === 0) {
+            return [];
+        }
+        if (urls.length <= SCRAPER.BATCH_SIZE) {
+            return this.processBatch(urls, options);
+        }
+        const result = await this.batchScrape(urls, options);
+        return result.results;
+    }
+    /**
+     * Batch scrape with progress callback
+     * NEVER throws - uses Promise.allSettled internally
+     */
+    async batchScrape(urls, options = {}, onBatchComplete) {
+        const totalBatches = Math.ceil(urls.length / SCRAPER.BATCH_SIZE);
+        const allResults = [];
+        let rateLimitHits = 0;
+        mcpLog('info', `Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`, 'scraper');
+        for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
+            const startIdx = batchNum * SCRAPER.BATCH_SIZE;
+            const endIdx = Math.min(startIdx + SCRAPER.BATCH_SIZE, urls.length);
+            const batchUrls = urls.slice(startIdx, endIdx);
+            mcpLog('info', `Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`, 'scraper');
+            // Limit to 10 concurrent scrapes within each batch to prevent connection exhaustion
+            const batchResults = await pMapSettled(batchUrls, url => this.scrapeWithFallback(url, options), 10);
+            for (let i = 0; i < batchResults.length; i++) {
+                const result = batchResults[i];
+                const url = batchUrls[i] || '';
+                if (result.status === 'fulfilled') {
+                    const scrapeResult = result.value;
+                    allResults.push({ ...scrapeResult, url });
+                    // Track rate limits
+                    if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {
+                        rateLimitHits++;
+                    }
+                }
+                else {
+                    // This shouldn't happen since scrapeWithFallback never throws,
+                    // but handle it gracefully just in case
+                    const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
+                    mcpLog('error', `Unexpected rejection for ${url}: ${errorMsg}`, 'scraper');
+                    allResults.push({
+                        url,
+                        content: `Error: Unexpected failure - ${errorMsg}`,
+                        statusCode: 500,
+                        credits: 0,
+                        error: classifyError(result.reason),
+                    });
+                }
+            }
+            // Safe callback invocation
+            try {
+                onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
+            }
+            catch (callbackError) {
+                mcpLog('error', `onBatchComplete callback error: ${callbackError}`, 'scraper');
+            }
+            mcpLog('info', `Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`, 'scraper');
+            // Small delay between batches to avoid overwhelming the API
+            if (batchNum < totalBatches - 1) {
+                await sleep(500);
+            }
+        }
+        return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
+    }
+    /**
+     * Process a single batch of URLs
+     * NEVER throws
+     */
+    async processBatch(urls, options) {
+        // Limit to 10 concurrent scrapes to prevent connection exhaustion
+        const results = await pMapSettled(urls, url => this.scrapeWithFallback(url, options), 10);
+        return results.map((result, index) => {
+            const url = urls[index] || '';
+            if (result.status === 'fulfilled') {
+                return { ...result.value, url };
+            }
+            // Shouldn't happen, but handle gracefully
+            return {
+                url,
+                content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
+                statusCode: 500,
+                credits: 0,
+                error: classifyError(result.reason),
+            };
+        });
+    }
+}
+//# sourceMappingURL=scraper.js.map

package/dist/clients/scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAwB5C,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9C,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,qCAAqC;gBAC9E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,MAAM,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;wBACrH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,UAAU,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBACpF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC5H,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gDAAgD;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG;gBACH,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,UAAU,GAAG,MAAM,CAAC;YAEpB,UAAU;YACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,MAAM,EAAE,gBAAgB,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC9G,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,wCAAwC;YACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC9B,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,+CAA+C;YAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC5C,MAAM,CAAC,OAAO,EAAE,4BAA4B,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;gBACvG,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,2CAA2C;YAC3C,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,EAAE,eAAe,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,4BAA4B,EAAE,SAAS,CAAC,CAAC;QACrH,CAAC;QAED,0EAA0E;QAC1E,MAAM,YAAY,GAAG,gBAAgB,QAAQ,CAAC,MAAM,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpG,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,CAAC,MAAM,EAAE,8BAA8B,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,EAAE,SAAS,CAAC,CAAC;QAEzG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,EAAE,oBAAoB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEzG,oFAAoF;YACpF,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,SAAS,EACT,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAC5C,EAAE,CACH,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,MAAM,CAAC,OAAO,EAAE,4BAA4B,GAAG,KAAK,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;oBAE3E,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,MAAM,CAAC,OAAO,EAAE,mCAAmC,aAAa,EAAE,EAAE,SAAS,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,CAAC,MAAM,EAAE,mBAAmB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,EAAE,SAAS,CAAC,CAAC;YAEzH,4DAA4D;YAC5D,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,kEAAkE;QAClE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAE1F,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/clients/search.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Web Search Client
+ * Generic interface for web search via Google (Serper implementation)
+ * Implements robust error handling that NEVER crashes
+ */
+import { type StructuredError } from '../utils/errors.js';
+interface SearchResult {
+    title: string;
+    link: string;
+    snippet: string;
+    date?: string;
+    position: number;
+}
+export interface KeywordSearchResult {
+    keyword: string;
+    results: SearchResult[];
+    totalResults: number;
+    related: string[];
+    error?: StructuredError;
+}
+interface MultipleSearchResponse {
+    searches: KeywordSearchResult[];
+    totalKeywords: number;
+    executionTime: number;
+    error?: StructuredError;
+}
+export interface RedditSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+    date?: string;
+}
+export declare class SearchClient {
+    private apiKey;
+    private baseURL;
+    constructor(apiKey?: string);
+    /**
+     * Calculate backoff delay
+     */
+    private calculateBackoff;
+    /**
+     * Check if error is retryable
+     */
+    private isRetryable;
+    /**
+     * Search multiple keywords in parallel
+     * NEVER throws - always returns a valid response
+     */
+    searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
+    /**
+     * Search Reddit via Google (adds site:reddit.com automatically)
+     * NEVER throws - returns empty array on failure
+     */
+    searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
+    /**
+     * Search Reddit with multiple queries (bounded concurrency)
+     * NEVER throws - searchReddit never throws, pMap preserves order
+     */
+    searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
+}
+export {};
+//# sourceMappingURL=search.d.ts.map

package/dist/clients/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAI5B,UAAU,YAAY;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,UAAU,sBAAsB;IAC9B,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAiBD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAsHzE;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuDpF;;;OAGG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAc9G"}