@staticn0va/wigolo 0.1.0

package/dist/fetch/http-client.js

@@ -0,0 +1,146 @@
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+const RETRYABLE_STATUSES = new Set([429, 502, 503]);
+const RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);
+const REDIRECT_STATUSES = new Set([301, 302, 307, 308]);
+const DEFAULT_USER_AGENTS = [
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+];
+function getRotatingUserAgent(config) {
+    if (config.userAgent)
+        return config.userAgent;
+    return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];
+}
+function isRetryableError(err) {
+    if (err instanceof Error) {
+        const code = err.code;
+        if (code && RETRYABLE_ERROR_CODES.has(code))
+            return true;
+        // AbortSignal timeout throws DOMException with name TimeoutError
+        if (err.name === 'TimeoutError')
+            return true;
+    }
+    return false;
+}
+function backoffMs(attempt) {
+    return 500 * Math.pow(2, attempt) + Math.random() * 500;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export async function httpFetch(url, options = {}) {
+    const config = getConfig();
+    const logger = createLogger('fetch');
+    const maxRetries = config.fetchMaxRetries;
+    const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;
+    const maxRedirects = config.maxRedirects;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        if (attempt > 0) {
+            const delay = backoffMs(attempt - 1);
+            logger.debug('retrying after backoff', { attempt, delayMs: delay, url });
+            await sleep(delay);
+        }
+        try {
+            const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);
+            return result;
+        }
+        catch (err) {
+            lastError = err;
+            if (err instanceof HttpFetchError && !err.retryable) {
+                throw err;
+            }
+            const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);
+            if (!retryable || attempt >= maxRetries) {
+                throw err;
+            }
+            logger.warn('fetch failed, will retry', {
+                attempt,
+                url,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    throw lastError;
+}
+class HttpFetchError extends Error {
+    retryable;
+    constructor(message, retryable) {
+        super(message);
+        this.retryable = retryable;
+        this.name = 'HttpFetchError';
+    }
+}
+async function fetchWithRedirects(originalUrl, options, timeoutMs, maxRedirects, logger) {
+    const visited = new Set();
+    let currentUrl = originalUrl;
+    let redirectCount = 0;
+    while (true) {
+        if (visited.has(currentUrl)) {
+            throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);
+        }
+        visited.add(currentUrl);
+        logger.debug('fetching', { url: currentUrl, attempt: redirectCount });
+        const signal = AbortSignal.timeout(timeoutMs);
+        let response;
+        try {
+            const ua = getRotatingUserAgent(getConfig());
+            const mergedHeaders = { 'User-Agent': ua, ...options.headers };
+            response = await fetch(currentUrl, {
+                headers: mergedHeaders,
+                redirect: 'manual',
+                signal,
+            });
+        }
+        catch (err) {
+            const isTimeout = err instanceof Error && err.name === 'TimeoutError';
+            const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has(err.code ?? '');
+            const retryable = isTimeout || isConnErr;
+            throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });
+        }
+        if (REDIRECT_STATUSES.has(response.status)) {
+            const location = response.headers.get('location');
+            if (!location) {
+                throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);
+            }
+            redirectCount++;
+            if (redirectCount > maxRedirects) {
+                throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);
+            }
+            // Resolve relative redirects
+            currentUrl = new URL(location, currentUrl).toString();
+            continue;
+        }
+        if (RETRYABLE_STATUSES.has(response.status)) {
+            throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);
+        }
+        const contentType = response.headers.get('content-type') ?? '';
+        const headers = {};
+        response.headers.forEach((value, key) => {
+            headers[key] = value;
+        });
+        const isPdf = contentType.includes('application/pdf');
+        let html;
+        let rawBuffer;
+        if (isPdf) {
+            const arrayBuf = await response.arrayBuffer();
+            rawBuffer = Buffer.from(arrayBuf);
+            html = '';
+        }
+        else {
+            html = await response.text();
+        }
+        return {
+            url: originalUrl,
+            finalUrl: currentUrl,
+            html,
+            contentType,
+            statusCode: response.status,
+            headers,
+            rawBuffer,
+        };
+    }
+}
+//# sourceMappingURL=http-client.js.map

package/dist/fetch/http-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAiB5C,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACpD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExD,MAAM,mBAAmB,GAAG;IAC1B,uHAAuH;IACvH,iHAAiH;IACjH,uGAAuG;CACxG,CAAC;AAEF,SAAS,oBAAoB,CAAC,MAAqC;IACjE,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC;IAC9C,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,IAAI,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzD,iEAAiE;QACjE,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAA4B,EAAE;IACzE,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC;IAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAEzC,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACvF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,CAAC;YAEhB,IAAI,GAAG,YAAY,cAAc,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpD,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,YAAY,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAExF,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBACxC,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,OAAO;gBACP,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,cAAe,SAAQ,KAAK;IACa;IAA7C,YAAY,OAAe,EAAkB,SAAkB;QAC7D,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,cAAS,GAAT,SAAS,CAAS;QAE7D,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,KAAK,UAAU,kBAAkB,CAC/B,WAAmB,EACnB,OAAyB,EACzB,SAAiB,EACjB,YAAoB,EACpB,MAAuC;IAEvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,6BAA6B,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QAEtE,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/D,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACjC,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ;gBAClB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;YACtE,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,qBAAqB,CAAC,GAAG,CAAE,GAA6B,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC/G,MAAM,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC;YACzC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,cAAc,CAAC,uCAAuC,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;YAED,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,cAAc,CAAC,wBAAwB,YAAY,UAAU,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC/F,CAAC;YAED,6BAA6B;YAC7B,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,SAAS;QACX,CAAC;QAED,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,IAAY,CAAC;QACjB,IAAI,SAA6B,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC9C,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,UAAU;YACpB,IAAI;YACJ,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,OAAO;YACP,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/fetch/router.d.ts

@@ -0,0 +1,45 @@
+import type { RawFetchResult } from '../types.js';
+export interface RouterFetchOptions {
+    renderJs?: 'auto' | 'always' | 'never';
+    useAuth?: boolean;
+    headers?: Record<string, string>;
+    screenshot?: boolean;
+}
+export interface HttpClient {
+    fetch(url: string, options?: {
+        headers?: Record<string, string>;
+        timeoutMs?: number;
+    }): Promise<{
+        url: string;
+        finalUrl: string;
+        html: string;
+        contentType: string;
+        statusCode: number;
+        headers: Record<string, string>;
+        rawBuffer?: Buffer;
+    }>;
+}
+export interface BrowserPoolInterface {
+    fetchWithBrowser(url: string, options?: {
+        headers?: Record<string, string>;
+        storageStatePath?: string;
+        userDataDir?: string;
+        screenshot?: boolean;
+    }): Promise<RawFetchResult>;
+}
+interface DomainStats {
+    failureCount: number;
+    preferPlaywright: boolean;
+}
+export declare class SmartRouter {
+    private readonly httpClient;
+    private readonly browserPool;
+    private readonly domainMap;
+    constructor(httpClient: HttpClient, browserPool: BrowserPoolInterface);
+    fetch(url: string, options?: RouterFetchOptions): Promise<RawFetchResult>;
+    getDomainStats(domain: string): DomainStats | undefined;
+    private ensureStats;
+    private toRawFetchResult;
+}
+export {};
+//# sourceMappingURL=router.d.ts.map

package/dist/fetch/router.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACjE,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GACpH,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;gBAGzC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,oBAAoB;IAG9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC;IA6DnF,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}

package/dist/fetch/router.js

@@ -0,0 +1,89 @@
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+import { contentAppearsEmpty } from './content-check.js';
+import { getAuthOptions } from './auth.js';
+export class SmartRouter {
+    httpClient;
+    browserPool;
+    domainMap = new Map();
+    constructor(httpClient, browserPool) {
+        this.httpClient = httpClient;
+        this.browserPool = browserPool;
+    }
+    async fetch(url, options = {}) {
+        const { renderJs = 'auto', useAuth = false, headers, screenshot } = options;
+        const config = getConfig();
+        const logger = createLogger('fetch');
+        const threshold = config.browserFallbackThreshold;
+        const domain = new URL(url).hostname;
+        // Always Playwright for auth or explicit override
+        if (renderJs === 'always' || useAuth) {
+            const authOptions = useAuth ? (getAuthOptions() ?? {}) : {};
+            logger.debug('routing to playwright', { url, reason: useAuth ? 'auth' : 'render_js=always' });
+            return this.browserPool.fetchWithBrowser(url, { headers, screenshot, ...authOptions });
+        }
+        // HTTP only, no fallback
+        if (renderJs === 'never') {
+            logger.debug('routing to http (never)', { url });
+            const result = await this.httpClient.fetch(url, { headers });
+            this.ensureStats(domain);
+            return this.toRawFetchResult(result);
+        }
+        // auto: check if domain is already marked for Playwright
+        const stats = this.ensureStats(domain);
+        if (stats.preferPlaywright) {
+            logger.debug('routing to playwright (domain marked)', { url, domain });
+            return this.browserPool.fetchWithBrowser(url, { headers, screenshot });
+        }
+        // Try HTTP first
+        try {
+            const result = await this.httpClient.fetch(url, { headers });
+            // Check for SPA shell / empty content
+            if (contentAppearsEmpty(result.html)) {
+                logger.info('SPA shell detected, marking domain for playwright', { url, domain });
+                stats.preferPlaywright = true;
+                return this.browserPool.fetchWithBrowser(url, { headers, screenshot });
+            }
+            return this.toRawFetchResult(result);
+        }
+        catch (err) {
+            stats.failureCount++;
+            logger.warn('http fetch failed', {
+                url,
+                domain,
+                failureCount: stats.failureCount,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            if (stats.failureCount >= threshold) {
+                logger.info('failure threshold reached, marking domain for playwright', { url, domain, threshold });
+                stats.preferPlaywright = true;
+                return this.browserPool.fetchWithBrowser(url, { headers, screenshot });
+            }
+            throw err;
+        }
+    }
+    getDomainStats(domain) {
+        return this.domainMap.get(domain);
+    }
+    ensureStats(domain) {
+        let stats = this.domainMap.get(domain);
+        if (!stats) {
+            stats = { failureCount: 0, preferPlaywright: false };
+            this.domainMap.set(domain, stats);
+        }
+        return stats;
+    }
+    toRawFetchResult(result) {
+        return {
+            url: result.url,
+            finalUrl: result.finalUrl,
+            html: result.html,
+            contentType: result.contentType,
+            statusCode: result.statusCode,
+            method: 'http',
+            headers: result.headers,
+            rawBuffer: result.rawBuffer,
+        };
+    }
+}
+//# sourceMappingURL=router.js.map

package/dist/fetch/router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAqC3C,MAAM,OAAO,WAAW;IAIH;IACA;IAJF,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE5D,YACmB,UAAsB,EACtB,WAAiC;QADjC,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAsB;IACjD,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAA8B,EAAE;QACvD,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;QAC5E,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,wBAAwB,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAErC,kDAAkD;QAClD,IAAI,QAAQ,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,yBAAyB;QACzB,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,yDAAyD;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEvC,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAE7D,sCAAsC;YACtC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,mDAAmD,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClF,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAC/B,GAAG;gBACH,MAAM;gBACN,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,0DAA0D,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpG,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAc;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QAChC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;YACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,gBAAgB,CACtB,MAAgD;QAEhD,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+import { parseCommand } from './cli/index.js';
+import { runWarmup } from './cli/warmup.js';
+import { runDaemon } from './cli/daemon.js';
+import { runHealthCheck } from './cli/health.js';
+import { startServer } from './server.js';
+const { command, args } = parseCommand(process.argv.slice(2));
+switch (command) {
+    case 'warmup':
+        await runWarmup(args);
+        break;
+    case 'serve':
+        runDaemon(args);
+        break;
+    case 'health':
+        runHealthCheck();
+        break;
+    case 'mcp':
+        await startServer();
+        break;
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAE9D,QAAQ,OAAO,EAAE,CAAC;IAChB,KAAK,QAAQ;QACX,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM;IAER,KAAK,OAAO;QACV,SAAS,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM;IAER,KAAK,QAAQ;QACX,cAAc,EAAE,CAAC;QACjB,MAAM;IAER,KAAK,KAAK;QACR,MAAM,WAAW,EAAE,CAAC;QACpB,MAAM;AACV,CAAC"}

package/dist/logger.d.ts

@@ -0,0 +1,10 @@
+type Module = 'fetch' | 'search' | 'crawl' | 'cache' | 'extract' | 'searxng' | 'server' | 'cli' | 'jsonld';
+export interface Logger {
+    debug(msg: string, data?: Record<string, unknown>): void;
+    info(msg: string, data?: Record<string, unknown>): void;
+    warn(msg: string, data?: Record<string, unknown>): void;
+    error(msg: string, data?: Record<string, unknown>): void;
+}
+export declare function createLogger(module: Module): Logger;
+export {};
+//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAGA,KAAK,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;AAS3G,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1D;AAmBD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAiBnD"}

package/dist/logger.js

@@ -0,0 +1,39 @@
+import { getConfig } from './config.js';
+const LEVEL_PRIORITY = {
+    debug: 0,
+    info: 1,
+    warn: 2,
+    error: 3,
+};
+function writeJson(level, module, msg, data) {
+    const line = JSON.stringify({
+        ts: new Date().toISOString(),
+        level,
+        msg,
+        module,
+        ...(data ? { data } : {}),
+    });
+    process.stderr.write(line + '\n');
+}
+function writeText(level, module, msg, data) {
+    const ts = new Date().toISOString();
+    const dataStr = data ? ' ' + Object.entries(data).map(([k, v]) => `${k}=${v}`).join(' ') : '';
+    process.stderr.write(`[${ts}] ${level.toUpperCase().padEnd(5)} [${module}] ${msg}${dataStr}\n`);
+}
+export function createLogger(module) {
+    const config = getConfig();
+    const minPriority = LEVEL_PRIORITY[config.logLevel];
+    const write = config.logFormat === 'json' ? writeJson : writeText;
+    function log(level, msg, data) {
+        if (LEVEL_PRIORITY[level] >= minPriority) {
+            write(level, module, msg, data);
+        }
+    }
+    return {
+        debug: (msg, data) => log('debug', msg, data),
+        info: (msg, data) => log('info', msg, data),
+        warn: (msg, data) => log('warn', msg, data),
+        error: (msg, data) => log('error', msg, data),
+    };
+}
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,MAAM,cAAc,GAA6B;IAC/C,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AASF,SAAS,SAAS,CAAC,KAAe,EAAE,MAAc,EAAE,GAAW,EAAE,IAA8B;IAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAC5B,KAAK;QACL,GAAG;QACH,MAAM;QACN,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAC,CAAC;IACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,SAAS,CAAC,KAAe,EAAE,MAAc,EAAE,GAAW,EAAE,IAA8B;IAC7F,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,KAAK,GAAG,GAAG,OAAO,IAAI,CAAC,CAAC;AAClG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IAElE,SAAS,GAAG,CAAC,KAAe,EAAE,GAAW,EAAE,IAA8B;QACvE,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;YACzC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;QAC7C,IAAI,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC;QAC3C,IAAI,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC;QAC3C,KAAK,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;KAC9C,CAAC;AACJ,CAAC"}

package/dist/search/dedup.d.ts

@@ -0,0 +1,10 @@
+import type { RawSearchResult } from '../types.js';
+export interface MergedSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+    relevance_score: number;
+    engines: string[];
+}
+export declare function deduplicateResults(results: RawSearchResult[]): MergedSearchResult[];
+//# sourceMappingURL=dedup.d.ts.map

package/dist/search/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/search/dedup.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,kBAAkB,EAAE,CAkCnF"}

package/dist/search/dedup.js

@@ -0,0 +1,35 @@
+import { normalizeUrl } from '../cache/store.js';
+export function deduplicateResults(results) {
+    const urlMap = new Map();
+    for (const result of results) {
+        let normalized;
+        try {
+            normalized = normalizeUrl(result.url);
+        }
+        catch {
+            normalized = result.url;
+        }
+        const existing = urlMap.get(normalized);
+        if (existing) {
+            if (result.relevance_score > existing.relevance_score) {
+                existing.relevance_score = result.relevance_score;
+                existing.title = result.title;
+                existing.snippet = result.snippet;
+            }
+            if (!existing.engines.includes(result.engine)) {
+                existing.engines.push(result.engine);
+            }
+        }
+        else {
+            urlMap.set(normalized, {
+                title: result.title,
+                url: result.url,
+                snippet: result.snippet,
+                relevance_score: result.relevance_score,
+                engines: [result.engine],
+            });
+        }
+    }
+    return [...urlMap.values()].sort((a, b) => b.relevance_score - a.relevance_score);
+}
+//# sourceMappingURL=dedup.js.map

package/dist/search/dedup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/search/dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAWjD,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;IAErD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACH,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC;QAC1B,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,eAAe,GAAG,QAAQ,CAAC,eAAe,EAAE,CAAC;gBACtD,QAAQ,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;gBAClD,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9B,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YACpC,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,OAAO,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;AACpF,CAAC"}

package/dist/search/engines/bing.d.ts

@@ -0,0 +1,7 @@
+import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
+export declare class BingEngine implements SearchEngine {
+    name: string;
+    search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
+    parseResults(html: string, maxResults: number): RawSearchResult[];
+}
+//# sourceMappingURL=bing.d.ts.map

package/dist/search/engines/bing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bing.d.ts","sourceRoot":"","sources":["../../../src/search/engines/bing.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAKzF,qBAAa,UAAW,YAAW,YAAY;IAC7C,IAAI,SAAU;IAER,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAuB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CA4BlE"}

package/dist/search/engines/bing.js

@@ -0,0 +1,48 @@
+import { parseHTML } from 'linkedom';
+import { createLogger } from '../../logger.js';
+const log = createLogger('search');
+export class BingEngine {
+    name = 'bing';
+    async search(query, options = {}) {
+        const timeoutMs = options.timeoutMs ?? 10000;
+        const maxResults = options.maxResults ?? 10;
+        const params = new URLSearchParams({ q: query });
+        const url = `https://www.bing.com/search?${params}`;
+        log.debug('scraping bing', { query });
+        const response = await fetch(url, {
+            signal: AbortSignal.timeout(timeoutMs),
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+                'Accept-Language': options.language ?? 'en-US,en;q=0.9',
+            },
+        });
+        if (!response.ok)
+            throw new Error(`Bing returned ${response.status}`);
+        const html = await response.text();
+        return this.parseResults(html, maxResults);
+    }
+    parseResults(html, maxResults) {
+        const { document } = parseHTML(html);
+        const results = [];
+        const items = document.querySelectorAll('li.b_algo');
+        const total = Math.min(items.length, maxResults);
+        for (let i = 0; i < total; i++) {
+            const item = items[i];
+            const link = item.querySelector('h2 a');
+            const snippetEl = item.querySelector('.b_lineclamp2, .b_lineclamp3, .b_caption p');
+            const href = link?.getAttribute('href');
+            const title = link?.textContent?.trim();
+            if (href && title) {
+                results.push({
+                    title,
+                    url: href,
+                    snippet: snippetEl?.textContent?.trim() ?? '',
+                    relevance_score: 1 - i / Math.max(items.length, 1),
+                    engine: 'bing',
+                });
+            }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=bing.js.map

package/dist/search/engines/bing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bing.js","sourceRoot":"","sources":["../../../src/search/engines/bing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,OAAO,UAAU;IACrB,IAAI,GAAG,MAAM,CAAC;IAEd,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA+B,EAAE;QAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC;QAEpD,GAAG,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAEtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;YACtC,OAAO,EAAE;gBACP,YAAY,EAAE,uHAAuH;gBACrI,iBAAiB,EAAE,OAAO,CAAC,QAAQ,IAAI,gBAAgB;aACxD;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,UAAkB;QAC3C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,4CAA4C,CAAC,CAAC;YAEnF,MAAM,IAAI,GAAG,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAExC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,GAAG,EAAE,IAAI;oBACT,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;oBAC7C,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClD,MAAM,EAAE,MAAM;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/search/engines/duckduckgo.d.ts

@@ -0,0 +1,7 @@
+import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
+export declare class DuckDuckGoEngine implements SearchEngine {
+    name: string;
+    search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
+    parseResults(html: string, maxResults: number): RawSearchResult[];
+}
+//# sourceMappingURL=duckduckgo.d.ts.map

package/dist/search/engines/duckduckgo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../../src/search/engines/duckduckgo.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAWzF,qBAAa,gBAAiB,YAAW,YAAY;IACnD,IAAI,SAAgB;IAEd,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CA4BlE"}

package/dist/search/engines/duckduckgo.js

@@ -0,0 +1,50 @@
+import { parseHTML } from 'linkedom';
+import { createLogger } from '../../logger.js';
+const log = createLogger('search');
+const USER_AGENTS = [
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+];
+export class DuckDuckGoEngine {
+    name = 'duckduckgo';
+    async search(query, options = {}) {
+        const timeoutMs = options.timeoutMs ?? 10000;
+        const maxResults = options.maxResults ?? 10;
+        const params = new URLSearchParams({ q: query });
+        const url = `https://lite.duckduckgo.com/lite/?${params}`;
+        log.debug('scraping duckduckgo', { query });
+        const response = await fetch(url, {
+            signal: AbortSignal.timeout(timeoutMs),
+            headers: { 'User-Agent': USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)] },
+        });
+        if (!response.ok)
+            throw new Error(`DDG returned ${response.status}`);
+        const html = await response.text();
+        return this.parseResults(html, maxResults);
+    }
+    parseResults(html, maxResults) {
+        const { document } = parseHTML(html);
+        const results = [];
+        const links = document.querySelectorAll('a.result-link');
+        const snippets = document.querySelectorAll('.result-snippet');
+        const total = Math.min(links.length, maxResults);
+        for (let i = 0; i < total; i++) {
+            const link = links[i];
+            const snippet = snippets[i];
+            const href = link?.getAttribute('href');
+            const title = link?.textContent?.trim();
+            if (href && title) {
+                results.push({
+                    title,
+                    url: href,
+                    snippet: snippet?.textContent?.trim() ?? '',
+                    relevance_score: 1 - i / Math.max(links.length, 1),
+                    engine: 'duckduckgo',
+                });
+            }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=duckduckgo.js.map

package/dist/search/engines/duckduckgo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../../src/search/engines/duckduckgo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,wEAAwE;CACzE,CAAC;AAEF,MAAM,OAAO,gBAAgB;IAC3B,IAAI,GAAG,YAAY,CAAC;IAEpB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA+B,EAAE;QAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,qCAAqC,MAAM,EAAE,CAAC;QAE1D,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;YACtC,OAAO,EAAE,EAAE,YAAY,EAAE,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE;SACvF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,YAAY,CAAC,IAAY,EAAE,UAAkB;QAC3C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAExC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,GAAG,EAAE,IAAI;oBACT,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;oBAC3C,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClD,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/search/engines/startpage.d.ts

@@ -0,0 +1,7 @@
+import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
+export declare class StartpageEngine implements SearchEngine {
+    name: string;
+    search(query: string, options?: SearchEngineOptions): Promise<RawSearchResult[]>;
+    parseResults(html: string, maxResults: number): RawSearchResult[];
+}
+//# sourceMappingURL=startpage.d.ts.map

package/dist/search/engines/startpage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"startpage.d.ts","sourceRoot":"","sources":["../../../src/search/engines/startpage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAKzF,qBAAa,eAAgB,YAAW,YAAY;IAClD,IAAI,SAAe;IAEb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB1F,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,eAAe,EAAE;CA4BlE"}