@staticn0va/wigolo 0.4.0 → 0.6.0

package/dist/fetch/lightpanda.js

@@ -0,0 +1,177 @@
+import { chromium } from 'playwright';
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+import { isCDPReachable } from './cdp-client.js';
+import { getDatabase } from '../cache/db.js';
+const log = createLogger('fetch');
+export function shouldUseLightpanda(domain) {
+    try {
+        const config = getConfig();
+        if (!config.lightpandaEnabled || !config.lightpandaUrl) {
+            return false;
+        }
+        const db = getDatabase();
+        const row = db.prepare('SELECT failure_count, prefer_chromium FROM lightpanda_routing WHERE domain = ?').get(domain);
+        if (!row)
+            return true;
+        if (row.prefer_chromium === 1) {
+            log.debug('domain prefers chromium over lightpanda', { domain });
+            return false;
+        }
+        if (row.failure_count >= config.lightpandaFailureThreshold) {
+            log.debug('domain failure threshold reached for lightpanda', {
+                domain,
+                failures: row.failure_count,
+                threshold: config.lightpandaFailureThreshold,
+            });
+            return false;
+        }
+        return true;
+    }
+    catch (err) {
+        log.warn('shouldUseLightpanda check failed, defaulting to chromium', {
+            domain,
+            error: String(err),
+        });
+        return false;
+    }
+}
+export function recordSuccess(domain) {
+    try {
+        const db = getDatabase();
+        db.prepare(`
+      INSERT INTO lightpanda_routing (domain, success_count, last_success, last_updated)
+      VALUES (?, 1, datetime('now'), datetime('now'))
+      ON CONFLICT(domain) DO UPDATE SET
+        success_count = success_count + 1,
+        last_success = datetime('now'),
+        last_updated = datetime('now')
+    `).run(domain);
+        log.debug('recorded lightpanda success', { domain });
+    }
+    catch (err) {
+        log.warn('failed to record lightpanda success', { domain, error: String(err) });
+    }
+}
+export function recordFailure(domain) {
+    try {
+        const config = getConfig();
+        const db = getDatabase();
+        db.prepare(`
+      INSERT INTO lightpanda_routing (domain, failure_count, last_failure, last_updated)
+      VALUES (?, 1, datetime('now'), datetime('now'))
+      ON CONFLICT(domain) DO UPDATE SET
+        failure_count = failure_count + 1,
+        last_failure = datetime('now'),
+        last_updated = datetime('now'),
+        prefer_chromium = CASE
+          WHEN failure_count + 1 >= ?
+          THEN 1
+          ELSE prefer_chromium
+        END
+    `).run(domain, config.lightpandaFailureThreshold);
+        log.debug('recorded lightpanda failure', { domain });
+    }
+    catch (err) {
+        log.warn('failed to record lightpanda failure', { domain, error: String(err) });
+    }
+}
+export function getDomainStats(domain) {
+    try {
+        const db = getDatabase();
+        const row = db.prepare('SELECT * FROM lightpanda_routing WHERE domain = ?').get(domain);
+        if (!row)
+            return null;
+        return {
+            domain: row.domain,
+            successCount: row.success_count,
+            failureCount: row.failure_count,
+            preferChromium: row.prefer_chromium === 1,
+            lastSuccess: row.last_success ?? undefined,
+            lastFailure: row.last_failure ?? undefined,
+        };
+    }
+    catch (err) {
+        log.warn('getDomainStats failed', { domain, error: String(err) });
+        return null;
+    }
+}
+export class LightpandaAdapter {
+    browser = null;
+    url;
+    constructor(url) {
+        const config = getConfig();
+        this.url = url ?? config.lightpandaUrl ?? 'http://localhost:9222';
+    }
+    async connect() {
+        try {
+            const reachable = await isCDPReachable(this.url);
+            if (!reachable) {
+                log.debug('lightpanda CDP not reachable', { url: this.url });
+                return { connected: false, error: 'CDP endpoint not reachable' };
+            }
+            this.browser = await chromium.connectOverCDP(this.url);
+            log.info('connected to lightpanda via CDP', { url: this.url });
+            return { connected: true };
+        }
+        catch (err) {
+            log.warn('lightpanda connection failed', {
+                url: this.url,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            this.browser = null;
+            return {
+                connected: false,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    }
+    async disconnect() {
+        try {
+            if (this.browser) {
+                await this.browser.close();
+                this.browser = null;
+                log.debug('disconnected from lightpanda');
+            }
+        }
+        catch (err) {
+            log.warn('lightpanda disconnect error', { error: String(err) });
+            this.browser = null;
+        }
+    }
+    async isHealthy() {
+        try {
+            if (!this.browser)
+                return false;
+            if (typeof this.browser.isConnected === 'function') {
+                if (!this.browser.isConnected())
+                    return false;
+            }
+            return await isCDPReachable(this.url);
+        }
+        catch {
+            return false;
+        }
+    }
+    async getContext() {
+        try {
+            if (!this.browser) {
+                const result = await this.connect();
+                if (!result.connected || !this.browser)
+                    return null;
+            }
+            const contexts = this.browser.contexts();
+            if (contexts.length > 0)
+                return contexts[0];
+            return await this.browser.newContext();
+        }
+        catch (err) {
+            log.warn('failed to get lightpanda context', { error: String(err) });
+            return null;
+        }
+    }
+    getBrowser() {
+        return this.browser;
+    }
+}
+//# sourceMappingURL=lightpanda.js.map

package/dist/fetch/lightpanda.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lightpanda.js","sourceRoot":"","sources":["../../src/fetch/lightpanda.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAgBlC,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CACpB,gFAAgF,CACjF,CAAC,GAAG,CAAC,MAAM,CAAmE,CAAC;QAEhF,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,IAAI,GAAG,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;YAC9B,GAAG,CAAC,KAAK,CAAC,yCAAyC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;YAC3D,GAAG,CAAC,KAAK,CAAC,iDAAiD,EAAE;gBAC3D,MAAM;gBACN,QAAQ,EAAE,GAAG,CAAC,aAAa;gBAC3B,SAAS,EAAE,MAAM,CAAC,0BAA0B;aAC7C,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,0DAA0D,EAAE;YACnE,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;SACnB,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;QACzB,EAAE,CAAC,OAAO,CAAC;;;;;;;KAOV,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEf,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;QAEzB,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;KAYV,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAElD,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CACpB,mDAAmD,CACpD,CAAC,GAAG,CAAC,MAAM,CAOC,CAAC;QAEd,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,YAAY,EAAE,GAAG,CAAC,aAAa;YAC/B,YAAY,EAAE,GAAG,CAAC,aAAa;YAC/B,cAAc,EAAE,GAAG,CAAC,eAAe,KAAK,CAAC;YACzC,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,SAAS;YAC1C,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,SAAS;SAC3C,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,OAAO,GAAmB,IAAI,CAAC;IAC/B,GAAG,CAAS;IAEpB,YAAY,GAAY;QACtB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,MAAM,CAAC,aAAa,IAAI,uBAAuB,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC7D,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC;YACnE,CAAC;YAED,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvD,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,8BAA8B,EAAE;gBACvC,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO;gBACL,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO,KAAK,CAAC;YAEhC,IAAI,OAAQ,IAAI,CAAC,OAAe,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBAC5D,IAAI,CAAE,IAAI,CAAC,OAAe,CAAC,WAAW,EAAE;oBAAE,OAAO,KAAK,CAAC;YACzD,CAAC;YAED,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,OAAO;oBAAE,OAAO,IAAI,CAAC;YACtD,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5C,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}

package/dist/fetch/router.d.ts

@@ -5,6 +5,7 @@ export interface RouterFetchOptions {
     headers?: Record<string, string>;
     screenshot?: boolean;
     actions?: BrowserAction[];
+    force_refresh?: boolean;
 }
 export interface HttpClient {
     fetch(url: string, options?: {

package/dist/fetch/router.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACjE,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChK,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;gBAGzC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,oBAAoB;IAG9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC;IAoEnF,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CACH,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACjE,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,CACd,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAChK,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;gBAGzC,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,oBAAoB;IAG9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC;IAoEnF,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIvD,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,gBAAgB;CAczB"}

package/dist/fetch/router.js.map

@@ -1 +1 @@
-{"version":3,"file":"router.js","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAsC3C,MAAM,OAAO,WAAW;IAIH;IACA;IAJF,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE5D,YACmB,UAAsB,EACtB,WAAiC;QADjC,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAsB;IACjD,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAA8B,EAAE;QACvD,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QACrF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,wBAAwB,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAErC,uEAAuE;QACvE,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC1E,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAClG,CAAC;QAED,kDAAkD;QAClD,IAAI,QAAQ,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,yBAAyB;QACzB,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,yDAAyD;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEvC,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAE7D,sCAAsC;YACtC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,mDAAmD,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClF,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAC/B,GAAG;gBACH,MAAM;gBACN,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,0DAA0D,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpG,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAc;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QAChC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;YACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,gBAAgB,CACtB,MAAgD;QAEhD,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../../src/fetch/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAuC3C,MAAM,OAAO,WAAW;IAIH;IACA;IAJF,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE5D,YACmB,UAAsB,EACtB,WAAiC;QADjC,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAsB;IACjD,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAA8B,EAAE;QACvD,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QACrF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,wBAAwB,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAErC,uEAAuE;QACvE,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC1E,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAClG,CAAC;QAED,kDAAkD;QAClD,IAAI,QAAQ,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,yBAAyB;QACzB,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,yDAAyD;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEvC,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAE7D,sCAAsC;YACtC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,mDAAmD,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClF,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAC/B,GAAG;gBACH,MAAM;gBACN,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,0DAA0D,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACpG,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAc;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QAChC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;YACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,gBAAgB,CACtB,MAAgD;QAEhD,OAAO;YACL,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;CACF"}

package/dist/instructions.d.ts

@@ -14,13 +14,16 @@
  * Parameter schemas (types, enums, required/optional) belong on the JSON
  * Schema, not here. Installation/configuration is for humans, not LLMs.
  */
-export declare const WIGOLO_INSTRUCTIONS = "Wigolo is a local-first web access layer: search the open web, fetch pages, crawl sites, and extract structured data. All results land in a local SQLite cache that persists across sessions.\n\n## When to use which tool\n\n- `search` \u2014 you need information on a topic but do not have a URL yet.\n- `fetch` \u2014 you already have a specific URL to read.\n- `crawl` \u2014 you need multiple pages from the same site (docs, wikis, references).\n- `cache` \u2014 you want to know if the content is already on disk from an earlier read.\n- `extract` \u2014 you need specific data points (tables, metadata, schema-shaped fields) rather than a whole page as markdown.\n\n## Check the cache before going to the network\n\nBefore every `search` or `fetch`, consider a `cache` call with the query text or URL pattern. Pages read in this or a prior session return instantly with their full markdown \u2014 no network, no rate limits.\n\n## Pick the right strategy\n\n- For documentation sites, prefer `crawl` with `strategy: \"sitemap\"` \u2014 it is faster and more complete than BFS because it reads sitemap.xml directly.\n- When you only need to discover what pages exist on a site, use `crawl` with `strategy: \"map\"`. It returns URLs only, no content, and is far cheaper than a full crawl. Follow up with targeted `fetch` calls.\n- For structured data (prices, specs, listings, table rows), use `extract` with `mode: \"schema\"` or `mode: \"tables\"`. Reach for `fetch` only when you want the whole page as markdown.\n\n## Scope searches, do not just broaden queries\n\n`search` accepts `include_domains` (e.g. `[\"react.dev\", \"developer.mozilla.org\"]`) and a `category` such as `\"docs\"`, `\"code\"`, `\"news\"`, or `\"papers\"`. A scoped query usually beats a broader query with post-filtering.\n\n## Performance\n\n- `max_results: 3` for focused lookups; `5` is the default; `10+` only for broad research.\n- `fetch` with `section: \"Heading Name\"` returns just the content under that heading. Cheaper and more relevant than the whole page.\n- Repeated fetches of the same URL are free \u2014 served directly from the SQLite cache.\n\n## Capabilities worth knowing\n\n- Localhost URLs work: `http://localhost:3000`, `http://127.0.0.1:8080`, and similar. Useful for reading local dev servers and internal docs.\n- `use_auth: true` on `fetch` and `crawl` reuses the user's configured browser session for pages behind a login.\n- `cache` accepts FTS5 query syntax (`AND`, `OR`, `NOT`, `\"exact phrase\"`) for precise lookups.\n- `crawl` accepts regex `include_patterns` and `exclude_patterns` to stay inside a section of a large site.";
+export declare const WIGOLO_INSTRUCTIONS = "Wigolo is a local-first web access layer: search the open web, fetch pages, crawl sites, extract structured data, find related content, run multi-step research, and execute agent-driven data gathering. All results land in a local SQLite cache that persists across sessions.\n\n## When to use which tool\n\n- `search` -- you need information on a topic but do not have a URL yet. Pass a query string or an array of 3-5 semantically varied keyword forms for broader coverage.\n- `fetch` -- you already have a specific URL to read.\n- `crawl` -- you need multiple pages from the same site (docs, wikis, references).\n- `cache` -- you want to know if the content is already on disk from an earlier read.\n- `extract` -- you need specific data points (tables, metadata, schema-shaped fields) rather than a whole page as markdown.\n- `find_similar` -- you have a URL or concept and want related content from the cache or web. Useful for \"more like this\" discovery.\n- `research` -- you have a complex question that needs multi-step investigation: question decomposition, parallel search, source synthesis into a report. Set `depth` to control thoroughness.\n- `agent` -- you need to gather structured or unstructured data from multiple sources based on a natural-language prompt. Provides full step transparency.\n\n## Routing by intent\n\n| Intent | Tool | Key parameters |\n|--------|------|----------------|\n| Documentation lookup | `search` | `include_domains: [\"react.dev\", \"docs.python.org\"]`, `category: \"docs\"` |\n| Error debugging | `search` | exact error string as query, `category: \"code\"` |\n| Library research | `crawl` | seed URL of docs site, `strategy: \"sitemap\"`, then `cache` for later queries |\n| Related content | `find_similar` | `url` of a known good page, or `concept` as free text |\n| Direct answer | `search` | `format: \"answer\"` for a synthesized direct response |\n| Comprehensive research | `research` | `depth: \"comprehensive\"`, optional `include_domains` to scope |\n| Data gathering | `agent` | natural-language `prompt`, optional `schema` for structured output |\n| Structured extraction | `extract` | `mode: \"schema\"` with a JSON Schema, or `mode: \"tables\"` |\n| Site inventory | `crawl` | `strategy: \"map\"` for URL-only discovery, no content fetched |\n\n## Rapidly changing content\n\nFor news, prices, status pages, release notes, or any content that changes frequently, bypass the cache with `force_refresh: true`:\n\n  search({ query: \"...\", force_refresh: true })\n  fetch({ url: \"...\", force_refresh: true })\n\nWhen freshness matters more than speed, use `force_refresh`. When speed matters more than freshness (documentation, tutorials, reference pages), let the cache work -- it is much faster.\n\n## Check the cache before going to the network\n\nBefore every `search` or `fetch`, consider a `cache` call with the query text or URL pattern. Pages read in this or a prior session return instantly with their full markdown -- no network, no rate limits. The `research` and `agent` tools check the cache internally, so you do not need a separate call for those.\n\n## Multi-query search strategy\n\nFor broad or exploratory queries, pass an array of 3-5 semantically varied keyword forms rather than a single natural-language question. Example: instead of \"how does React handle state management\", pass `[\"react state management\", \"useState useReducer patterns\", \"react hooks state\", \"react context vs redux\"]`. The search tool deduplicates across sub-queries automatically.\n\n## Pick the right strategy\n\n- For documentation sites, prefer `crawl` with `strategy: \"sitemap\"` -- it is faster and more complete than BFS because it reads sitemap.xml directly.\n- When you only need to discover what pages exist on a site, use `crawl` with `strategy: \"map\"`. It returns URLs only, no content, and is far cheaper than a full crawl. Follow up with targeted `fetch` calls.\n- For structured data (prices, specs, listings, table rows), use `extract` with `mode: \"schema\"` or `mode: \"tables\"`. Reach for `fetch` only when you want the whole page as markdown.\n- For complex questions requiring synthesis from multiple sources, use `research` instead of manually chaining `search` + `fetch` calls.\n- For natural-language data gathering tasks (\"find the pricing for the top 5 CRM tools\"), use `agent` with an optional `schema` to structure the output.\n\n## Scope searches, do not just broaden queries\n\n`search` accepts `include_domains` (e.g. `[\"react.dev\", \"developer.mozilla.org\"]`) and a `category` such as `\"docs\"`, `\"code\"`, `\"news\"`, or `\"papers\"`. A scoped query usually beats a broader query with post-filtering.\n\n## Performance\n\n- `max_results: 3` for focused lookups; `5` is the default; `10+` only for broad research.\n- `fetch` with `section: \"Heading Name\"` returns just the content under that heading. Cheaper and more relevant than the whole page.\n- Repeated fetches of the same URL are free -- served directly from the SQLite cache.\n- `research` with `depth: \"quick\"` is fast (~15s) and sufficient for most factual questions. Reserve `\"comprehensive\"` for topics requiring deep investigation.\n- `agent` respects `max_pages` (default 10) and `max_time_ms` (default 60s) to bound resource usage.\n\n## Capabilities worth knowing\n\n- Localhost URLs work: `http://localhost:3000`, `http://127.0.0.1:8080`, and similar. Useful for reading local dev servers and internal docs.\n- `use_auth: true` on `fetch` and `crawl` reuses the user's configured browser session for pages behind a login.\n- `cache` accepts FTS5 query syntax (`AND`, `OR`, `NOT`, `\"exact phrase\"`) for precise lookups.\n- `crawl` accepts regex `include_patterns` and `exclude_patterns` to stay inside a section of a large site.\n- `find_similar` uses cached embeddings when available -- no network call needed if the content has been seen before.\n- `research` and `agent` use MCP requestSampling for intelligent decomposition and synthesis when the client supports it. Without sampling support, they return raw sources in context format.";
 export declare const TOOL_DESCRIPTIONS: {
-    readonly fetch: "Fetch a single URL and return clean markdown. Use when you have a specific URL to read. Automatically detects if JavaScript rendering is needed.\n\nKey parameters:\n- section: extract content under a specific heading (e.g., section: \"API Reference\") — faster than reading the whole page\n- use_auth: true to use stored browser session for authenticated/private pages\n- render_js: \"auto\" (default, detects JS need), \"always\" (force browser), \"never\" (HTTP only, fastest)\n- headers: custom HTTP headers if needed\n\nReturns title, markdown content, links, images, and metadata. Result is cached locally — subsequent fetches of the same URL return instantly. Works with localhost URLs (localhost:3000, etc.) for reading local dev servers.";
-    readonly search: "Search the web and return full markdown content from top results. Use for finding information on any topic — returns extracted page content, not just snippets.\n\nKey parameters:\n- include_domains/exclude_domains: scope results to specific sites (e.g., include_domains: [\"react.dev\"])\n- category: \"general\", \"news\", \"code\", \"docs\", \"papers\" — filters by content type\n- from_date/to_date: ISO dates for time-bounded queries\n- max_results: default 5. Use 3 for focused queries, 10+ for research.\n\nResults include title, URL, relevance_score, and full markdown_content per result. Previously fetched pages are served from local cache.";
-    readonly crawl: "Crawl a website starting from a URL and return content from multiple pages. Use for indexing documentation sites, wikis, or any multi-page resource.\n\nKey parameters:\n- strategy: \"bfs\" (breadth-first, default), \"dfs\" (depth-first), \"sitemap\" (use sitemap.xml — fastest for doc sites), \"map\" (URL discovery only, no content — fastest for scoping a site)\n- max_depth: how many links deep to follow (default 2)\n- max_pages: maximum pages to fetch (default 20)\n- include_patterns/exclude_patterns: regex filters on URLs\n\nReturns an array of pages with title, markdown, and depth. Content is deduplicated across pages (repeated nav/headers/footers stripped). All pages are cached for later cache queries.";
-    readonly cache: "Search previously fetched content without hitting the network. Use before searching the web — if relevant content was already fetched or crawled, this returns it instantly.\n\nKey parameters:\n- query: full-text search over cached markdown and titles (supports FTS5 syntax: AND, OR, NOT, \"phrase match\")\n- url_pattern: glob filter on URLs (e.g., \"*react.dev*\")\n- since: ISO date — only results cached after this date\n- stats: true to get cache size, entry count, oldest/newest dates\n- clear: true to delete matching entries\n\nReturns matching cached pages with full markdown content. Cache persists across sessions in local SQLite.";
-    readonly extract: "Extract structured data from a URL or raw HTML. Use when you need specific data points, tables, or metadata rather than full page markdown.\n\nKey parameters:\n- mode: \"selector\" (CSS selector → text), \"tables\" (HTML tables → JSON rows), \"metadata\" (title/author/date/description), \"schema\" (JSON Schema → heuristic field extraction)\n- css_selector: required for mode=\"selector\" — any valid CSS selector\n- schema: for mode=\"schema\", a JSON Schema object describing the fields to extract\n- multiple: true to return array of all matches (mode=\"selector\" only)\n\nFor mode=\"tables\", returns array of table objects with headers and row data. For mode=\"schema\", pass { price: \"string\", name: \"string\" } and get structured fields extracted from the page.";
+    readonly fetch: "Fetch a single URL and return clean markdown. Use when you have a specific URL to read. Automatically detects if JavaScript rendering is needed.\n\nKey parameters:\n- section: extract content under a specific heading (e.g., section: \"API Reference\") -- faster than reading the whole page\n- use_auth: true to use stored browser session for authenticated/private pages\n- render_js: \"auto\" (default, detects JS need), \"always\" (force browser), \"never\" (HTTP only, fastest)\n- headers: custom HTTP headers if needed\n- force_refresh: true to bypass cache and fetch fresh content from the network\n\nReturns title, markdown content, links, images, and metadata. Result is cached locally -- subsequent fetches of the same URL return instantly. Works with localhost URLs (localhost:3000, etc.) for reading local dev servers.\n\nUse force_refresh: true for pages that change frequently (news sites, changelogs, dashboards, API status pages). By default, previously fetched pages are served from local cache for speed.";
+    readonly search: "Search the web and return full markdown content from top results. Use for finding information on any topic -- returns extracted page content, not just snippets.\n\nKey parameters:\n- query: a search string, or an array of 3-5 semantically varied keyword forms for broader coverage. Arrays are deduplicated and merged automatically.\n- include_domains/exclude_domains: scope results to specific sites (e.g., include_domains: [\"react.dev\"])\n- category: \"general\", \"news\", \"code\", \"docs\", \"papers\" -- filters by content type\n- from_date/to_date: ISO dates for time-bounded queries\n- max_results: default 5. Use 3 for focused queries, 10+ for research.\n- format: \"full\" (default, structured JSON), \"context\" (single token-budgeted string for LLM injection), \"answer\" (synthesized direct answer via requestSampling), \"stream_answer\" (same as answer, with MCP progress notifications emitted between pipeline phases)\n- force_refresh: true to bypass all caches (search results and page content)\n\nThe \"answer\" format uses the MCP client's sampling capability to synthesize a direct response from search results. If sampling is not supported, falls back to \"context\" format. \"stream_answer\" emits notifications/progress messages at each pipeline phase (search, fetch, synthesize) when the client provides a progressToken via request._meta — token-level streaming of the LLM response is not supported by MCP sampling, so the answer itself still arrives as one block.\n\nResults include title, URL, relevance_score, and full markdown_content per result. Previously fetched pages are served from local cache.\n\nUse force_refresh: true when you need current information that may have changed since the last search. Default behavior uses cached results when available.";
+    readonly crawl: "Crawl a website starting from a URL and return content from multiple pages. Use for indexing documentation sites, wikis, or any multi-page resource.\n\nKey parameters:\n- strategy: \"bfs\" (breadth-first, default), \"dfs\" (depth-first), \"sitemap\" (use sitemap.xml -- fastest for doc sites), \"map\" (URL discovery only, no content -- fastest for scoping a site)\n- max_depth: how many links deep to follow (default 2)\n- max_pages: maximum pages to fetch (default 20)\n- include_patterns/exclude_patterns: regex filters on URLs\n\nReturns an array of pages with title, markdown, and depth. Content is deduplicated across pages (repeated nav/headers/footers stripped). All pages are cached for later cache queries.";
+    readonly cache: "Search previously fetched content without hitting the network. Use before searching the web -- if relevant content was already fetched or crawled, this returns it instantly.\n\nKey parameters:\n- query: full-text search over cached markdown and titles (supports FTS5 syntax: AND, OR, NOT, \"phrase match\")\n- url_pattern: glob filter on URLs (e.g., \"*example.com*\")\n- since: ISO date -- only results cached after this date\n- stats: true to get cache size, entry count, oldest/newest dates\n- clear: true to delete matching entries\n\nReturns matching cached pages with full markdown content. Cache persists across sessions in local SQLite.";
+    readonly extract: "Extract structured data from a URL or raw HTML. Use when you need specific data points, tables, or metadata rather than full page markdown.\n\nKey parameters:\n- mode: \"selector\" (CSS selector -> text), \"tables\" (HTML tables -> JSON rows), \"metadata\" (title/author/date/description), \"schema\" (JSON Schema -> heuristic field extraction)\n- css_selector: required for mode=\"selector\" -- any valid CSS selector\n- schema: for mode=\"schema\", a JSON Schema object describing the fields to extract\n- multiple: true to return array of all matches (mode=\"selector\" only)\n\nFor mode=\"tables\", returns array of table objects with headers and row data. For mode=\"schema\", pass { price: \"string\", name: \"string\" } and get structured fields extracted from the page.";
+    readonly find_similar: "Find content related to a URL or concept. Use when you have a known-good page or topic and want to discover similar resources from the cache or web.\n\nKey parameters:\n- url: a URL to find content similar to. The page's content and embeddings are used for similarity matching.\n- concept: free-text description of what you want similar content for. Use when you do not have a specific URL.\n- max_results: number of similar items to return (default 5)\n- include_cached: true (default) to search the local cache first, false to skip cache and search the web only\n- threshold: minimum similarity score (0-1, default 0.5) -- higher values return fewer, more relevant results\n\nProvide either url or concept (not both). Results are ranked by cosine similarity of embeddings. Cached content with embeddings is searched first for instant results. If fewer than max_results are found in cache, a web search supplements the results.\n\nReturns title, URL, similarity_score, and markdown_content per result.";
+    readonly research: "Run multi-step research on a complex question. Decomposes the question into sub-queries, searches in parallel, fetches top sources, and synthesizes a report with citations.\n\nKey parameters:\n- question: the research question to investigate\n- depth: \"quick\" (~15s, 2 sub-queries, 5-8 sources), \"standard\" (~40s, 4 sub-queries, 10-15 sources, default), \"comprehensive\" (~80s, 7 sub-queries, 20-25 sources)\n- max_sources: override the default source count for the chosen depth\n- include_domains/exclude_domains: scope research to specific sites\n- schema: optional JSON Schema -- if provided, the report is structured to extract fields matching the schema\n- stream: true to receive progress notifications as each research phase completes\n\nThe pipeline: (1) decompose question into sub-queries, (2) parallel search across sub-queries, (3) fetch and extract top unique sources, (4) synthesize report with citations from all sources, (5) optionally structure report fields if schema is provided.\n\nUses MCP requestSampling for intelligent decomposition and synthesis. Without sampling support, returns raw sources in context format with sampling_supported: false.\n\nReturns report (markdown), citations array, sources with full content, sub_queries used, depth level, and total_time_ms.";
+    readonly agent: "Execute a natural-language data gathering task. Plans search queries and URLs from a prompt, executes them in parallel, and synthesizes results. Full step transparency.\n\nKey parameters:\n- prompt: natural-language description of what data to gather (e.g., \"find pricing for the top 5 CRM tools\")\n- urls: optional array of specific URLs to include in the gathering\n- schema: optional JSON Schema -- if provided, extracts structured data matching the schema from each page and merges results\n- max_pages: maximum pages to fetch (default 10)\n- max_time_ms: maximum execution time in milliseconds (default 60000)\n- stream: true to receive progress notifications as each step completes\n\nThe pipeline: (1) plan -- interpret prompt to determine search queries and URLs to visit, (2) execute -- run searches and fetch URLs in parallel within budget, (3) extract -- if schema provided, apply schema extraction to each page and merge, (4) synthesize -- produce natural-language or structured result.\n\nThe steps array in the output provides full transparency into every action taken (plan, search, fetch, extract, synthesize) with timing. This differentiates from black-box alternatives.\n\nUses MCP requestSampling for planning and synthesis. Without sampling support, uses keyword extraction for planning and returns raw content.\n\nReturns result (string or structured object), sources array, pages_fetched count, steps array with action/detail/time_ms, total_time_ms, and sampling_supported flag.";
 };
 export type ToolName = keyof typeof TOOL_DESCRIPTIONS;
 //# sourceMappingURL=instructions.d.ts.map

package/dist/instructions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"instructions.d.ts","sourceRoot":"","sources":["../src/instructions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,eAAO,MAAM,mBAAmB,olFAmCkF,CAAC;AAEnH,eAAO,MAAM,iBAAiB;;;;;;CAmDpB,CAAC;AAEX,MAAM,MAAM,QAAQ,GAAG,MAAM,OAAO,iBAAiB,CAAC"}
+{"version":3,"file":"instructions.d.ts","sourceRoot":"","sources":["../src/instructions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,eAAO,MAAM,mBAAmB,i/LAuEmK,CAAC;AAEpM,eAAO,MAAM,iBAAiB;;;;;;;;;CA4GpB,CAAC;AAEX,MAAM,MAAM,QAAQ,GAAG,MAAM,OAAO,iBAAiB,CAAC"}

package/dist/instructions.js

@@ -14,25 +14,57 @@
  * Parameter schemas (types, enums, required/optional) belong on the JSON
  * Schema, not here. Installation/configuration is for humans, not LLMs.
  */
-export const WIGOLO_INSTRUCTIONS = `Wigolo is a local-first web access layer: search the open web, fetch pages, crawl sites, and extract structured data. All results land in a local SQLite cache that persists across sessions.
+export const WIGOLO_INSTRUCTIONS = `Wigolo is a local-first web access layer: search the open web, fetch pages, crawl sites, extract structured data, find related content, run multi-step research, and execute agent-driven data gathering. All results land in a local SQLite cache that persists across sessions.
 
 ## When to use which tool
 
-- \`search\` — you need information on a topic but do not have a URL yet.
-- \`fetch\` — you already have a specific URL to read.
-- \`crawl\` — you need multiple pages from the same site (docs, wikis, references).
-- \`cache\` — you want to know if the content is already on disk from an earlier read.
-- \`extract\` — you need specific data points (tables, metadata, schema-shaped fields) rather than a whole page as markdown.
+- \`search\` -- you need information on a topic but do not have a URL yet. Pass a query string or an array of 3-5 semantically varied keyword forms for broader coverage.
+- \`fetch\` -- you already have a specific URL to read.
+- \`crawl\` -- you need multiple pages from the same site (docs, wikis, references).
+- \`cache\` -- you want to know if the content is already on disk from an earlier read.
+- \`extract\` -- you need specific data points (tables, metadata, schema-shaped fields) rather than a whole page as markdown.
+- \`find_similar\` -- you have a URL or concept and want related content from the cache or web. Useful for "more like this" discovery.
+- \`research\` -- you have a complex question that needs multi-step investigation: question decomposition, parallel search, source synthesis into a report. Set \`depth\` to control thoroughness.
+- \`agent\` -- you need to gather structured or unstructured data from multiple sources based on a natural-language prompt. Provides full step transparency.
+
+## Routing by intent
+
+| Intent | Tool | Key parameters |
+|--------|------|----------------|
+| Documentation lookup | \`search\` | \`include_domains: ["react.dev", "docs.python.org"]\`, \`category: "docs"\` |
+| Error debugging | \`search\` | exact error string as query, \`category: "code"\` |
+| Library research | \`crawl\` | seed URL of docs site, \`strategy: "sitemap"\`, then \`cache\` for later queries |
+| Related content | \`find_similar\` | \`url\` of a known good page, or \`concept\` as free text |
+| Direct answer | \`search\` | \`format: "answer"\` for a synthesized direct response |
+| Comprehensive research | \`research\` | \`depth: "comprehensive"\`, optional \`include_domains\` to scope |
+| Data gathering | \`agent\` | natural-language \`prompt\`, optional \`schema\` for structured output |
+| Structured extraction | \`extract\` | \`mode: "schema"\` with a JSON Schema, or \`mode: "tables"\` |
+| Site inventory | \`crawl\` | \`strategy: "map"\` for URL-only discovery, no content fetched |
+
+## Rapidly changing content
+
+For news, prices, status pages, release notes, or any content that changes frequently, bypass the cache with \`force_refresh: true\`:
+
+  search({ query: "...", force_refresh: true })
+  fetch({ url: "...", force_refresh: true })
+
+When freshness matters more than speed, use \`force_refresh\`. When speed matters more than freshness (documentation, tutorials, reference pages), let the cache work -- it is much faster.
 
 ## Check the cache before going to the network
 
-Before every \`search\` or \`fetch\`, consider a \`cache\` call with the query text or URL pattern. Pages read in this or a prior session return instantly with their full markdown — no network, no rate limits.
+Before every \`search\` or \`fetch\`, consider a \`cache\` call with the query text or URL pattern. Pages read in this or a prior session return instantly with their full markdown -- no network, no rate limits. The \`research\` and \`agent\` tools check the cache internally, so you do not need a separate call for those.
+
+## Multi-query search strategy
+
+For broad or exploratory queries, pass an array of 3-5 semantically varied keyword forms rather than a single natural-language question. Example: instead of "how does React handle state management", pass \`["react state management", "useState useReducer patterns", "react hooks state", "react context vs redux"]\`. The search tool deduplicates across sub-queries automatically.
 
 ## Pick the right strategy
 
-- For documentation sites, prefer \`crawl\` with \`strategy: "sitemap"\` — it is faster and more complete than BFS because it reads sitemap.xml directly.
+- For documentation sites, prefer \`crawl\` with \`strategy: "sitemap"\` -- it is faster and more complete than BFS because it reads sitemap.xml directly.
 - When you only need to discover what pages exist on a site, use \`crawl\` with \`strategy: "map"\`. It returns URLs only, no content, and is far cheaper than a full crawl. Follow up with targeted \`fetch\` calls.
 - For structured data (prices, specs, listings, table rows), use \`extract\` with \`mode: "schema"\` or \`mode: "tables"\`. Reach for \`fetch\` only when you want the whole page as markdown.
+- For complex questions requiring synthesis from multiple sources, use \`research\` instead of manually chaining \`search\` + \`fetch\` calls.
+- For natural-language data gathering tasks ("find the pricing for the top 5 CRM tools"), use \`agent\` with an optional \`schema\` to structure the output.
 
 ## Scope searches, do not just broaden queries
 
@@ -42,48 +74,62 @@ Before every \`search\` or \`fetch\`, consider a \`cache\` call with the query t
 
 - \`max_results: 3\` for focused lookups; \`5\` is the default; \`10+\` only for broad research.
 - \`fetch\` with \`section: "Heading Name"\` returns just the content under that heading. Cheaper and more relevant than the whole page.
-- Repeated fetches of the same URL are free — served directly from the SQLite cache.
+- Repeated fetches of the same URL are free -- served directly from the SQLite cache.
+- \`research\` with \`depth: "quick"\` is fast (~15s) and sufficient for most factual questions. Reserve \`"comprehensive"\` for topics requiring deep investigation.
+- \`agent\` respects \`max_pages\` (default 10) and \`max_time_ms\` (default 60s) to bound resource usage.
 
 ## Capabilities worth knowing
 
 - Localhost URLs work: \`http://localhost:3000\`, \`http://127.0.0.1:8080\`, and similar. Useful for reading local dev servers and internal docs.
 - \`use_auth: true\` on \`fetch\` and \`crawl\` reuses the user's configured browser session for pages behind a login.
 - \`cache\` accepts FTS5 query syntax (\`AND\`, \`OR\`, \`NOT\`, \`"exact phrase"\`) for precise lookups.
-- \`crawl\` accepts regex \`include_patterns\` and \`exclude_patterns\` to stay inside a section of a large site.`;
+- \`crawl\` accepts regex \`include_patterns\` and \`exclude_patterns\` to stay inside a section of a large site.
+- \`find_similar\` uses cached embeddings when available -- no network call needed if the content has been seen before.
+- \`research\` and \`agent\` use MCP requestSampling for intelligent decomposition and synthesis when the client supports it. Without sampling support, they return raw sources in context format.`;
 export const TOOL_DESCRIPTIONS = {
     fetch: `Fetch a single URL and return clean markdown. Use when you have a specific URL to read. Automatically detects if JavaScript rendering is needed.
 
 Key parameters:
-- section: extract content under a specific heading (e.g., section: "API Reference") — faster than reading the whole page
+- section: extract content under a specific heading (e.g., section: "API Reference") -- faster than reading the whole page
 - use_auth: true to use stored browser session for authenticated/private pages
 - render_js: "auto" (default, detects JS need), "always" (force browser), "never" (HTTP only, fastest)
 - headers: custom HTTP headers if needed
+- force_refresh: true to bypass cache and fetch fresh content from the network
+
+Returns title, markdown content, links, images, and metadata. Result is cached locally -- subsequent fetches of the same URL return instantly. Works with localhost URLs (localhost:3000, etc.) for reading local dev servers.
 
-Returns title, markdown content, links, images, and metadata. Result is cached locally — subsequent fetches of the same URL return instantly. Works with localhost URLs (localhost:3000, etc.) for reading local dev servers.`,
-    search: `Search the web and return full markdown content from top results. Use for finding information on any topic — returns extracted page content, not just snippets.
+Use force_refresh: true for pages that change frequently (news sites, changelogs, dashboards, API status pages). By default, previously fetched pages are served from local cache for speed.`,
+    search: `Search the web and return full markdown content from top results. Use for finding information on any topic -- returns extracted page content, not just snippets.
 
 Key parameters:
+- query: a search string, or an array of 3-5 semantically varied keyword forms for broader coverage. Arrays are deduplicated and merged automatically.
 - include_domains/exclude_domains: scope results to specific sites (e.g., include_domains: ["react.dev"])
-- category: "general", "news", "code", "docs", "papers" — filters by content type
+- category: "general", "news", "code", "docs", "papers" -- filters by content type
 - from_date/to_date: ISO dates for time-bounded queries
 - max_results: default 5. Use 3 for focused queries, 10+ for research.
+- format: "full" (default, structured JSON), "context" (single token-budgeted string for LLM injection), "answer" (synthesized direct answer via requestSampling), "stream_answer" (same as answer, with MCP progress notifications emitted between pipeline phases)
+- force_refresh: true to bypass all caches (search results and page content)
+
+The "answer" format uses the MCP client's sampling capability to synthesize a direct response from search results. If sampling is not supported, falls back to "context" format. "stream_answer" emits notifications/progress messages at each pipeline phase (search, fetch, synthesize) when the client provides a progressToken via request._meta — token-level streaming of the LLM response is not supported by MCP sampling, so the answer itself still arrives as one block.
 
-Results include title, URL, relevance_score, and full markdown_content per result. Previously fetched pages are served from local cache.`,
+Results include title, URL, relevance_score, and full markdown_content per result. Previously fetched pages are served from local cache.
+
+Use force_refresh: true when you need current information that may have changed since the last search. Default behavior uses cached results when available.`,
     crawl: `Crawl a website starting from a URL and return content from multiple pages. Use for indexing documentation sites, wikis, or any multi-page resource.
 
 Key parameters:
-- strategy: "bfs" (breadth-first, default), "dfs" (depth-first), "sitemap" (use sitemap.xml — fastest for doc sites), "map" (URL discovery only, no content — fastest for scoping a site)
+- strategy: "bfs" (breadth-first, default), "dfs" (depth-first), "sitemap" (use sitemap.xml -- fastest for doc sites), "map" (URL discovery only, no content -- fastest for scoping a site)
 - max_depth: how many links deep to follow (default 2)
 - max_pages: maximum pages to fetch (default 20)
 - include_patterns/exclude_patterns: regex filters on URLs
 
 Returns an array of pages with title, markdown, and depth. Content is deduplicated across pages (repeated nav/headers/footers stripped). All pages are cached for later cache queries.`,
-    cache: `Search previously fetched content without hitting the network. Use before searching the web — if relevant content was already fetched or crawled, this returns it instantly.
+    cache: `Search previously fetched content without hitting the network. Use before searching the web -- if relevant content was already fetched or crawled, this returns it instantly.
 
 Key parameters:
 - query: full-text search over cached markdown and titles (supports FTS5 syntax: AND, OR, NOT, "phrase match")
-- url_pattern: glob filter on URLs (e.g., "*react.dev*")
-- since: ISO date — only results cached after this date
+- url_pattern: glob filter on URLs (e.g., "*example.com*")
+- since: ISO date -- only results cached after this date
 - stats: true to get cache size, entry count, oldest/newest dates
 - clear: true to delete matching entries
 
@@ -91,11 +137,55 @@ Returns matching cached pages with full markdown content. Cache persists across
     extract: `Extract structured data from a URL or raw HTML. Use when you need specific data points, tables, or metadata rather than full page markdown.
 
 Key parameters:
-- mode: "selector" (CSS selector → text), "tables" (HTML tables → JSON rows), "metadata" (title/author/date/description), "schema" (JSON Schema → heuristic field extraction)
-- css_selector: required for mode="selector" — any valid CSS selector
+- mode: "selector" (CSS selector -> text), "tables" (HTML tables -> JSON rows), "metadata" (title/author/date/description), "schema" (JSON Schema -> heuristic field extraction)
+- css_selector: required for mode="selector" -- any valid CSS selector
 - schema: for mode="schema", a JSON Schema object describing the fields to extract
 - multiple: true to return array of all matches (mode="selector" only)
 
 For mode="tables", returns array of table objects with headers and row data. For mode="schema", pass { price: "string", name: "string" } and get structured fields extracted from the page.`,
+    find_similar: `Find content related to a URL or concept. Use when you have a known-good page or topic and want to discover similar resources from the cache or web.
+
+Key parameters:
+- url: a URL to find content similar to. The page's content and embeddings are used for similarity matching.
+- concept: free-text description of what you want similar content for. Use when you do not have a specific URL.
+- max_results: number of similar items to return (default 5)
+- include_cached: true (default) to search the local cache first, false to skip cache and search the web only
+- threshold: minimum similarity score (0-1, default 0.5) -- higher values return fewer, more relevant results
+
+Provide either url or concept (not both). Results are ranked by cosine similarity of embeddings. Cached content with embeddings is searched first for instant results. If fewer than max_results are found in cache, a web search supplements the results.
+
+Returns title, URL, similarity_score, and markdown_content per result.`,
+    research: `Run multi-step research on a complex question. Decomposes the question into sub-queries, searches in parallel, fetches top sources, and synthesizes a report with citations.
+
+Key parameters:
+- question: the research question to investigate
+- depth: "quick" (~15s, 2 sub-queries, 5-8 sources), "standard" (~40s, 4 sub-queries, 10-15 sources, default), "comprehensive" (~80s, 7 sub-queries, 20-25 sources)
+- max_sources: override the default source count for the chosen depth
+- include_domains/exclude_domains: scope research to specific sites
+- schema: optional JSON Schema -- if provided, the report is structured to extract fields matching the schema
+- stream: true to receive progress notifications as each research phase completes
+
+The pipeline: (1) decompose question into sub-queries, (2) parallel search across sub-queries, (3) fetch and extract top unique sources, (4) synthesize report with citations from all sources, (5) optionally structure report fields if schema is provided.
+
+Uses MCP requestSampling for intelligent decomposition and synthesis. Without sampling support, returns raw sources in context format with sampling_supported: false.
+
+Returns report (markdown), citations array, sources with full content, sub_queries used, depth level, and total_time_ms.`,
+    agent: `Execute a natural-language data gathering task. Plans search queries and URLs from a prompt, executes them in parallel, and synthesizes results. Full step transparency.
+
+Key parameters:
+- prompt: natural-language description of what data to gather (e.g., "find pricing for the top 5 CRM tools")
+- urls: optional array of specific URLs to include in the gathering
+- schema: optional JSON Schema -- if provided, extracts structured data matching the schema from each page and merges results
+- max_pages: maximum pages to fetch (default 10)
+- max_time_ms: maximum execution time in milliseconds (default 60000)
+- stream: true to receive progress notifications as each step completes
+
+The pipeline: (1) plan -- interpret prompt to determine search queries and URLs to visit, (2) execute -- run searches and fetch URLs in parallel within budget, (3) extract -- if schema provided, apply schema extraction to each page and merge, (4) synthesize -- produce natural-language or structured result.
+
+The steps array in the output provides full transparency into every action taken (plan, search, fetch, extract, synthesize) with timing. This differentiates from black-box alternatives.
+
+Uses MCP requestSampling for planning and synthesis. Without sampling support, uses keyword extraction for planning and returns raw content.
+
+Returns result (string or structured object), sources array, pages_fetched count, steps array with action/detail/time_ms, total_time_ms, and sampling_supported flag.`,
 };
 //# sourceMappingURL=instructions.js.map

package/dist/instructions.js.map

@@ -1 +1 @@
-{"version":3,"file":"instructions.js","sourceRoot":"","sources":["../src/instructions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kHAmC+E,CAAC;AAEnH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,KAAK,EAAE;;;;;;;;8NAQqN;IAE5N,MAAM,EAAE;;;;;;;;yIAQ+H;IAEvI,KAAK,EAAE;;;;;;;;uLAQ8K;IAErL,KAAK,EAAE;;;;;;;;;0GASiG;IAExG,OAAO,EAAE;;;;;;;;4LAQiL;CAClL,CAAC"}
+{"version":3,"file":"instructions.js","sourceRoot":"","sources":["../src/instructions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mMAuEgK,CAAC;AAEpM,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,KAAK,EAAE;;;;;;;;;;;6LAWoL;IAE3L,MAAM,EAAE;;;;;;;;;;;;;;;4JAekJ;IAE1J,KAAK,EAAE;;;;;;;;uLAQ8K;IAErL,KAAK,EAAE;;;;;;;;;0GASiG;IAExG,OAAO,EAAE;;;;;;;;4LAQiL;IAE1L,YAAY,EAAE;;;;;;;;;;;uEAWuD;IAErE,QAAQ,EAAE;;;;;;;;;;;;;;yHAc6G;IAEvH,KAAK,EAAE;;;;;;;;;;;;;;;;sKAgB6J;CAC5J,CAAC"}

package/dist/logger.d.ts

@@ -1,4 +1,4 @@
-type Module = 'fetch' | 'search' | 'crawl' | 'cache' | 'extract' | 'searxng' | 'server' | 'cli' | 'jsonld' | 'repl';
+type Module = 'fetch' | 'search' | 'crawl' | 'cache' | 'extract' | 'searxng' | 'server' | 'cli' | 'jsonld' | 'repl' | 'embedding' | 'research' | 'agent';
 export interface Logger {
     debug(msg: string, data?: Record<string, unknown>): void;
     info(msg: string, data?: Record<string, unknown>): void;

package/dist/logger.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAGA,KAAK,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AASpH,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1D;AAmBD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAiBnD"}
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAGA,KAAK,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC;AASzJ,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACxD,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1D;AAmBD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAiBnD"}

package/dist/repl/commands/agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentOutput } from '../../types.js';
+import type { ParsedArgs } from '../parser.js';
+import type { ReplDeps } from './types.js';
+export declare function executeAgent(args: ParsedArgs, deps: ReplDeps): Promise<AgentOutput>;
+//# sourceMappingURL=agent.d.ts.map

package/dist/repl/commands/agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../src/repl/commands/agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAM3C,wBAAsB,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,CA6CzF"}

package/dist/repl/commands/agent.js

@@ -0,0 +1,48 @@
+import { handleAgent } from '../../tools/agent.js';
+import { createLogger } from '../../logger.js';
+const log = createLogger('repl');
+export async function executeAgent(args, deps) {
+    try {
+        const prompt = args.positional.join(' ').trim();
+        if (!prompt) {
+            return {
+                result: '',
+                sources: [],
+                pages_fetched: 0,
+                steps: [],
+                total_time_ms: 0,
+                sampling_supported: false,
+                error: 'Usage: agent <prompt> [--urls=u1,u2] [--max-pages=N] [--max-time=MS]',
+            };
+        }
+        const input = { prompt };
+        if (args.flags.urls) {
+            input.urls = args.flags.urls.split(',').map(u => u.trim());
+        }
+        if (args.flags['max-pages']) {
+            input.max_pages = parseInt(args.flags['max-pages'], 10);
+        }
+        if (args.flags['max-time']) {
+            input.max_time_ms = parseInt(args.flags['max-time'], 10);
+        }
+        if (args.flags['max-time-ms']) {
+            input.max_time_ms = parseInt(args.flags['max-time-ms'], 10);
+        }
+        log.debug('executing agent command', { prompt, flags: args.flags });
+        return await handleAgent(input, deps.engines, deps.router, deps.backendStatus);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        log.error('agent command failed', { error: msg });
+        return {
+            result: '',
+            sources: [],
+            pages_fetched: 0,
+            steps: [],
+            total_time_ms: 0,
+            sampling_supported: false,
+            error: msg,
+        };
+    }
+}
+//# sourceMappingURL=agent.js.map

package/dist/repl/commands/agent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../src/repl/commands/agent.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;AAEjC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAgB,EAAE,IAAc;IACjE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;gBACX,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,EAAE;gBACT,aAAa,EAAE,CAAC;gBAChB,kBAAkB,EAAE,KAAK;gBACzB,KAAK,EAAE,sEAAsE;aAC9E,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAe,EAAE,MAAM,EAAE,CAAC;QAErC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IACjF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,OAAO;YACL,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,EAAE;YACX,aAAa,EAAE,CAAC;YAChB,KAAK,EAAE,EAAE;YACT,aAAa,EAAE,CAAC;YAChB,kBAAkB,EAAE,KAAK;YACzB,KAAK,EAAE,GAAG;SACX,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/repl/commands/find-similar.d.ts

@@ -0,0 +1,5 @@
+import type { FindSimilarOutput } from '../../types.js';
+import type { ParsedArgs } from '../parser.js';
+import type { ReplDeps } from './types.js';
+export declare function executeFindSimilar(args: ParsedArgs, deps: ReplDeps): Promise<FindSimilarOutput>;
+//# sourceMappingURL=find-similar.d.ts.map

package/dist/repl/commands/find-similar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-similar.d.ts","sourceRoot":"","sources":["../../../src/repl/commands/find-similar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAoB,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAM3C,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,iBAAiB,CAAC,CA2D5B"}