@staticn0va/wigolo 0.1.0

package/dist/server.js

@@ -0,0 +1,311 @@
+import { join } from 'node:path';
+import { mkdirSync } from 'node:fs';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { ListToolsRequestSchema, CallToolRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { SmartRouter } from './fetch/router.js';
+import { BrowserPool } from './fetch/browser-pool.js';
+import { httpFetch } from './fetch/http-client.js';
+import { initDatabase, closeDatabase } from './cache/db.js';
+import { handleFetch } from './tools/fetch.js';
+import { handleSearch } from './tools/search.js';
+import { handleCrawl } from './tools/crawl.js';
+import { handleCache } from './tools/cache.js';
+import { handleExtract } from './tools/extract.js';
+import { SearxngClient } from './search/searxng.js';
+import { DuckDuckGoEngine } from './search/engines/duckduckgo.js';
+import { BingEngine } from './search/engines/bing.js';
+import { StartpageEngine } from './search/engines/startpage.js';
+import { resolveSearchBackend } from './searxng/bootstrap.js';
+import { SearxngProcess } from './searxng/process.js';
+import { DockerSearxng } from './searxng/docker.js';
+import { getConfig } from './config.js';
+import { createLogger } from './logger.js';
+const log = createLogger('server');
+const FETCH_TOOL_SCHEMA = {
+    type: 'object',
+    properties: {
+        url: { type: 'string', description: 'URL to fetch' },
+        render_js: {
+            type: 'string',
+            enum: ['auto', 'always', 'never'],
+            description: 'JavaScript rendering mode (default: auto)',
+        },
+        use_auth: {
+            type: 'boolean',
+            description: 'Use stored auth credentials (default: false)',
+        },
+        max_chars: {
+            type: 'number',
+            description: 'Maximum characters to return',
+        },
+        section: {
+            type: 'string',
+            description: 'Extract a specific section by heading text',
+        },
+        section_index: {
+            type: 'number',
+            description: 'Index of the section match (default: 0)',
+        },
+        screenshot: {
+            type: 'boolean',
+            description: 'Capture a screenshot (default: false)',
+        },
+        headers: {
+            type: 'object',
+            description: 'Additional HTTP headers',
+            additionalProperties: { type: 'string' },
+        },
+    },
+    required: ['url'],
+};
+const SEARCH_TOOL_SCHEMA = {
+    type: 'object',
+    properties: {
+        query: { type: 'string', description: 'Search query' },
+        max_results: { type: 'number', description: 'Max results to return (default 5, max 20)' },
+        include_content: { type: 'boolean', description: 'Fetch full content for results (default true)' },
+        content_max_chars: { type: 'number', description: 'Max chars per result content (default 30000)' },
+        max_total_chars: { type: 'number', description: 'Max total chars across all results (default 50000)' },
+        time_range: { type: 'string', enum: ['day', 'week', 'month', 'year'], description: 'Time range filter' },
+        search_engines: { type: 'array', items: { type: 'string' }, description: 'Override engine selection' },
+        language: { type: 'string', description: 'Language preference' },
+        include_domains: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'Only return results from these domains (e.g. ["react.dev", "github.com"])',
+        },
+        exclude_domains: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'Never return results from these domains',
+        },
+        from_date: {
+            type: 'string',
+            description: 'ISO date (YYYY-MM-DD) — only return results published after this date',
+        },
+        to_date: {
+            type: 'string',
+            description: 'ISO date (YYYY-MM-DD) — only return results published before this date',
+        },
+        category: {
+            type: 'string',
+            enum: ['general', 'news', 'code', 'docs', 'papers', 'images'],
+            description: 'Category of search (general, news, code, docs, papers, images)',
+        },
+    },
+    required: ['query'],
+};
+const CRAWL_TOOL_SCHEMA = {
+    type: 'object',
+    properties: {
+        url: { type: 'string', description: 'Seed URL to start crawling from' },
+        max_depth: { type: 'number', description: 'Maximum link depth from seed (default: 2)' },
+        max_pages: { type: 'number', description: 'Maximum pages to crawl (default: 20)' },
+        strategy: {
+            type: 'string',
+            enum: ['bfs', 'dfs', 'sitemap', 'map'],
+            description: 'Crawl strategy: bfs (breadth-first), dfs (depth-first), sitemap (use sitemap.xml), map (URL-only discovery — returns list of URLs without content, faster than full crawl)',
+        },
+        include_patterns: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'URL regex whitelist — only crawl matching URLs',
+        },
+        exclude_patterns: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'URL regex blacklist — skip matching URLs',
+        },
+        use_auth: { type: 'boolean', description: 'Use stored auth credentials (default: false)' },
+        extract_links: { type: 'boolean', description: 'Return link graph between pages (default: false)' },
+        max_total_chars: { type: 'number', description: 'Max total chars across all pages (default: 100000)' },
+    },
+    required: ['url'],
+};
+const CACHE_TOOL_SCHEMA = {
+    type: 'object',
+    properties: {
+        query: { type: 'string', description: 'Full-text search over cached content' },
+        url_pattern: {
+            type: 'string',
+            description: 'Filter by URL glob pattern (e.g., "*example.com*")',
+        },
+        since: {
+            type: 'string',
+            description: 'ISO date — only results cached after this date',
+        },
+        clear: {
+            type: 'boolean',
+            description: 'Clear matching cache entries (requires at least one filter: query, url_pattern, or since)',
+        },
+        stats: {
+            type: 'boolean',
+            description: 'Return cache statistics (total URLs, size, date range)',
+        },
+    },
+};
+const EXTRACT_TOOL_SCHEMA = {
+    type: 'object',
+    properties: {
+        url: { type: 'string', description: 'URL to fetch and extract from' },
+        html: { type: 'string', description: 'Raw HTML to extract from (url takes priority if both provided)' },
+        mode: {
+            type: 'string',
+            enum: ['selector', 'tables', 'metadata', 'schema'],
+            description: 'Extraction mode: selector (CSS), tables (HTML tables), metadata (meta tags + JSON-LD), schema (extract fields matching a JSON Schema via heuristic matching)',
+        },
+        css_selector: {
+            type: 'string',
+            description: 'CSS selector to match (required when mode="selector")',
+        },
+        multiple: {
+            type: 'boolean',
+            description: 'Return array of all matches instead of first (default: false, only for mode="selector")',
+        },
+        schema: {
+            type: 'object',
+            description: 'JSON Schema defining fields to extract. Field names are matched against page content via CSS classes, ARIA labels, microdata, and JSON-LD. Required when mode="schema".',
+        },
+    },
+};
+export async function startServer() {
+    const config = getConfig();
+    mkdirSync(config.dataDir, { recursive: true });
+    initDatabase(join(config.dataDir, 'wigolo.db'));
+    const httpClient = {
+        fetch: (url, options) => httpFetch(url, options),
+    };
+    const browserPool = new BrowserPool();
+    const router = new SmartRouter(httpClient, browserPool);
+    // --- Search backend initialization ---
+    const backend = await resolveSearchBackend();
+    const searchEngines = [];
+    let searxngProcess = null;
+    let dockerSearxng = null;
+    if (backend.type === 'external' && backend.url) {
+        searchEngines.push(new SearxngClient(backend.url));
+    }
+    else if (backend.type === 'native' && backend.searxngPath) {
+        searxngProcess = new SearxngProcess(backend.searxngPath, config.dataDir);
+        const url = await searxngProcess.start();
+        if (url) {
+            searchEngines.push(new SearxngClient(url));
+        }
+        else {
+            log.warn('SearXNG failed to start, using direct scraping fallback');
+        }
+    }
+    else if (backend.type === 'docker') {
+        dockerSearxng = new DockerSearxng();
+        const url = await dockerSearxng.start();
+        if (url) {
+            searchEngines.push(new SearxngClient(url));
+        }
+        else {
+            log.warn('Docker SearXNG failed to start, using direct scraping fallback');
+        }
+    }
+    searchEngines.push(new BingEngine(), new DuckDuckGoEngine(), new StartpageEngine());
+    const server = new Server({ name: 'wigolo', version: '0.1.0' }, { capabilities: { tools: {} } });
+    server.setRequestHandler(ListToolsRequestSchema, async () => ({
+        tools: [
+            {
+                name: 'fetch',
+                description: 'Fetch a web page and return its content as clean markdown. ' +
+                    'Supports JavaScript rendering, auth, section extraction, and caching.',
+                inputSchema: FETCH_TOOL_SCHEMA,
+            },
+            {
+                name: 'search',
+                description: 'Search the web and return results with optional full content extraction. ' +
+                    'One call: query in, clean markdown out.',
+                inputSchema: SEARCH_TOOL_SCHEMA,
+            },
+            {
+                name: 'crawl',
+                description: 'Crawl a website starting from a seed URL. Supports BFS, DFS, and sitemap strategies ' +
+                    'with depth/page limits, URL filtering, and cross-page content deduplication.',
+                inputSchema: CRAWL_TOOL_SCHEMA,
+            },
+            {
+                name: 'cache',
+                description: 'Query the local knowledge base of previously fetched content. ' +
+                    'Search cached pages by full-text query, URL pattern, or date. ' +
+                    'Can also return cache statistics or clear entries.',
+                inputSchema: CACHE_TOOL_SCHEMA,
+            },
+            {
+                name: 'extract',
+                description: 'Extract structured data from a web page. Supports CSS selector extraction, ' +
+                    'table-to-JSON conversion, metadata extraction (title, author, date, JSON-LD), ' +
+                    'and schema-based extraction (provide a JSON Schema to heuristically extract matching fields).',
+                inputSchema: EXTRACT_TOOL_SCHEMA,
+            },
+        ],
+    }));
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const { name, arguments: args } = request.params;
+        if (name === 'fetch') {
+            const input = (args ?? {});
+            const result = await handleFetch(input, router);
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                isError: !!result.error,
+            };
+        }
+        if (name === 'search') {
+            const input = (args ?? {});
+            const result = await handleSearch(input, searchEngines, router);
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                isError: !!result.error,
+            };
+        }
+        if (name === 'crawl') {
+            const input = (args ?? {});
+            const result = await handleCrawl(input, router);
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                isError: !!result.error,
+            };
+        }
+        if (name === 'cache') {
+            const input = (args ?? {});
+            const result = handleCache(input);
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                isError: !!result.error,
+            };
+        }
+        if (name === 'extract') {
+            const input = (args ?? {});
+            const result = await handleExtract(input, router);
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                isError: !!result.error,
+            };
+        }
+        return {
+            content: [{ type: 'text', text: `Unknown tool: ${name}` }],
+            isError: true,
+        };
+    });
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    log.info('MCP server started');
+    const shutdown = async () => {
+        log.info('Shutting down');
+        if (searxngProcess)
+            await searxngProcess.stop();
+        if (dockerSearxng)
+            await dockerSearxng.stop();
+        await browserPool.shutdown();
+        closeDatabase();
+        await server.close();
+        process.exit(0);
+    };
+    process.on('SIGINT', shutdown);
+    process.on('SIGTERM', shutdown);
+}
+//# sourceMappingURL=server.js.map

package/dist/server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAmB,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,QAAiB;IACvB,UAAU,EAAE;QACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;QACpD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;YACjC,WAAW,EAAE,2CAA2C;SACzD;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,8CAA8C;SAC5D;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,8BAA8B;SAC5C;QACD,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,4CAA4C;SAC1D;QACD,aAAa,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yCAAyC;SACvD;QACD,UAAU,EAAE;YACV,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,uCAAuC;SACrD;QACD,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yBAAyB;YACtC,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SACzC;KACF;IACD,QAAQ,EAAE,CAAC,KAAK,CAAC;CAClB,CAAC;AAEF,MAAM,kBAAkB,GAAG;IACzB,IAAI,EAAE,QAAiB;IACvB,UAAU,EAAE;QACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;QACtD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2CAA2C,EAAE;QACzF,eAAe,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,+CAA+C,EAAE;QAClG,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE;QAClG,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oDAAoD,EAAE;QACtG,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,mBAAmB,EAAE;QACxG,cAAc,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE;QACtG,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE;QAChE,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,WAAW,EAAE,2EAA2E;SACzF;QACD,eAAe,EAAE;YACf,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,WAAW,EAAE,yCAAyC;SACvD;QACD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uEAAuE;SACrF;QACD,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,wEAAwE;SACtF;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC;YAC7D,WAAW,EAAE,gEAAgE;SAC9E;KACF;IACD,QAAQ,EAAE,CAAC,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,QAAiB;IACvB,UAAU,EAAE;QACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iCAAiC,EAAE;QACvE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2CAA2C,EAAE;QACvF,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sCAAsC,EAAE;QAClF,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC;YACtC,WAAW,EAAE,4KAA4K;SAC1L;QACD,gBAAgB,EAAE;YAChB,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,WAAW,EAAE,gDAAgD;SAC9D;QACD,gBAAgB,EAAE;YAChB,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,WAAW,EAAE,0CAA0C;SACxD;QACD,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,8CAA8C,EAAE;QAC1F,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kDAAkD,EAAE;QACnG,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oDAAoD,EAAE;KACvG;IACD,QAAQ,EAAE,CAAC,KAAK,CAAC;CAClB,CAAC;AAEF,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,QAAiB;IACvB,UAAU,EAAE;QACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sCAAsC,EAAE;QAC9E,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,oDAAoD;SAClE;QACD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gDAAgD;SAC9D;QACD,KAAK,EAAE;YACL,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,2FAA2F;SACzG;QACD,KAAK,EAAE;YACL,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,wDAAwD;SACtE;KACF;CACF,CAAC;AAEF,MAAM,mBAAmB,GAAG;IAC1B,IAAI,EAAE,QAAiB;IACvB,UAAU,EAAE;QACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,+BAA+B,EAAE;QACrE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gEAAgE,EAAE;QACvG,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC;YAClD,WAAW,EAAE,8JAA8J;SAC5K;QACD,YAAY,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,uDAAuD;SACrE;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,yFAAyF;SACvG;QACD,MAAM,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,yKAAyK;SACvL;KACF;CACF,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;IAEhD,MAAM,UAAU,GAAe;QAC7B,KAAK,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;KACjD,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAExD,wCAAwC;IACxC,MAAM,OAAO,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAC7C,MAAM,aAAa,GAAmB,EAAE,CAAC;IACzC,IAAI,cAAc,GAA0B,IAAI,CAAC;IACjD,IAAI,aAAa,GAAyB,IAAI,CAAC;IAE/C,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAC/C,aAAa,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QAC5D,cAAc,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACzE,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,aAAa,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,GAAG,EAAE,CAAC;YACR,aAAa,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE,EAAE,IAAI,gBAAgB,EAAE,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,EACpC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAChC,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;QAC5D,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,OAAO;gBACb,WAAW,EACT,6DAA6D;oBAC7D,uEAAuE;gBACzE,WAAW,EAAE,iBAAiB;aAC/B;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2EAA2E;oBAC3E,yCAAyC;gBAC3C,WAAW,EAAE,kBAAkB;aAChC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,WAAW,EACT,sFAAsF;oBACtF,8EAA8E;gBAChF,WAAW,EAAE,iBAAiB;aAC/B;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,WAAW,EACT,gEAAgE;oBAChE,gEAAgE;oBAChE,oDAAoD;gBACtD,WAAW,EAAE,iBAAiB;aAC/B;YACD;gBACE,IAAI,EAAE,SAAS;gBACf,WAAW,EACT,6EAA6E;oBAC7E,gFAAgF;oBAChF,+FAA+F;gBACjG,WAAW,EAAE,mBAAmB;aACjC;SACF;KACF,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAEjD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAA0B,CAAC;YACpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAChD,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAA2B,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YAChE,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAA0B,CAAC;YACpD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAChD,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAA0B,CAAC;YACpD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;YACtD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClD,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,IAAI,EAAE,EAAE,CAAC;YAC1D,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAE/B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC1B,IAAI,cAAc;YAAE,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,aAAa;YAAE,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC9C,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC7B,aAAa,EAAE,CAAC;QAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC"}

package/dist/tools/cache.d.ts

@@ -0,0 +1,3 @@
+import type { CacheInput, CacheOutput } from '../types.js';
+export declare function handleCache(input: CacheInput): CacheOutput;
+//# sourceMappingURL=cache.d.ts.map

package/dist/tools/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/tools/cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI3D,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CA+C1D"}

package/dist/tools/cache.js

@@ -0,0 +1,50 @@
+import { searchCacheFiltered, getCacheStats, clearCacheEntries } from '../cache/store.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('cache');
+export function handleCache(input) {
+    try {
+        if (input.stats) {
+            log.debug('Cache stats requested');
+            return { stats: getCacheStats() };
+        }
+        if (input.clear) {
+            if (!input.query && !input.url_pattern && !input.since) {
+                return { error: 'clear requires at least one filter (query, url_pattern, or since)' };
+            }
+            log.info('Clearing cache entries', {
+                query: input.query,
+                urlPattern: input.url_pattern,
+                since: input.since,
+            });
+            const count = clearCacheEntries({
+                query: input.query,
+                urlPattern: input.url_pattern,
+                since: input.since,
+            });
+            return { cleared: count };
+        }
+        log.debug('Cache search', {
+            query: input.query,
+            urlPattern: input.url_pattern,
+            since: input.since,
+        });
+        const results = searchCacheFiltered({
+            query: input.query,
+            urlPattern: input.url_pattern,
+            since: input.since,
+        });
+        return {
+            results: results.map((r) => ({
+                url: r.url,
+                title: r.title,
+                markdown: r.markdown,
+                fetched_at: r.fetchedAt,
+            })),
+        };
+    }
+    catch (err) {
+        log.error('Cache tool error', { error: String(err) });
+        return { error: err instanceof Error ? err.message : String(err) };
+    }
+}
+//# sourceMappingURL=cache.js.map

package/dist/tools/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/tools/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,CAAC;QACpC,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACvD,OAAO,EAAE,KAAK,EAAE,mEAAmE,EAAE,CAAC;YACxF,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACjC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,WAAW;gBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,iBAAiB,CAAC;gBAC9B,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,WAAW;gBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,UAAU,EAAE,KAAK,CAAC,WAAW;YAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,mBAAmB,CAAC;YAClC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,UAAU,EAAE,KAAK,CAAC,WAAW;YAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3B,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,UAAU,EAAE,CAAC,CAAC,SAAS;aACxB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrE,CAAC;AACH,CAAC"}

package/dist/tools/crawl.d.ts

@@ -0,0 +1,6 @@
+import type { CrawlInput, CrawlOutput, MapOutput } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+export declare function handleCrawl(input: CrawlInput, router: SmartRouter): Promise<CrawlOutput | (MapOutput & {
+    crawled: number;
+})>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/tools/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAWtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,GAAG,CAAC,SAAS,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAgE1D"}

package/dist/tools/crawl.js

@@ -0,0 +1,97 @@
+import { Crawler } from '../crawl/crawler.js';
+import { deduplicatePages } from '../crawl/dedup.js';
+import { mapUrls } from '../crawl/mapper.js';
+import { handleFetch } from './fetch.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('crawl');
+const DEFAULT_MAX_TOTAL_CHARS = 100000;
+export async function handleCrawl(input, router) {
+    try {
+        // Map strategy: lightweight URL-only discovery, skip full crawl pipeline
+        if (input.strategy === 'map') {
+            return handleMapStrategy(input, router);
+        }
+        const fetchFn = async (url) => handleFetch({ url, use_auth: input.use_auth }, router);
+        const rawFetchFn = async (url) => router.fetch(url, { renderJs: 'never' });
+        const crawler = new Crawler(fetchFn, rawFetchFn);
+        const result = await crawler.crawl(input);
+        // Deduplicate cross-page content (pass domain for SQLite boilerplate caching)
+        const domain = new URL(input.url).hostname;
+        const dedupedPages = deduplicatePages(result.pages.map((p) => ({ url: p.url, markdown: p.markdown })), domain);
+        // Apply deduped markdown back to pages
+        const pages = result.pages.map((page, i) => ({
+            ...page,
+            markdown: dedupedPages[i]?.markdown ?? page.markdown,
+        }));
+        // Enforce max_total_chars budget
+        const maxTotalChars = input.max_total_chars ?? DEFAULT_MAX_TOTAL_CHARS;
+        const budgetedPages = [];
+        let charCount = 0;
+        for (const page of pages) {
+            if (charCount + page.markdown.length > maxTotalChars && budgetedPages.length > 0) {
+                break;
+            }
+            budgetedPages.push(page);
+            charCount += page.markdown.length;
+        }
+        log.info('Crawl complete', {
+            url: input.url,
+            crawled: result.crawled,
+            returned: budgetedPages.length,
+            totalChars: charCount,
+        });
+        return {
+            pages: budgetedPages,
+            total_found: result.total_found,
+            crawled: result.crawled,
+            ...(result.links ? { links: result.links } : {}),
+        };
+    }
+    catch (err) {
+        log.error('Crawl failed', { url: input.url, error: String(err) });
+        return {
+            pages: [],
+            total_found: 0,
+            crawled: 0,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+async function handleMapStrategy(input, router) {
+    const httpFetchFn = async (url) => {
+        const raw = await router.fetch(url, { renderJs: 'never' });
+        return { html: raw.html, finalUrl: raw.finalUrl, statusCode: raw.statusCode };
+    };
+    try {
+        const mapResult = await mapUrls({
+            url: input.url,
+            max_depth: input.max_depth,
+            max_pages: input.max_pages,
+            include_patterns: input.include_patterns,
+            exclude_patterns: input.exclude_patterns,
+        }, httpFetchFn);
+        log.info('Map complete', {
+            url: input.url,
+            total_found: mapResult.total_found,
+            sitemap_found: mapResult.sitemap_found,
+        });
+        return {
+            urls: mapResult.urls,
+            total_found: mapResult.total_found,
+            sitemap_found: mapResult.sitemap_found,
+            crawled: 0,
+            ...(mapResult.error ? { error: mapResult.error } : {}),
+        };
+    }
+    catch (err) {
+        log.error('Map strategy failed', { url: input.url, error: String(err) });
+        return {
+            urls: [],
+            total_found: 0,
+            sitemap_found: false,
+            crawled: 0,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+//# sourceMappingURL=crawl.js.map

package/dist/tools/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,yEAAyE;QACzE,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC7B,OAAO,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACpC,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1C,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC3C,MAAM,YAAY,GAAG,gBAAgB,CACnC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAC/D,MAAM,CACP,CAAC;QAEF,uCAAuC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG,IAAI;YACP,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ;SACrD,CAAC,CAAC,CAAC;QAEJ,iCAAiC;QACjC,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,uBAAuB,CAAC;QACvE,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM;YACR,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE,aAAa,CAAC,MAAM;YAC9B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAiB,EACjB,MAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAChF,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAC7B;YACE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC,EACD,WAAW,CACZ,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;SACvC,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;YACtC,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO;YACL,IAAI,EAAE,EAAE;YACR,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,KAAK;YACpB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/tools/extract.d.ts

@@ -0,0 +1,4 @@
+import type { ExtractInput, ExtractOutput } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+export declare function handleExtract(input: ExtractInput, router: SmartRouter): Promise<ExtractOutput>;
+//# sourceMappingURL=extract.d.ts.map

package/dist/tools/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA8BtD,wBAAsB,aAAa,CACjC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,aAAa,CAAC,CAoDxB"}

package/dist/tools/extract.js

@@ -0,0 +1,69 @@
+import { extractMetadata, extractSelector, extractTables } from '../extraction/extract.js';
+import { extractWithSchema } from '../extraction/schema.js';
+import { extractJsonLd } from '../extraction/jsonld.js';
+import { getCachedContent, isExpired } from '../cache/store.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('extract');
+async function resolveHtml(input, router) {
+    if (input.url) {
+        const cached = getCachedContent(input.url);
+        if (cached && !isExpired(cached)) {
+            log.info('Using cached HTML', { url: input.url });
+            return { html: cached.rawHtml, sourceUrl: cached.url };
+        }
+        const raw = await router.fetch(input.url, {
+            renderJs: 'auto',
+            useAuth: false,
+        });
+        return { html: raw.html, sourceUrl: raw.finalUrl };
+    }
+    return { html: input.html };
+}
+export async function handleExtract(input, router) {
+    const mode = input.mode ?? 'metadata';
+    if (!input.url && !input.html) {
+        return { data: {}, mode, error: 'Either url or html must be provided' };
+    }
+    if (mode === 'selector' && !input.css_selector) {
+        return { data: '', mode, error: 'css_selector is required when mode is "selector"' };
+    }
+    if (mode === 'schema' && (!input.schema || !input.schema.properties)) {
+        return { data: {}, mode, error: 'schema is required when mode is "schema" and must have properties' };
+    }
+    try {
+        const { html, sourceUrl } = await resolveHtml(input, router);
+        let data;
+        switch (mode) {
+            case 'selector':
+                data = extractSelector(html, input.css_selector, input.multiple ?? false);
+                break;
+            case 'tables':
+                data = extractTables(html);
+                break;
+            case 'schema':
+                data = extractWithSchema(html, input.schema);
+                break;
+            case 'metadata':
+            default: {
+                const meta = extractMetadata(html);
+                const jsonld = extractJsonLd(html);
+                if (jsonld.length > 0) {
+                    meta.jsonld = jsonld;
+                }
+                data = meta;
+                break;
+            }
+        }
+        return { data, source_url: sourceUrl, mode };
+    }
+    catch (err) {
+        log.error('Extract failed', { url: input.url, error: String(err) });
+        return {
+            data: mode === 'selector' ? '' : mode === 'tables' ? [] : {},
+            source_url: input.url,
+            mode,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+//# sourceMappingURL=extract.js.map

package/dist/tools/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;AAEpC,KAAK,UAAU,WAAW,CACxB,KAAmB,EACnB,MAAmB;IAEnB,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;QACzD,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAmB,EACnB,MAAmB;IAEnB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,UAAU,CAAC;IAEtC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;IAC1E,CAAC;IAED,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,kDAAkD,EAAE,CAAC;IACvF,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,mEAAmE,EAAE,CAAC;IACxG,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAE7D,IAAI,IAA2B,CAAC;QAEhC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBACb,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,YAAa,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC;gBAC3E,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,MAAO,CAAC,CAAC;gBAC9C,MAAM;YACR,KAAK,UAAU,CAAC;YAChB,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC;gBACZ,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,OAAO;YACL,IAAI,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YAC5D,UAAU,EAAE,KAAK,CAAC,GAAG;YACrB,IAAI;YACJ,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/tools/fetch.d.ts

@@ -0,0 +1,4 @@
+import type { FetchInput, FetchOutput } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+export declare function handleFetch(input: FetchInput, router: SmartRouter): Promise<FetchOutput>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/tools/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAoCtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,CAgDtB"}