@staticn0va/wigolo 0.4.0 → 0.6.0

package/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: wigolo
-description: Local-first web search MCP server for AI coding agents. Search, fetch, crawl, cache, and extract from the web with zero API keys, zero cloud, zero cost.
+description: Local-first web access MCP server for AI coding agents. Eight tools for search, fetch, crawl, cache, extract, find similar, research, and agent-driven data gathering. No API keys. Results cached in local SQLite.
 author: KnockOutEZ
 license: BUSL-1.1
 repository: https://github.com/KnockOutEZ/wigolo
@@ -9,23 +9,29 @@ install: npx @staticn0va/wigolo
 runtime: node
 min_runtime_version: "20"
 tools:
-  - name: search
-    description: Search the web and return results with optional full content extraction. Supports domain filtering, date ranges, categories, and ML reranking.
   - name: fetch
-    description: Fetch a web page and return its content as clean markdown. Supports JavaScript rendering, authenticated browsing, section extraction, and caching.
+    description: Fetch one URL, return clean markdown. Auto-routes between HTTP and Playwright. Supports sections, auth, screenshots, browser actions.
+  - name: search
+    description: Search the web, return extracted markdown per result. Single query or array of query variants. Domain, category, date filters. Optional synthesized answer via MCP sampling.
   - name: crawl
-    description: Crawl a website starting from a seed URL. Supports BFS, DFS, sitemap, and map (URL-only) strategies with depth/page limits and URL filtering.
+    description: Crawl a site from a seed URL. BFS, DFS, sitemap, or map (URL-only) strategies with regex include/exclude filters.
   - name: cache
-    description: Query the local knowledge base of previously fetched content. Full-text search over cached pages by query, URL pattern, or date. Cache stats and clearing.
+    description: FTS5 search over previously fetched content. URL glob, date filters, stats, clear, and change detection via re-fetch.
   - name: extract
-    description: Extract structured data from a web page. Supports CSS selector extraction, HTML table parsing, metadata extraction (title, author, JSON-LD), and JSON Schema heuristic matching.
+    description: Structured extraction from URL or raw HTML. Modes: selector (CSS), tables, metadata (meta + JSON-LD), schema (heuristic field matching).
+  - name: find_similar
+    description: Find pages similar to a URL or concept. Hybrid cache (FTS5 + embeddings) + optional web supplement.
+  - name: research
+    description: Multi-step research pipeline. Question decomposition, parallel sub-search, source synthesis with citations. Quick, standard, or comprehensive depth.
+  - name: agent
+    description: Natural-language data gathering. Plans searches/URLs, fetches in parallel within page and time budgets, optionally applies a JSON Schema to each page.
 ---
 
 # wigolo
 
-Local-first web search MCP server for AI coding agents.
+Local-first web search MCP server for AI coding agents. Ships eight tools over stdio. All network results land in a local SQLite cache.
 
-## Installation
+## Quick Setup
 
 **Claude Code:**
 ```bash
@@ -44,106 +50,311 @@ claude mcp add wigolo -- npx @staticn0va/wigolo
 }
 ```
 
-**Optional warmup (improves search quality):**
+**Warmup (recommended, one-time):**
 ```bash
-npx @staticn0va/wigolo warmup
+npx @staticn0va/wigolo warmup          # installs Playwright Chromium + bootstraps SearXNG
+npx @staticn0va/wigolo warmup --all    # also installs Firefox, WebKit, reranker, embeddings, trafilatura
+npx @staticn0va/wigolo warmup --force  # wipe SearXNG state and rebuild
 ```
 
-## Tools
+Warmup flags: `--force`, `--all`, `--trafilatura`, `--reranker`, `--firefox`, `--webkit`, `--embeddings`, `--lightpanda`.
 
-### search
+## Tools
 
-Search the web and get full markdown content in one call.
+### fetch
 
+Fetch a single URL and return clean markdown. Use when you already have a specific URL.
+
+Parameters:
+- `url` (string, required)
+- `render_js`: `"auto"` (default) | `"always"` | `"never"`
+- `use_auth`: boolean (default `false`) — reuses the user's browser session
+- `max_chars`: number
+- `section`: string — return only the content under a heading
+- `section_index`: number (default `0`) — which heading match when multiple hit
+- `screenshot`: boolean (default `false`)
+- `headers`: object
+- `force_refresh`: boolean — bypass cache
+- `actions`: array of `{type, selector, text, ms, timeout, direction, amount}` — `click`, `type`, `wait`, `wait_for`, `scroll`, `screenshot`. Forces Playwright when present.
+
+Example:
 ```json
-{
-  "query": "React Server Components best practices",
-  "max_results": 5,
-  "include_domains": ["react.dev", "vercel.com"],
-  "from_date": "2024-01-01"
-}
+{ "url": "https://react.dev/reference/react/useState", "section": "Parameters" }
 ```
 
-Returns: titles, URLs, relevance scores, and full extracted markdown per result.
+Tip: `section` is much cheaper than reading the full page. Repeat fetches of the same URL are free from cache unless `force_refresh: true`.
 
-### fetch
-
-Fetch any URL and get clean markdown.
+### search
 
+Search the web and return extracted markdown per result. Use when you don't have a URL yet.
+
+Parameters:
+- `query` (string OR `string[]`, required) — array runs variants in parallel and dedupes
+- `max_results`: number (default `5`, cap `20`)
+- `include_content`: boolean (default `true`)
+- `content_max_chars`: number (default `30000`)
+- `max_total_chars`: number (default `50000`)
+- `time_range`: `"day"` | `"week"` | `"month"` | `"year"`
+- `include_domains` / `exclude_domains`: `string[]`
+- `from_date` / `to_date`: ISO `YYYY-MM-DD`
+- `category`: `"general"` | `"news"` | `"code"` | `"docs"` | `"papers"` | `"images"`
+- `language`: string
+- `search_engines`: `string[]` — override engine selection
+- `format`: `"full"` (default) | `"context"` (token-budgeted string) | `"answer"` (synthesized via MCP sampling) | `"stream_answer"` (answer + phase progress notifications)
+- `force_refresh`: boolean
+
+Example:
 ```json
-{
-  "url": "https://docs.react.dev/reference/react/useState",
-  "section": "Parameters"
-}
+{ "query": ["react server components patterns", "RSC data fetching", "react server components streaming"], "category": "docs", "include_domains": ["react.dev"], "max_results": 5 }
 ```
 
-Returns: clean markdown, links, images, metadata. Cached for future use.
+Tip: keyword queries beat natural-language questions. A 3–5 item `query` array usually finds more unique sources than one longer query.
 
 ### crawl
 
-Crawl a site from a seed URL.
+Crawl a site starting from a seed URL.
+
+Parameters:
+- `url` (string, required)
+- `strategy`: `"bfs"` (default) | `"dfs"` | `"sitemap"` | `"map"` (URL-only discovery, no content)
+- `max_depth`: number (default `2`)
+- `max_pages`: number (default `20`)
+- `include_patterns` / `exclude_patterns`: regex `string[]`
+- `use_auth`: boolean (default `false`)
+- `extract_links`: boolean (default `false`) — returns inter-page link graph
+- `max_total_chars`: number (default `100000`)
 
+Example:
 ```json
-{
-  "url": "https://docs.example.com",
-  "strategy": "sitemap",
-  "max_pages": 50
-}
+{ "url": "https://docs.python.org/3/library/", "strategy": "sitemap", "max_pages": 30, "include_patterns": ["^https://docs\\.python\\.org/3/library/asyncio"] }
 ```
 
-Returns: array of pages with titles, markdown, depth. Supports `bfs`, `dfs`, `sitemap`, `map` strategies.
+Tip: `strategy: "sitemap"` is faster and more complete than BFS on doc sites. `strategy: "map"` returns URLs only — cheap way to scope before targeted fetches.
 
 ### cache
 
-Query previously fetched content without hitting the network.
+Search previously fetched content without hitting the network.
+
+Parameters:
+- `query`: FTS5 syntax — supports `AND`, `OR`, `NOT`, `"exact phrase"`
+- `url_pattern`: glob (e.g. `"*react.dev*"`)
+- `since`: ISO date
+- `stats`: boolean — returns total URLs, size, date range
+- `clear`: boolean — deletes matching entries (requires one of `query`, `url_pattern`, `since`)
+- `check_changes`: boolean — re-fetches matching URLs, reports changed/unchanged with diff summaries
 
+Example:
 ```json
-{
-  "query": "React hooks",
-  "url_pattern": "*react.dev*"
-}
+{ "query": "useState OR useReducer", "url_pattern": "*react.dev*" }
 ```
 
-Returns: matching cached pages with full markdown and fetch timestamps.
+Tip: cache hits are instant and cross-session. Run this before `search` or `fetch` when you suspect the content is already on disk.
 
 ### extract
 
-Structured data extraction from any URL or HTML.
+Structured extraction from URL or raw HTML.
+
+Parameters:
+- `url` OR `html` (one required; `url` wins if both provided)
+- `mode`: `"metadata"` (default) | `"selector"` | `"tables"` | `"schema"`
+- `css_selector`: string — required for `mode: "selector"`
+- `multiple`: boolean (default `false`) — return all matches, selector mode only
+- `schema`: JSON Schema object with `properties` — required for `mode: "schema"`
 
+Example:
 ```json
-{
-  "url": "https://example.com/product",
-  "mode": "schema",
-  "schema": {
-    "type": "object",
-    "properties": {
-      "price": { "type": "string" },
-      "name": { "type": "string" }
-    }
-  }
-}
+{ "url": "https://example.com/product", "mode": "schema", "schema": { "type": "object", "properties": { "price": { "type": "string" }, "name": { "type": "string" }, "sku": { "type": "string" } } } }
+```
+
+Tip: `mode: "schema"` does heuristic matching over CSS classes, ARIA labels, microdata, and JSON-LD — no LLM call required.
+
+### find_similar
+
+Find pages related to a URL or a free-text concept.
+
+Parameters:
+- `url` OR `concept` (one required)
+- `max_results`: number (default `10`, cap `50`)
+- `include_domains` / `exclude_domains`: `string[]`
+- `include_cache`: boolean (default `true`)
+- `include_web`: boolean (default `true`)
+
+Example:
+```json
+{ "url": "https://react.dev/reference/react/useState", "max_results": 8, "include_domains": ["react.dev", "developer.mozilla.org"] }
+```
+
+Tip: uses hybrid 3-way search — FTS5 over titles, FTS5 over body, plus embeddings when available. Cache path is near-instant; web supplement runs only if cache yields too few results.
+
+### research
+
+Multi-step research pipeline with decomposition, parallel search, and cited synthesis.
+
+Parameters:
+- `question` (string, required)
+- `depth`: `"quick"` (~15s, 2 sub-queries, 5–8 sources) | `"standard"` (~40s, default) | `"comprehensive"` (~80s, 7 sub-queries, 20–25 sources)
+- `max_sources`: number (cap `50`) — overrides depth default
+- `include_domains` / `exclude_domains`: `string[]`
+- `schema`: JSON Schema — if present, report is structured to fill these fields
+- `stream`: boolean — emit progress notifications per phase
+
+Example:
+```json
+{ "question": "How do modern JS bundlers tree-shake ESM vs CJS?", "depth": "standard", "include_domains": ["webpack.js.org", "rollupjs.org", "esbuild.github.io", "vitejs.dev"] }
+```
+
+Tip: `research` checks cache internally — no need to pre-probe. Requires MCP sampling-capable client for synthesis; without sampling, returns raw sources in context format.
+
+### agent
+
+Natural-language data gathering. Plans queries and URLs from a prompt, runs them in parallel within budget, optionally applies a schema.
+
+Parameters:
+- `prompt` (string, required)
+- `urls`: `string[]` — seed URLs to include
+- `schema`: JSON Schema — extract structured fields per page and merge
+- `max_pages`: number (default `10`, cap `100`)
+- `max_time_ms`: number (default `60000`, cap `600000`)
+- `stream`: boolean
+
+Example:
+```json
+{ "prompt": "Compare pricing tiers for Supabase, Firebase, and Clerk", "schema": { "type": "object", "properties": { "provider": { "type": "string" }, "free_tier": { "type": "string" }, "paid_start": { "type": "string" } } }, "max_pages": 12 }
+```
+
+Tip: output includes a `steps` array showing every action (plan, search, fetch, extract, synthesize) with timings. Use this to debug why an agent run produced a weak result.
+
+## Workflow Patterns
+
+Quick routing:
+- Use when `search` — you need information but don't have a URL.
+- Use when `fetch` — you already have the URL.
+- Use when `crawl` — you need multiple pages from one site.
+- Use when `cache` — you want to check whether something is already on disk.
+- Use when `extract` — you need specific fields, tables, or metadata, not the whole page.
+- Use when `find_similar` — you have a good page/concept and want related content.
+- Use when `research` — a question needs decomposition and multi-source synthesis.
+- Use when `agent` — a natural-language task needs multi-step data gathering.
+
+**Cache-first lookup.** Before any `fetch` or `search`, probe the cache.
+```json
+cache({ "query": "oauth2 pkce", "url_pattern": "*auth0.com*" })
+// empty? fall through to search
+search({ "query": "oauth2 pkce flow", "include_domains": ["auth0.com"] })
+```
+
+**Fresh content (news, dashboards, changelogs).** Bypass cache explicitly.
+```json
+search({ "query": "node.js 22 release notes", "force_refresh": true, "time_range": "week" })
+fetch({ "url": "https://nodejs.org/en/blog", "force_refresh": true })
 ```
 
-Modes: `selector` (CSS), `tables` (HTML tables), `metadata` (meta tags + JSON-LD), `schema` (JSON Schema heuristic matching).
+**Scoped documentation research.** Crawl the relevant slice, then query cache.
+```json
+crawl({ "url": "https://docs.astro.build", "strategy": "sitemap", "max_pages": 40 })
+cache({ "query": "server islands hydration", "url_pattern": "*docs.astro.build*" })
+```
+
+**Broad exploration.** Pass a query array; dedup is automatic.
+```json
+search({ "query": ["rust async runtimes comparison", "tokio vs async-std vs smol", "rust executor benchmarks"], "max_results": 8 })
+```
+
+**More like this.** Start with a known-good URL, widen via `find_similar`.
+```json
+find_similar({ "url": "https://react.dev/reference/react/useMemo", "max_results": 6, "include_domains": ["react.dev"] })
+```
+
+**Complex synthesis.** One `research` call replaces 5+ manual search/fetch cycles.
+```json
+research({ "question": "Tradeoffs of vector DBs for RAG at 100M+ embeddings", "depth": "comprehensive" })
+```
+
+**Structured data from multiple sources.** Use `agent` with a schema.
+```json
+agent({ "prompt": "Find latency and pricing for top 5 edge compute providers", "schema": { "type": "object", "properties": { "provider": {"type":"string"}, "cold_start_ms": {"type":"string"}, "price_per_million": {"type":"string"} } } })
+```
+
+**Table extraction.** Skip markdown entirely.
+```json
+extract({ "url": "https://en.wikipedia.org/wiki/List_of_programming_languages", "mode": "tables" })
+```
+
+## Parameter Cheat Sheet
+
+| Situation | Tool + parameters |
+|---|---|
+| Focused lookup, known site | `search` + `max_results: 3` + `include_domains` |
+| Broad topic survey | `search` + `query: [...3-5 variants]` + `max_results: 8` |
+| Fresh content required | any tool + `force_refresh: true` |
+| Doc site indexing | `crawl` + `strategy: "sitemap"` |
+| Site URL inventory only | `crawl` + `strategy: "map"` |
+| Single heading from long page | `fetch` + `section: "..."` |
+| Behind login | `fetch` / `crawl` + `use_auth: true` |
+| Direct answer (sampling client) | `search` + `format: "answer"` |
+| LLM-ready context blob | `search` + `format: "context"` |
+| Complex question, multi-source | `research` + `depth: "standard"` |
+| Structured multi-page extraction | `agent` + `schema` |
+| One-page structured data | `extract` + `mode: "schema"` or `"tables"` |
+| Change tracking | `cache` + `check_changes: true` |
+
+## Anti-Patterns
+
+**Do not skip the cache.** Running `search` or `fetch` without probing `cache` wastes time on content already on disk. `research` and `agent` check cache internally; manual `search`/`fetch` do not.
+
+**Do not send natural-language questions to `search`.** Use keywords. `"how do I debounce in React hooks"` loses to `"react useDebounce hook custom"`.
+
+**Do not retry an identical failing query.** Reformulate keywords, swap `category`, or add `include_domains`. Same query → same empty result.
+
+**Do not use `agent` or `research` for one-URL lookups.** Use `fetch`. `agent` is for multi-source gathering; `research` is for decomposable questions.
+
+**Do not crawl `max_pages: 100` without filters.** Always add `include_patterns` to stay in-scope. Unfiltered crawls fetch nav, footer, and sitemap garbage.
+
+**Do not fetch whole pages when you need one section.** `fetch` + `section` reads under one heading only.
+
+**Do not set `force_refresh: true` by default.** It defeats the cache. Use it for news, status, changelogs — content that actually churns.
+
+**Do not pass a JSON Schema to `extract` without `properties`.** The handler rejects schemas that lack a `properties` key.
+
+## CLI Commands
+
+```bash
+wigolo                  # default: start MCP server on stdio
+wigolo mcp              # explicit: start MCP server
+wigolo warmup [flags]   # install Playwright, bootstrap SearXNG, optional extras
+wigolo serve            # start HTTP daemon on WIGOLO_DAEMON_PORT (default 3333)
+wigolo health           # health probe, exits 0 if ok
+wigolo doctor           # environment diagnostics (Python, Docker, Playwright, SearXNG)
+wigolo auth discover    # list CDP sessions (needs WIGOLO_CDP_URL)
+wigolo auth status      # show configured auth paths
+wigolo plugin add <git-url>    # clone plugin into ~/.wigolo/plugins/
+wigolo plugin list             # list installed plugins
+wigolo plugin remove <name>    # remove a plugin
+wigolo shell [--json]   # interactive REPL against subsystems
+```
 
-## Key Features
+## Configuration
 
-- Zero API keys required
-- Zero cloud dependency -- runs entirely local
-- Authenticated browsing (Chrome profiles, session state)
-- Localhost access (develop against local servers)
-- SQLite FTS5 cache with full-text search
-- ML reranking (optional, via FlashRank)
-- Extraction ensemble: site-specific, Defuddle, Trafilatura, Readability, Turndown
+Top environment variables. All optional — defaults are safe.
 
-## Requirements
+| Variable | Default | Purpose |
+|---|---|---|
+| `WIGOLO_DATA_DIR` | `~/.wigolo` | Cache DB, SearXNG state, plugins, embeddings |
+| `SEARXNG_URL` | unset | Point at an existing SearXNG (skips native bootstrap) |
+| `SEARXNG_MODE` | `native` | `native` runs local Python SearXNG; `docker` runs container |
+| `WIGOLO_CHROME_PROFILE_PATH` | unset | Chrome profile for `use_auth: true` |
+| `WIGOLO_CDP_URL` | unset | Chrome DevTools endpoint (e.g. `http://localhost:9222`) |
+| `MAX_BROWSERS` | `3` | Playwright pool size |
+| `WIGOLO_BROWSER_TYPES` | `chromium` | Comma list: `chromium,firefox,webkit` |
+| `WIGOLO_RERANKER` | `none` | `flashrank` for ML reranking |
+| `WIGOLO_EMBEDDING_MODEL` | `BAAI/bge-small-en-v1.5` | Used by `find_similar` |
+| `CACHE_TTL_CONTENT` | `604800` (7d) | Seconds before cached pages expire |
+| `LOG_LEVEL` | `info` | `debug` \| `info` \| `warn` \| `error` |
 
-- Node.js 20+
-- Python 3.8+ (recommended, for embedded SearXNG search)
-- Docker (optional, alternative to Python for SearXNG)
+Full list: see `src/config.ts`.
 
 ## Links
 
 - Repository: https://github.com/KnockOutEZ/wigolo
 - npm: https://www.npmjs.com/package/@staticn0va/wigolo
-- License: BSL 1.1 (converts to MIT on 2029-04-12)
+- License: BUSL-1.1 (converts to open source on 2029-04-12)

package/dist/agent/executor.d.ts

@@ -0,0 +1,13 @@
+import type { AgentPlan } from './planner.js';
+import type { AgentSource, AgentStep, SearchEngine } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+export interface ExecutionBudget {
+    maxPages: number;
+    deadlineMs: number;
+}
+export interface ExecutionResult {
+    sources: AgentSource[];
+    steps: AgentStep[];
+}
+export declare function executeAgentPlan(plan: AgentPlan, engines: SearchEngine[], router: SmartRouter, budget: ExecutionBudget): Promise<ExecutionResult>;
+//# sourceMappingURL=executor.d.ts.map

package/dist/agent/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/agent/executor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAMtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,KAAK,EAAE,SAAS,EAAE,CAAC;CACpB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,eAAe,CAAC,CAgD1B"}

package/dist/agent/executor.js

@@ -0,0 +1,128 @@
+import { createLogger } from '../logger.js';
+import { deduplicateResults } from '../search/dedup.js';
+import { extractContent } from '../extraction/pipeline.js';
+import { cacheContent } from '../cache/store.js';
+const log = createLogger('agent');
+const FETCH_TIMEOUT_MS = 15000;
+export async function executeAgentPlan(plan, engines, router, budget) {
+    const steps = [];
+    const allUrls = new Set();
+    try {
+        // Phase 1: Execute search queries
+        if (plan.searches.length > 0) {
+            const searchStart = Date.now();
+            const searchResults = await executeSearches(plan.searches, engines, budget.deadlineMs);
+            steps.push({
+                action: 'search',
+                detail: `Searched ${plan.searches.length} queries, found ${searchResults.length} results`,
+                time_ms: Date.now() - searchStart,
+            });
+            for (const result of searchResults) {
+                allUrls.add(result.url);
+            }
+        }
+        // Phase 2: Add explicit URLs
+        for (const url of plan.urls) {
+            allUrls.add(url);
+        }
+        if (allUrls.size === 0) {
+            return { sources: [], steps };
+        }
+        // Phase 3: Fetch pages within budget
+        const urlsToFetch = [...allUrls].slice(0, budget.maxPages);
+        const fetchStart = Date.now();
+        const sources = await fetchPages(urlsToFetch, router, budget);
+        steps.push({
+            action: 'fetch',
+            detail: `Fetched ${sources.filter((s) => s.fetched).length}/${urlsToFetch.length} pages`,
+            time_ms: Date.now() - fetchStart,
+        });
+        return { sources, steps };
+    }
+    catch (err) {
+        log.error('execution failed', {
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return { sources: [], steps };
+    }
+}
+async function executeSearches(queries, engines, deadlineMs) {
+    const allRaw = [];
+    const searchPromises = engines.flatMap((engine) => queries.map(async (query) => {
+        if (Date.now() >= deadlineMs)
+            return;
+        try {
+            const results = await engine.search(query, { maxResults: 10 });
+            for (const r of results) {
+                allRaw.push(r);
+            }
+        }
+        catch (err) {
+            log.warn('agent search query failed', {
+                engine: engine.name,
+                query,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }));
+    await Promise.allSettled(searchPromises);
+    const merged = deduplicateResults(allRaw);
+    return merged.map((m) => ({
+        url: m.url,
+        title: m.title,
+        snippet: m.snippet,
+        relevance_score: m.relevance_score,
+    }));
+}
+async function fetchPages(urls, router, budget) {
+    const fetchPromises = urls.map(async (url) => {
+        if (Date.now() >= budget.deadlineMs) {
+            return {
+                url,
+                title: '',
+                markdown_content: '',
+                fetched: false,
+                fetch_error: 'budget exceeded',
+            };
+        }
+        try {
+            const timeRemaining = budget.deadlineMs - Date.now();
+            const fetchTimeout = Math.min(FETCH_TIMEOUT_MS, Math.max(timeRemaining, 1000));
+            const raw = await Promise.race([
+                router.fetch(url, { renderJs: 'auto' }),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('fetch timeout')), fetchTimeout)),
+            ]);
+            const extraction = await extractContent(raw.html, raw.finalUrl, {
+                maxChars: 30000,
+                contentType: raw.contentType,
+            });
+            try {
+                cacheContent(raw, extraction);
+            }
+            catch (err) {
+                log.debug('failed to cache agent source', { url, error: String(err) });
+            }
+            return {
+                url,
+                title: extraction.title,
+                markdown_content: extraction.markdown,
+                fetched: true,
+            };
+        }
+        catch (err) {
+            log.debug('agent fetch failed', {
+                url,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            return {
+                url,
+                title: '',
+                markdown_content: '',
+                fetched: false,
+                fetch_error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    });
+    return Promise.all(fetchPromises);
+}
+//# sourceMappingURL=executor.js.map

package/dist/agent/executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.js","sourceRoot":"","sources":["../../src/agent/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAY/B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAe,EACf,OAAuB,EACvB,MAAmB,EACnB,MAAuB;IAEvB,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,IAAI,CAAC;QACH,kCAAkC;QAClC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAEvF,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,YAAY,IAAI,CAAC,QAAQ,CAAC,MAAM,mBAAmB,aAAa,CAAC,MAAM,UAAU;gBACzF,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;aAClC,CAAC,CAAC;YAEH,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,qCAAqC;QACrC,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAE9D,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,WAAW,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,QAAQ;YACxF,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACjC,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC5B,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAiB,EACjB,OAAuB,EACvB,UAAkB;IAElB,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAChD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU;YAAE,OAAO;QAErC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACpC,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,KAAK;gBACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAEzC,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxB,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAc,EACd,MAAmB,EACnB,MAAuB;IAEvB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAwB,EAAE;QACjE,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpC,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,gBAAgB,EAAE,EAAE;gBACpB,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,iBAAiB;aAC/B,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;YAE/E,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC7B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,YAAY,CAAC,CACnE;aACF,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;gBAC9D,QAAQ,EAAE,KAAK;gBACf,WAAW,EAAE,GAAG,CAAC,WAAW;aAC7B,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,gBAAgB,EAAE,UAAU,CAAC,QAAQ;gBACrC,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE;gBAC9B,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,gBAAgB,EAAE,EAAE;gBACpB,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;AACpC,CAAC"}

package/dist/agent/pipeline.d.ts

@@ -0,0 +1,5 @@
+import { type SamplingCapableServer } from '../search/sampling.js';
+import type { AgentInput, AgentOutput, SearchEngine } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+export declare function runAgentPipeline(input: AgentInput, engines: SearchEngine[], router: SmartRouter, server?: SamplingCapableServer): Promise<AgentOutput>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/agent/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/agent/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EAGX,YAAY,EACb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAQtD,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,WAAW,CAAC,CAyFtB"}