@staticn0va/wigolo 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +97 -50
- package/dist/agent/executor.d.ts +13 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +128 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/pipeline.d.ts +5 -0
- package/dist/agent/pipeline.d.ts.map +1 -0
- package/dist/agent/pipeline.js +198 -0
- package/dist/agent/pipeline.js.map +1 -0
- package/dist/agent/planner.d.ts +9 -0
- package/dist/agent/planner.d.ts.map +1 -0
- package/dist/agent/planner.js +190 -0
- package/dist/agent/planner.js.map +1 -0
- package/dist/cache/db.d.ts.map +1 -1
- package/dist/cache/db.js +32 -0
- package/dist/cache/db.js.map +1 -1
- package/dist/cache/store.d.ts +14 -0
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +69 -0
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/warmup.d.ts +4 -0
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +58 -0
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +8 -0
- package/dist/config.js.map +1 -1
- package/dist/embedding/embed.d.ts +19 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +131 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/key-terms.d.ts +12 -0
- package/dist/embedding/key-terms.d.ts.map +1 -0
- package/dist/embedding/key-terms.js +138 -0
- package/dist/embedding/key-terms.js.map +1 -0
- package/dist/embedding/subprocess.d.ts +31 -0
- package/dist/embedding/subprocess.d.ts.map +1 -0
- package/dist/embedding/subprocess.js +213 -0
- package/dist/embedding/subprocess.js.map +1 -0
- package/dist/embedding/vector-index.d.ts +26 -0
- package/dist/embedding/vector-index.d.ts.map +1 -0
- package/dist/embedding/vector-index.js +78 -0
- package/dist/embedding/vector-index.js.map +1 -0
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +61 -0
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-types.js +1 -1
- package/dist/fetch/browser-types.js.map +1 -1
- package/dist/fetch/lightpanda.d.ts +28 -0
- package/dist/fetch/lightpanda.d.ts.map +1 -0
- package/dist/fetch/lightpanda.js +177 -0
- package/dist/fetch/lightpanda.js.map +1 -0
- package/dist/instructions.d.ts +9 -6
- package/dist/instructions.d.ts.map +1 -1
- package/dist/instructions.js +95 -20
- package/dist/instructions.js.map +1 -1
- package/dist/logger.d.ts +1 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/repl/commands/agent.d.ts +5 -0
- package/dist/repl/commands/agent.d.ts.map +1 -0
- package/dist/repl/commands/agent.js +48 -0
- package/dist/repl/commands/agent.js.map +1 -0
- package/dist/repl/commands/find-similar.d.ts +5 -0
- package/dist/repl/commands/find-similar.d.ts.map +1 -0
- package/dist/repl/commands/find-similar.js +61 -0
- package/dist/repl/commands/find-similar.js.map +1 -0
- package/dist/repl/commands/research.d.ts +5 -0
- package/dist/repl/commands/research.d.ts.map +1 -0
- package/dist/repl/commands/research.js +50 -0
- package/dist/repl/commands/research.js.map +1 -0
- package/dist/repl/formatters.d.ts +4 -1
- package/dist/repl/formatters.d.ts.map +1 -1
- package/dist/repl/formatters.js +73 -0
- package/dist/repl/formatters.js.map +1 -1
- package/dist/repl/shell.d.ts.map +1 -1
- package/dist/repl/shell.js +22 -1
- package/dist/repl/shell.js.map +1 -1
- package/dist/research/decompose.d.ts +7 -0
- package/dist/research/decompose.d.ts.map +1 -0
- package/dist/research/decompose.js +195 -0
- package/dist/research/decompose.js.map +1 -0
- package/dist/research/pipeline.d.ts +5 -0
- package/dist/research/pipeline.d.ts.map +1 -0
- package/dist/research/pipeline.js +135 -0
- package/dist/research/pipeline.js.map +1 -0
- package/dist/research/synthesize.d.ts +10 -0
- package/dist/research/synthesize.d.ts.map +1 -0
- package/dist/research/synthesize.js +119 -0
- package/dist/research/synthesize.js.map +1 -0
- package/dist/search/answer-synthesis.d.ts +13 -0
- package/dist/search/answer-synthesis.d.ts.map +1 -0
- package/dist/search/answer-synthesis.js +120 -0
- package/dist/search/answer-synthesis.js.map +1 -0
- package/dist/search/find-similar.d.ts +5 -0
- package/dist/search/find-similar.d.ts.map +1 -0
- package/dist/search/find-similar.js +329 -0
- package/dist/search/find-similar.js.map +1 -0
- package/dist/search/multi-query.d.ts +22 -0
- package/dist/search/multi-query.d.ts.map +1 -0
- package/dist/search/multi-query.js +157 -0
- package/dist/search/multi-query.js.map +1 -0
- package/dist/search/rrf.d.ts +17 -0
- package/dist/search/rrf.d.ts.map +1 -0
- package/dist/search/rrf.js +48 -0
- package/dist/search/rrf.js.map +1 -0
- package/dist/search/sampling.d.ts +25 -0
- package/dist/search/sampling.d.ts.map +1 -0
- package/dist/search/sampling.js +52 -0
- package/dist/search/sampling.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +165 -4
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts +5 -0
- package/dist/tools/agent.d.ts.map +1 -0
- package/dist/tools/agent.js +67 -0
- package/dist/tools/agent.js.map +1 -0
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +10 -0
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts +5 -0
- package/dist/tools/find-similar.d.ts.map +1 -0
- package/dist/tools/find-similar.js +48 -0
- package/dist/tools/find-similar.js.map +1 -0
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +50 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/search.d.ts +2 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +169 -13
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +100 -3
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -3
package/SKILL.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: wigolo
|
|
3
|
-
description: Local-first web search MCP server for AI coding agents. Search, fetch, crawl, cache,
|
|
3
|
+
description: Local-first web search MCP server for AI coding agents. Search, fetch, crawl, cache, extract, find similar pages, deep research, and autonomous agent mode with zero API keys.
|
|
4
4
|
author: KnockOutEZ
|
|
5
5
|
license: BUSL-1.1
|
|
6
6
|
repository: https://github.com/KnockOutEZ/wigolo
|
|
@@ -18,7 +18,13 @@ tools:
|
|
|
18
18
|
- name: cache
|
|
19
19
|
description: Query the local knowledge base of previously fetched content. Full-text search over cached pages by query, URL pattern, or date. Cache stats and clearing.
|
|
20
20
|
- name: extract
|
|
21
|
-
description: Extract structured data from a web page.
|
|
21
|
+
description: Extract structured data from a web page. CSS selector extraction, HTML table parsing, metadata extraction (title, author, JSON-LD), and JSON Schema heuristic matching.
|
|
22
|
+
- name: find_similar
|
|
23
|
+
description: Find pages semantically similar to a given URL or concept. Uses cached embeddings and web search to discover related content.
|
|
24
|
+
- name: research
|
|
25
|
+
description: Deep multi-step research on a question. Decomposes into sub-queries, searches in parallel, fetches sources, and synthesizes a report with citations.
|
|
26
|
+
- name: agent
|
|
27
|
+
description: Autonomous data gathering agent. Plans search queries from a prompt, fetches pages within budget, optionally extracts structured data via JSON Schema, and synthesizes results.
|
|
22
28
|
---
|
|
23
29
|
|
|
24
30
|
# wigolo
|
|
@@ -52,79 +58,120 @@ npx @staticn0va/wigolo warmup
|
|
|
52
58
|
## Tools
|
|
53
59
|
|
|
54
60
|
### search
|
|
55
|
-
|
|
56
61
|
Search the web and get full markdown content in one call.
|
|
57
|
-
|
|
58
62
|
```json
|
|
59
|
-
{
|
|
60
|
-
"query": "React Server Components best practices",
|
|
61
|
-
"max_results": 5,
|
|
62
|
-
"include_domains": ["react.dev", "vercel.com"],
|
|
63
|
-
"from_date": "2024-01-01"
|
|
64
|
-
}
|
|
63
|
+
{ "query": "React Server Components best practices", "max_results": 5, "include_domains": ["react.dev"] }
|
|
65
64
|
```
|
|
66
65
|
|
|
67
|
-
Returns: titles, URLs, relevance scores, and full extracted markdown per result.
|
|
68
|
-
|
|
69
66
|
### fetch
|
|
70
|
-
|
|
71
67
|
Fetch any URL and get clean markdown.
|
|
72
|
-
|
|
73
68
|
```json
|
|
74
|
-
{
|
|
75
|
-
"url": "https://docs.react.dev/reference/react/useState",
|
|
76
|
-
"section": "Parameters"
|
|
77
|
-
}
|
|
69
|
+
{ "url": "https://docs.react.dev/reference/react/useState", "section": "Parameters" }
|
|
78
70
|
```
|
|
79
71
|
|
|
80
|
-
Returns: clean markdown, links, images, metadata. Cached for future use.
|
|
81
|
-
|
|
82
72
|
### crawl
|
|
83
|
-
|
|
84
73
|
Crawl a site from a seed URL.
|
|
85
|
-
|
|
86
74
|
```json
|
|
87
|
-
{
|
|
88
|
-
"url": "https://docs.example.com",
|
|
89
|
-
"strategy": "sitemap",
|
|
90
|
-
"max_pages": 50
|
|
91
|
-
}
|
|
75
|
+
{ "url": "https://docs.example.com", "strategy": "sitemap", "max_pages": 50 }
|
|
92
76
|
```
|
|
93
77
|
|
|
94
|
-
Returns: array of pages with titles, markdown, depth. Supports `bfs`, `dfs`, `sitemap`, `map` strategies.
|
|
95
|
-
|
|
96
78
|
### cache
|
|
97
|
-
|
|
98
79
|
Query previously fetched content without hitting the network.
|
|
99
|
-
|
|
100
80
|
```json
|
|
101
|
-
{
|
|
102
|
-
"query": "React hooks",
|
|
103
|
-
"url_pattern": "*react.dev*"
|
|
104
|
-
}
|
|
81
|
+
{ "query": "React hooks", "url_pattern": "*react.dev*" }
|
|
105
82
|
```
|
|
106
83
|
|
|
107
|
-
Returns: matching cached pages with full markdown and fetch timestamps.
|
|
108
|
-
|
|
109
84
|
### extract
|
|
110
|
-
|
|
111
85
|
Structured data extraction from any URL or HTML.
|
|
86
|
+
```json
|
|
87
|
+
{ "url": "https://example.com/product", "mode": "schema", "schema": { "type": "object", "properties": { "price": { "type": "string" }, "name": { "type": "string" } } } }
|
|
88
|
+
```
|
|
112
89
|
|
|
90
|
+
### find_similar
|
|
91
|
+
Find pages related to a URL or concept.
|
|
113
92
|
```json
|
|
114
|
-
{
|
|
115
|
-
"url": "https://example.com/product",
|
|
116
|
-
"mode": "schema",
|
|
117
|
-
"schema": {
|
|
118
|
-
"type": "object",
|
|
119
|
-
"properties": {
|
|
120
|
-
"price": { "type": "string" },
|
|
121
|
-
"name": { "type": "string" }
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
93
|
+
{ "url": "https://react.dev/reference/react/useState", "max_results": 5 }
|
|
125
94
|
```
|
|
126
95
|
|
|
127
|
-
|
|
96
|
+
### research
|
|
97
|
+
Deep multi-step research that plans queries, fetches, and synthesizes.
|
|
98
|
+
```json
|
|
99
|
+
{ "question": "How do modern bundlers handle tree-shaking of ESM vs CJS", "depth": "standard", "max_sources": 10 }
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### agent
|
|
103
|
+
Autonomous data gathering from a natural-language prompt.
|
|
104
|
+
```json
|
|
105
|
+
{ "prompt": "Compare authentication strategies of Supabase, Firebase, and Clerk", "max_pages": 15, "max_time_ms": 90000 }
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Workflow Patterns
|
|
109
|
+
|
|
110
|
+
Use the right tool for the right situation.
|
|
111
|
+
|
|
112
|
+
**When you know the URL** -- use `fetch`. One URL, clean markdown. Add `section` to read only the heading you need.
|
|
113
|
+
|
|
114
|
+
**When you need to find information** -- use `search`. Formulate a keyword query (not a natural language question). Scope with `include_domains` and `category` when you know where the answer lives.
|
|
115
|
+
|
|
116
|
+
**When you need multiple pages from one site** -- use `crawl`. For documentation sites, use `strategy: "sitemap"`. When you just want to discover what pages exist, use `strategy: "map"` (URL list only) then follow up with targeted `fetch` calls.
|
|
117
|
+
|
|
118
|
+
**When you need structured data** -- use `extract` with `mode: "tables"` or `mode: "schema"`. Do not use `fetch` when you need prices, specs, or table rows.
|
|
119
|
+
|
|
120
|
+
**When you already have content and want related pages** -- use `find_similar`. It searches the local cache by semantic similarity. No network calls needed.
|
|
121
|
+
|
|
122
|
+
**When you need a thorough answer on a complex topic** -- use `research`. It plans multiple search queries, fetches sources, and produces a cited synthesis. Prefer this over running 5+ manual search/fetch cycles.
|
|
123
|
+
|
|
124
|
+
**When the task requires multi-step data gathering** -- use `agent`. It breaks prompts into search queries and URL fetches, respects page and time budgets, and can extract structured data via JSON Schema.
|
|
125
|
+
|
|
126
|
+
**Before any network call** -- check `cache` first. Pages from prior sessions are still there. A cache hit is instant and free.
|
|
127
|
+
|
|
128
|
+
## Parameter Optimization
|
|
129
|
+
|
|
130
|
+
### search
|
|
131
|
+
- `max_results: 3` for focused lookups, `5` for exploration (default), `10+` for broad research
|
|
132
|
+
- `include_domains` narrows to trusted sources -- always use when you know the domain
|
|
133
|
+
- `category: "code"` for programming, `"docs"` for library docs, `"news"` for recent events
|
|
134
|
+
- `from_date` / `to_date` for time-sensitive queries
|
|
135
|
+
- `format: "context"` returns a single token-budgeted string for LLM injection
|
|
136
|
+
|
|
137
|
+
### fetch
|
|
138
|
+
- `section: "heading text"` extracts only content under that heading -- much cheaper than the full page
|
|
139
|
+
- `render_js: "never"` is fastest for static sites; `"always"` for SPAs
|
|
140
|
+
- `use_auth: true` to access pages behind login using the user's browser session
|
|
141
|
+
|
|
142
|
+
### crawl
|
|
143
|
+
- `strategy: "sitemap"` is 5-10x faster than BFS for doc sites
|
|
144
|
+
- `strategy: "map"` returns URLs only -- use to scope a site before targeted fetches
|
|
145
|
+
- `include_patterns` / `exclude_patterns` accept regex to stay in one section
|
|
146
|
+
|
|
147
|
+
### research
|
|
148
|
+
- `depth: "quick"` (~15s, 2 sub-queries), `"standard"` (~40s, default), `"comprehensive"` (~80s, 7 sub-queries)
|
|
149
|
+
- `max_sources` overrides the default source count for the chosen depth
|
|
150
|
+
|
|
151
|
+
### agent
|
|
152
|
+
- `max_pages` caps total page fetches (default 10, max 100)
|
|
153
|
+
- `max_time_ms` caps total execution time (default 60000)
|
|
154
|
+
- `schema` enables structured extraction from each page -- results are merged across sources
|
|
155
|
+
|
|
156
|
+
## Anti-Patterns
|
|
157
|
+
|
|
158
|
+
These waste tokens, time, and rate limits. Avoid them.
|
|
159
|
+
|
|
160
|
+
**Do not retry the same query.** If `search` returns no results, reformulate with different keywords. Repeating an identical query returns the same empty results.
|
|
161
|
+
|
|
162
|
+
**Do not skip the cache.** Every `fetch`, `search`, and `crawl` result is cached locally. Before any network call, run `cache` with the URL pattern or query text. Cached results return instantly.
|
|
163
|
+
|
|
164
|
+
**Do not send natural language questions as search queries.** Search engines work best with keywords. Instead of `"What is the best way to handle authentication in Next.js?"`, use `"Next.js authentication best practices 2025"`.
|
|
165
|
+
|
|
166
|
+
**Do not use `agent` for simple lookups.** One fact from one URL = `fetch`. Quick search result = `search`. Reserve `agent` for tasks requiring multiple search/fetch cycles.
|
|
167
|
+
|
|
168
|
+
**Do not use `research` when you already know the URLs.** If you have URLs to read, use `fetch` or `crawl`. `research` is for when you need the tool to discover sources autonomously.
|
|
169
|
+
|
|
170
|
+
**Do not fetch entire pages when you need one section.** Use `fetch` with `section` to extract just the relevant part.
|
|
171
|
+
|
|
172
|
+
**Do not crawl with high max_pages without filtering.** A `max_pages: 100` crawl without `include_patterns` fetches navigation pages, footers, and irrelevant content.
|
|
173
|
+
|
|
174
|
+
**Do not ignore `format: "context"` for search.** When injecting search results into a prompt, use `format: "context"` instead of manually concatenating results.
|
|
128
175
|
|
|
129
176
|
## Key Features
|
|
130
177
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AgentPlan } from './planner.js';
|
|
2
|
+
import type { AgentSource, AgentStep, SearchEngine } from '../types.js';
|
|
3
|
+
import type { SmartRouter } from '../fetch/router.js';
|
|
4
|
+
export interface ExecutionBudget {
|
|
5
|
+
maxPages: number;
|
|
6
|
+
deadlineMs: number;
|
|
7
|
+
}
|
|
8
|
+
export interface ExecutionResult {
|
|
9
|
+
sources: AgentSource[];
|
|
10
|
+
steps: AgentStep[];
|
|
11
|
+
}
|
|
12
|
+
export declare function executeAgentPlan(plan: AgentPlan, engines: SearchEngine[], router: SmartRouter, budget: ExecutionBudget): Promise<ExecutionResult>;
|
|
13
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/agent/executor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAMtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,KAAK,EAAE,SAAS,EAAE,CAAC;CACpB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,eAAe,CAAC,CAgD1B"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { createLogger } from '../logger.js';
|
|
2
|
+
import { deduplicateResults } from '../search/dedup.js';
|
|
3
|
+
import { extractContent } from '../extraction/pipeline.js';
|
|
4
|
+
import { cacheContent } from '../cache/store.js';
|
|
5
|
+
const log = createLogger('agent');
|
|
6
|
+
const FETCH_TIMEOUT_MS = 15000;
|
|
7
|
+
export async function executeAgentPlan(plan, engines, router, budget) {
|
|
8
|
+
const steps = [];
|
|
9
|
+
const allUrls = new Set();
|
|
10
|
+
try {
|
|
11
|
+
// Phase 1: Execute search queries
|
|
12
|
+
if (plan.searches.length > 0) {
|
|
13
|
+
const searchStart = Date.now();
|
|
14
|
+
const searchResults = await executeSearches(plan.searches, engines, budget.deadlineMs);
|
|
15
|
+
steps.push({
|
|
16
|
+
action: 'search',
|
|
17
|
+
detail: `Searched ${plan.searches.length} queries, found ${searchResults.length} results`,
|
|
18
|
+
time_ms: Date.now() - searchStart,
|
|
19
|
+
});
|
|
20
|
+
for (const result of searchResults) {
|
|
21
|
+
allUrls.add(result.url);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Phase 2: Add explicit URLs
|
|
25
|
+
for (const url of plan.urls) {
|
|
26
|
+
allUrls.add(url);
|
|
27
|
+
}
|
|
28
|
+
if (allUrls.size === 0) {
|
|
29
|
+
return { sources: [], steps };
|
|
30
|
+
}
|
|
31
|
+
// Phase 3: Fetch pages within budget
|
|
32
|
+
const urlsToFetch = [...allUrls].slice(0, budget.maxPages);
|
|
33
|
+
const fetchStart = Date.now();
|
|
34
|
+
const sources = await fetchPages(urlsToFetch, router, budget);
|
|
35
|
+
steps.push({
|
|
36
|
+
action: 'fetch',
|
|
37
|
+
detail: `Fetched ${sources.filter((s) => s.fetched).length}/${urlsToFetch.length} pages`,
|
|
38
|
+
time_ms: Date.now() - fetchStart,
|
|
39
|
+
});
|
|
40
|
+
return { sources, steps };
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
log.error('execution failed', {
|
|
44
|
+
error: err instanceof Error ? err.message : String(err),
|
|
45
|
+
});
|
|
46
|
+
return { sources: [], steps };
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async function executeSearches(queries, engines, deadlineMs) {
|
|
50
|
+
const allRaw = [];
|
|
51
|
+
const searchPromises = engines.flatMap((engine) => queries.map(async (query) => {
|
|
52
|
+
if (Date.now() >= deadlineMs)
|
|
53
|
+
return;
|
|
54
|
+
try {
|
|
55
|
+
const results = await engine.search(query, { maxResults: 10 });
|
|
56
|
+
for (const r of results) {
|
|
57
|
+
allRaw.push(r);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
log.warn('agent search query failed', {
|
|
62
|
+
engine: engine.name,
|
|
63
|
+
query,
|
|
64
|
+
error: err instanceof Error ? err.message : String(err),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}));
|
|
68
|
+
await Promise.allSettled(searchPromises);
|
|
69
|
+
const merged = deduplicateResults(allRaw);
|
|
70
|
+
return merged.map((m) => ({
|
|
71
|
+
url: m.url,
|
|
72
|
+
title: m.title,
|
|
73
|
+
snippet: m.snippet,
|
|
74
|
+
relevance_score: m.relevance_score,
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
async function fetchPages(urls, router, budget) {
|
|
78
|
+
const fetchPromises = urls.map(async (url) => {
|
|
79
|
+
if (Date.now() >= budget.deadlineMs) {
|
|
80
|
+
return {
|
|
81
|
+
url,
|
|
82
|
+
title: '',
|
|
83
|
+
markdown_content: '',
|
|
84
|
+
fetched: false,
|
|
85
|
+
fetch_error: 'budget exceeded',
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
const timeRemaining = budget.deadlineMs - Date.now();
|
|
90
|
+
const fetchTimeout = Math.min(FETCH_TIMEOUT_MS, Math.max(timeRemaining, 1000));
|
|
91
|
+
const raw = await Promise.race([
|
|
92
|
+
router.fetch(url, { renderJs: 'auto' }),
|
|
93
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('fetch timeout')), fetchTimeout)),
|
|
94
|
+
]);
|
|
95
|
+
const extraction = await extractContent(raw.html, raw.finalUrl, {
|
|
96
|
+
maxChars: 30000,
|
|
97
|
+
contentType: raw.contentType,
|
|
98
|
+
});
|
|
99
|
+
try {
|
|
100
|
+
cacheContent(raw, extraction);
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
log.debug('failed to cache agent source', { url, error: String(err) });
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
url,
|
|
107
|
+
title: extraction.title,
|
|
108
|
+
markdown_content: extraction.markdown,
|
|
109
|
+
fetched: true,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
log.debug('agent fetch failed', {
|
|
114
|
+
url,
|
|
115
|
+
error: err instanceof Error ? err.message : String(err),
|
|
116
|
+
});
|
|
117
|
+
return {
|
|
118
|
+
url,
|
|
119
|
+
title: '',
|
|
120
|
+
markdown_content: '',
|
|
121
|
+
fetched: false,
|
|
122
|
+
fetch_error: err instanceof Error ? err.message : String(err),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
return Promise.all(fetchPromises);
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=executor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.js","sourceRoot":"","sources":["../../src/agent/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAY/B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAe,EACf,OAAuB,EACvB,MAAmB,EACnB,MAAuB;IAEvB,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,IAAI,CAAC;QACH,kCAAkC;QAClC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAEvF,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,YAAY,IAAI,CAAC,QAAQ,CAAC,MAAM,mBAAmB,aAAa,CAAC,MAAM,UAAU;gBACzF,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;aAClC,CAAC,CAAC;YAEH,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,qCAAqC;QACrC,MAAM,WAAW,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAE9D,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,WAAW,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,QAAQ;YACxF,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACjC,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC5B,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAiB,EACjB,OAAuB,EACvB,UAAkB;IAElB,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAChD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU;YAAE,OAAO;QAErC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACpC,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,KAAK;gBACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAEzC,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxB,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAc,EACd,MAAmB,EACnB,MAAuB;IAEvB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAwB,EAAE;QACjE,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpC,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,gBAAgB,EAAE,EAAE;gBACpB,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,iBAAiB;aAC/B,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;YAE/E,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC7B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,YAAY,CAAC,CACnE;aACF,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;gBAC9D,QAAQ,EAAE,KAAK;gBACf,WAAW,EAAE,GAAG,CAAC,WAAW;aAC7B,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,gBAAgB,EAAE,UAAU,CAAC,QAAQ;gBACrC,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE;gBAC9B,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,gBAAgB,EAAE,EAAE;gBACpB,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type SamplingCapableServer } from '../search/sampling.js';
|
|
2
|
+
import type { AgentInput, AgentOutput, SearchEngine } from '../types.js';
|
|
3
|
+
import type { SmartRouter } from '../fetch/router.js';
|
|
4
|
+
export declare function runAgentPipeline(input: AgentInput, engines: SearchEngine[], router: SmartRouter, server?: SamplingCapableServer): Promise<AgentOutput>;
|
|
5
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/agent/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EAGX,YAAY,EACb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAQtD,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,WAAW,CAAC,CAyFtB"}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { createLogger } from '../logger.js';
|
|
2
|
+
import { planExecution } from './planner.js';
|
|
3
|
+
import { executeAgentPlan } from './executor.js';
|
|
4
|
+
import { extractWithSchema } from '../extraction/schema.js';
|
|
5
|
+
import { requestSampling, checkSamplingSupport, } from '../search/sampling.js';
|
|
6
|
+
const log = createLogger('agent');
|
|
7
|
+
const DEFAULT_MAX_PAGES = 10;
|
|
8
|
+
const DEFAULT_MAX_TIME_MS = 60000;
|
|
9
|
+
export async function runAgentPipeline(input, engines, router, server) {
|
|
10
|
+
const start = Date.now();
|
|
11
|
+
const maxPages = input.max_pages ?? DEFAULT_MAX_PAGES;
|
|
12
|
+
const maxTimeMs = input.max_time_ms ?? DEFAULT_MAX_TIME_MS;
|
|
13
|
+
const deadlineMs = start + maxTimeMs;
|
|
14
|
+
const steps = [];
|
|
15
|
+
try {
|
|
16
|
+
const planStart = Date.now();
|
|
17
|
+
log.info('agent pipeline started', { prompt: input.prompt.slice(0, 100), maxPages, maxTimeMs });
|
|
18
|
+
const plan = await planExecution(input.prompt, input.urls, server);
|
|
19
|
+
steps.push({
|
|
20
|
+
action: 'plan',
|
|
21
|
+
detail: `Generated ${plan.searches.length} searches, ${plan.urls.length} URLs${plan.samplingUsed ? ' (via sampling)' : ' (keyword extraction)'}`,
|
|
22
|
+
time_ms: Date.now() - planStart,
|
|
23
|
+
});
|
|
24
|
+
log.info('plan generated', {
|
|
25
|
+
searches: plan.searches.length,
|
|
26
|
+
urls: plan.urls.length,
|
|
27
|
+
samplingUsed: plan.samplingUsed,
|
|
28
|
+
});
|
|
29
|
+
const execResult = await executeAgentPlan(plan, engines, router, {
|
|
30
|
+
maxPages,
|
|
31
|
+
deadlineMs,
|
|
32
|
+
});
|
|
33
|
+
steps.push(...execResult.steps);
|
|
34
|
+
const sources = execResult.sources;
|
|
35
|
+
const pagesFetched = sources.filter((s) => s.fetched).length;
|
|
36
|
+
if (input.schema && sources.some((s) => s.fetched)) {
|
|
37
|
+
const extractStart = Date.now();
|
|
38
|
+
const schemaResult = applySchemaExtraction(sources, input.schema);
|
|
39
|
+
steps.push({
|
|
40
|
+
action: 'extract',
|
|
41
|
+
detail: `Applied schema extraction to ${sources.filter((s) => s.fetched).length} sources`,
|
|
42
|
+
time_ms: Date.now() - extractStart,
|
|
43
|
+
});
|
|
44
|
+
if (schemaResult) {
|
|
45
|
+
return {
|
|
46
|
+
result: schemaResult,
|
|
47
|
+
sources,
|
|
48
|
+
pages_fetched: pagesFetched,
|
|
49
|
+
steps,
|
|
50
|
+
total_time_ms: Date.now() - start,
|
|
51
|
+
sampling_supported: !!server,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const synthStart = Date.now();
|
|
56
|
+
const result = await synthesizeResult(input.prompt, sources, server);
|
|
57
|
+
steps.push({
|
|
58
|
+
action: 'synthesize',
|
|
59
|
+
detail: `Produced ${typeof result === 'string' ? result.length : JSON.stringify(result).length} char result${server ? ' (via sampling)' : ''}`,
|
|
60
|
+
time_ms: Date.now() - synthStart,
|
|
61
|
+
});
|
|
62
|
+
return {
|
|
63
|
+
result,
|
|
64
|
+
sources,
|
|
65
|
+
pages_fetched: pagesFetched,
|
|
66
|
+
steps,
|
|
67
|
+
total_time_ms: Date.now() - start,
|
|
68
|
+
sampling_supported: !!server,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
log.error('agent pipeline failed', {
|
|
73
|
+
prompt: input.prompt.slice(0, 100),
|
|
74
|
+
error: err instanceof Error ? err.message : String(err),
|
|
75
|
+
});
|
|
76
|
+
return {
|
|
77
|
+
result: '',
|
|
78
|
+
sources: [],
|
|
79
|
+
pages_fetched: 0,
|
|
80
|
+
steps,
|
|
81
|
+
total_time_ms: Date.now() - start,
|
|
82
|
+
sampling_supported: !!server,
|
|
83
|
+
error: err instanceof Error ? err.message : String(err),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function applySchemaExtraction(sources, schema) {
|
|
88
|
+
try {
|
|
89
|
+
const fetchedSources = sources.filter((s) => s.fetched && s.markdown_content.length > 0);
|
|
90
|
+
if (fetchedSources.length === 0)
|
|
91
|
+
return null;
|
|
92
|
+
const mergedData = {};
|
|
93
|
+
for (const source of fetchedSources) {
|
|
94
|
+
try {
|
|
95
|
+
const html = `<html><body>${source.markdown_content}</body></html>`;
|
|
96
|
+
const extracted = extractWithSchema(html, schema);
|
|
97
|
+
for (const [key, value] of Object.entries(extracted)) {
|
|
98
|
+
if (value !== undefined && value !== null && value !== '') {
|
|
99
|
+
if (!(key in mergedData)) {
|
|
100
|
+
mergedData[key] = value;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
log.debug('schema extraction failed for source', {
|
|
107
|
+
url: source.url,
|
|
108
|
+
error: err instanceof Error ? err.message : String(err),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return Object.keys(mergedData).length > 0 ? mergedData : null;
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
log.warn('schema extraction phase failed', {
|
|
116
|
+
error: err instanceof Error ? err.message : String(err),
|
|
117
|
+
});
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async function synthesizeResult(prompt, sources, server) {
|
|
122
|
+
const fetchedSources = sources.filter((s) => s.fetched && s.markdown_content.length > 0);
|
|
123
|
+
if (fetchedSources.length === 0) {
|
|
124
|
+
return 'No data could be gathered for this request.';
|
|
125
|
+
}
|
|
126
|
+
if (server) {
|
|
127
|
+
try {
|
|
128
|
+
const result = await synthesizeWithSampling(prompt, fetchedSources, server);
|
|
129
|
+
if (result)
|
|
130
|
+
return result;
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
log.warn('sampling synthesis failed, using fallback', {
|
|
134
|
+
error: err instanceof Error ? err.message : String(err),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return buildFallbackSynthesis(prompt, fetchedSources);
|
|
139
|
+
}
|
|
140
|
+
async function synthesizeWithSampling(prompt, sources, server) {
|
|
141
|
+
try {
|
|
142
|
+
const maxCharsPerSource = 3000;
|
|
143
|
+
const sourceBlocks = sources.map((s, i) => {
|
|
144
|
+
const content = s.markdown_content.slice(0, maxCharsPerSource);
|
|
145
|
+
return `[${i + 1}] ${s.title} (${s.url})\n${content}`;
|
|
146
|
+
});
|
|
147
|
+
const totalSourceText = sourceBlocks.join('\n\n');
|
|
148
|
+
const truncatedSourceText = totalSourceText.slice(0, 40000);
|
|
149
|
+
const samplingPrompt = `You are a data gathering assistant. Based on the user's request and the gathered sources, synthesize a comprehensive result.
|
|
150
|
+
|
|
151
|
+
User's request: ${prompt}
|
|
152
|
+
|
|
153
|
+
Gathered sources:
|
|
154
|
+
${truncatedSourceText}
|
|
155
|
+
|
|
156
|
+
Provide a clear, well-organized response that addresses the user's request based on the gathered data. Include source references [1], [2], etc.`;
|
|
157
|
+
if (!checkSamplingSupport(server)) {
|
|
158
|
+
log.debug('client does not support sampling for synthesis');
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
const response = await requestSampling(server, [{ role: 'user', content: { type: 'text', text: samplingPrompt } }], 2000);
|
|
162
|
+
if (response?.content?.text && response.content.text.trim().length > 0) {
|
|
163
|
+
return response.content.text.trim();
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
log.debug('sampling synthesis failed', {
|
|
169
|
+
error: err instanceof Error ? err.message : String(err),
|
|
170
|
+
});
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
function buildFallbackSynthesis(prompt, sources) {
|
|
175
|
+
const header = `## Results: ${prompt}\n\nGathered from ${sources.length} source(s):\n\n`;
|
|
176
|
+
let result = header;
|
|
177
|
+
const maxTotal = 6000;
|
|
178
|
+
let remaining = maxTotal - header.length;
|
|
179
|
+
for (let i = 0; i < sources.length && remaining > 0; i++) {
|
|
180
|
+
const source = sources[i];
|
|
181
|
+
const sourceHeader = `### [${i + 1}] ${source.title}\n**URL:** ${source.url}\n\n`;
|
|
182
|
+
if (remaining < sourceHeader.length + 20)
|
|
183
|
+
break;
|
|
184
|
+
result += sourceHeader;
|
|
185
|
+
remaining -= sourceHeader.length;
|
|
186
|
+
const contentBudget = Math.min(remaining - 10, source.markdown_content.length, 1500);
|
|
187
|
+
if (contentBudget > 0) {
|
|
188
|
+
let content = source.markdown_content.slice(0, contentBudget);
|
|
189
|
+
if (content.length < source.markdown_content.length) {
|
|
190
|
+
content = content.slice(0, Math.max(contentBudget - 3, 0)) + '...';
|
|
191
|
+
}
|
|
192
|
+
result += content + '\n\n';
|
|
193
|
+
remaining -= content.length + 2;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return result.trimEnd();
|
|
197
|
+
}
|
|
198
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/agent/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAEL,eAAe,EACf,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAW/B,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAiB,EACjB,OAAuB,EACvB,MAAmB,EACnB,MAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,iBAAiB,CAAC;IACtD,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC3D,MAAM,UAAU,GAAG,KAAK,GAAG,SAAS,CAAC;IACrC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAEhG,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEnE,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,aAAa,IAAI,CAAC,QAAQ,CAAC,MAAM,cAAc,IAAI,CAAC,IAAI,CAAC,MAAM,QAAQ,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,uBAAuB,EAAE;YAChJ,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAChC,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YAC9B,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;YACtB,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE;YAC/D,QAAQ;YACR,UAAU;SACX,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QACnC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAE7D,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAoB,CAAC,CAAC;YAEhF,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,gCAAgC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU;gBACzF,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;aACnC,CAAC,CAAC;YAEH,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO;oBACL,MAAM,EAAE,YAAY;oBACpB,OAAO;oBACP,aAAa,EAAE,YAAY;oBAC3B,KAAK;oBACL,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;oBACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM;iBAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAErE,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,YAAY,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,eAAe,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9I,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACjC,CAAC,CAAC;QAEH,OAAO;YACL,MAAM;YACN,OAAO;YACP,aAAa,EAAE,YAAY;YAC3B,KAAK;YACL,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE;YACjC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO;YACL,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,EAAE;YACX,aAAa,EAAE,CAAC;YAChB,KAAK;YACL,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM;YAC5B,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAsB,EACtB,MAAkB;IAElB,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAE7C,MAAM,UAAU,GAA4B,EAAE,CAAC;QAE/C,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,eAAe,MAAM,CAAC,gBAAgB,gBAAgB,CAAC;gBACpE,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAElD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;oBACrD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;wBAC1D,IAAI,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC;4BACzB,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;wBAC1B,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;oBAC/C,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE;YACzC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,MAAc,EACd,OAAsB,EACtB,MAA8B;IAE9B,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,6CAA6C,CAAC;IACvD,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YAC5E,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBACpD,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,sBAAsB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,OAAsB,EACtB,MAA6B;IAE7B,IAAI,CAAC;QACH,MAAM,iBAAiB,GAAG,IAAI,CAAC;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,MAAM,OAAO,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,mBAAmB,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAE5D,MAAM,cAAc,GAAG;;kBAET,MAAM;;;EAGtB,mBAAmB;;gJAE2H,CAAC;QAE7I,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,GAAG,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAC5D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,eAAe,CACpC,MAAM,EACN,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,CAAC,EACnE,IAAI,CACL,CAAC;QAEF,IAAI,QAAQ,EAAE,OAAO,EAAE,IAAI,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvE,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,2BAA2B,EAAE;YACrC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAc,EAAE,OAAsB;IACpE,MAAM,MAAM,GAAG,eAAe,MAAM,qBAAqB,OAAO,CAAC,MAAM,iBAAiB,CAAC;IACzF,IAAI,MAAM,GAAG,MAAM,CAAC;IACpB,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtB,IAAI,SAAS,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,cAAc,MAAM,CAAC,GAAG,MAAM,CAAC;QAElF,IAAI,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,EAAE;YAAE,MAAM;QAEhD,MAAM,IAAI,YAAY,CAAC;QACvB,SAAS,IAAI,YAAY,CAAC,MAAM,CAAC;QAEjC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACrF,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;YAC9D,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACpD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;YACrE,CAAC;YACD,MAAM,IAAI,OAAO,GAAG,MAAM,CAAC;YAC3B,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type SamplingCapableServer } from '../search/sampling.js';
|
|
2
|
+
export interface AgentPlan {
|
|
3
|
+
searches: string[];
|
|
4
|
+
urls: string[];
|
|
5
|
+
notes: string;
|
|
6
|
+
samplingUsed: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function planExecution(prompt: string, urls?: string[], server?: SamplingCapableServer): Promise<AgentPlan>;
|
|
9
|
+
//# sourceMappingURL=planner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../src/agent/planner.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAC;AAO/B,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,SAAS,CAAC,CAwCpB"}
|