better-web-search-mcp 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -112
- package/dist/extraction/router.d.ts +2 -0
- package/dist/extraction/router.d.ts.map +1 -1
- package/dist/extraction/router.js +81 -2
- package/dist/extraction/router.js.map +1 -1
- package/dist/index.js +55 -32
- package/dist/index.js.map +1 -1
- package/dist/providers/brave.js +1 -1
- package/dist/providers/brave.js.map +1 -1
- package/dist/providers/index.d.ts +10 -4
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +28 -7
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/tavily.js +1 -1
- package/dist/providers/tavily.js.map +1 -1
- package/dist/tools/extract.d.ts +15 -3
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +21 -7
- package/dist/tools/extract.js.map +1 -1
- package/dist/utils/retry.d.ts +5 -1
- package/dist/utils/retry.d.ts.map +1 -1
- package/dist/utils/retry.js +9 -0
- package/dist/utils/retry.js.map +1 -1
- package/package.json +2 -2
- package/dist/extraction/alternative.test.d.ts +0 -2
- package/dist/extraction/alternative.test.d.ts.map +0 -1
- package/dist/extraction/alternative.test.js +0 -145
- package/dist/extraction/alternative.test.js.map +0 -1
- package/dist/extraction/apiIntercept.test.d.ts +0 -2
- package/dist/extraction/apiIntercept.test.d.ts.map +0 -1
- package/dist/extraction/apiIntercept.test.js +0 -144
- package/dist/extraction/apiIntercept.test.js.map +0 -1
- package/dist/extraction/browser.test.d.ts +0 -2
- package/dist/extraction/browser.test.d.ts.map +0 -1
- package/dist/extraction/browser.test.js +0 -137
- package/dist/extraction/browser.test.js.map +0 -1
- package/dist/extraction/fetch.test.d.ts +0 -2
- package/dist/extraction/fetch.test.d.ts.map +0 -1
- package/dist/extraction/fetch.test.js +0 -85
- package/dist/extraction/fetch.test.js.map +0 -1
- package/dist/extraction/fusion.test.d.ts +0 -2
- package/dist/extraction/fusion.test.d.ts.map +0 -1
- package/dist/extraction/fusion.test.js +0 -91
- package/dist/extraction/fusion.test.js.map +0 -1
- package/dist/extraction/metadata.test.d.ts +0 -2
- package/dist/extraction/metadata.test.d.ts.map +0 -1
- package/dist/extraction/metadata.test.js +0 -45
- package/dist/extraction/metadata.test.js.map +0 -1
- package/dist/extraction/readability.test.d.ts +0 -2
- package/dist/extraction/readability.test.d.ts.map +0 -1
- package/dist/extraction/readability.test.js +0 -51
- package/dist/extraction/readability.test.js.map +0 -1
- package/dist/extraction/router.test.d.ts +0 -2
- package/dist/extraction/router.test.d.ts.map +0 -1
- package/dist/extraction/router.test.js +0 -238
- package/dist/extraction/router.test.js.map +0 -1
- package/dist/extraction/structured.test.d.ts +0 -2
- package/dist/extraction/structured.test.d.ts.map +0 -1
- package/dist/extraction/structured.test.js +0 -85
- package/dist/extraction/structured.test.js.map +0 -1
- package/dist/providers/brave.test.d.ts +0 -2
- package/dist/providers/brave.test.d.ts.map +0 -1
- package/dist/providers/brave.test.js +0 -150
- package/dist/providers/brave.test.js.map +0 -1
- package/dist/providers/duckduckgo.test.d.ts +0 -2
- package/dist/providers/duckduckgo.test.d.ts.map +0 -1
- package/dist/providers/duckduckgo.test.js +0 -102
- package/dist/providers/duckduckgo.test.js.map +0 -1
- package/dist/providers/index.test.d.ts +0 -2
- package/dist/providers/index.test.d.ts.map +0 -1
- package/dist/providers/index.test.js +0 -114
- package/dist/providers/index.test.js.map +0 -1
- package/dist/providers/tavily.test.d.ts +0 -2
- package/dist/providers/tavily.test.d.ts.map +0 -1
- package/dist/providers/tavily.test.js +0 -123
- package/dist/providers/tavily.test.js.map +0 -1
- package/dist/ranking/deduplicate.test.d.ts +0 -2
- package/dist/ranking/deduplicate.test.d.ts.map +0 -1
- package/dist/ranking/deduplicate.test.js +0 -76
- package/dist/ranking/deduplicate.test.js.map +0 -1
- package/dist/ranking/domainScore.test.d.ts +0 -2
- package/dist/ranking/domainScore.test.d.ts.map +0 -1
- package/dist/ranking/domainScore.test.js +0 -37
- package/dist/ranking/domainScore.test.js.map +0 -1
- package/dist/ranking/rerank.test.d.ts +0 -2
- package/dist/ranking/rerank.test.d.ts.map +0 -1
- package/dist/ranking/rerank.test.js +0 -71
- package/dist/ranking/rerank.test.js.map +0 -1
- package/dist/scaffold.test.d.ts +0 -2
- package/dist/scaffold.test.d.ts.map +0 -1
- package/dist/scaffold.test.js +0 -7
- package/dist/scaffold.test.js.map +0 -1
- package/dist/tools/extract.test.d.ts +0 -2
- package/dist/tools/extract.test.d.ts.map +0 -1
- package/dist/tools/extract.test.js +0 -110
- package/dist/tools/extract.test.js.map +0 -1
- package/dist/tools/research.test.d.ts +0 -2
- package/dist/tools/research.test.d.ts.map +0 -1
- package/dist/tools/research.test.js +0 -181
- package/dist/tools/research.test.js.map +0 -1
- package/dist/tools/search.test.d.ts +0 -2
- package/dist/tools/search.test.d.ts.map +0 -1
- package/dist/tools/search.test.js +0 -101
- package/dist/tools/search.test.js.map +0 -1
- package/dist/utils/cache.test.d.ts +0 -2
- package/dist/utils/cache.test.d.ts.map +0 -1
- package/dist/utils/cache.test.js +0 -172
- package/dist/utils/cache.test.js.map +0 -1
- package/dist/utils/config.test.d.ts +0 -2
- package/dist/utils/config.test.d.ts.map +0 -1
- package/dist/utils/config.test.js +0 -52
- package/dist/utils/config.test.js.map +0 -1
- package/dist/utils/domainProfile.test.d.ts +0 -2
- package/dist/utils/domainProfile.test.d.ts.map +0 -1
- package/dist/utils/domainProfile.test.js +0 -74
- package/dist/utils/domainProfile.test.js.map +0 -1
- package/dist/utils/queries.test.d.ts +0 -2
- package/dist/utils/queries.test.d.ts.map +0 -1
- package/dist/utils/queries.test.js +0 -36
- package/dist/utils/queries.test.js.map +0 -1
- package/dist/utils/retry.test.d.ts +0 -2
- package/dist/utils/retry.test.d.ts.map +0 -1
- package/dist/utils/retry.test.js +0 -143
- package/dist/utils/retry.test.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,112 +1,112 @@
|
|
|
1
|
-
# BetterWebSearch MCP
|
|
2
|
-
|
|
3
|
-
> Keyless-first web search for AI agents. DuckDuckGo works out of the box. No API keys.
|
|
4
|
-
|
|
5
|
-
[](https://www.npmjs.com/package/better-web-search-mcp)
|
|
6
|
-
[](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions)
|
|
7
|
-
[](https://phantompixeldev.github.io/BetterWebSearch-MCP/)
|
|
8
|
-
[](LICENSE)
|
|
9
|
-
[](https://smithery.ai/server/@PhantomPixelDev/better-web-search-mcp)
|
|
10
|
-
|
|
11
|
-
<p align="center">
|
|
12
|
-
<img src="./images/banner.jpg" alt="BetterWebSearch MCP — Smarter Web Search · Better Data Extraction · Flexible & Extensible · Open Source" width="100%">
|
|
13
|
-
</p>
|
|
14
|
-
|
|
15
|
-
## What is this?
|
|
16
|
-
|
|
17
|
-
BetterWebSearch MCP is a [Model Context Protocol](https://modelcontextprotocol.io) server that gives AI assistants genuinely better web research. It works without API keys by default (DuckDuckGo), and optionally adds Brave or Tavily for richer ranking and recency.
|
|
18
|
-
|
|
19
|
-
Built for anyone who wants their AI tools to find things on the web reliably, not just when a paid API key is configured.
|
|
20
|
-
|
|
21
|
-
## Quick start (30 seconds)
|
|
22
|
-
|
|
23
|
-
Add this to your MCP client config (Claude Desktop, Cursor, VS Code Copilot, OpenCode, or any MCP client):
|
|
24
|
-
|
|
25
|
-
```json
|
|
26
|
-
{
|
|
27
|
-
"mcpServers": {
|
|
28
|
-
"better-web-search-mcp": {
|
|
29
|
-
"command": "npx",
|
|
30
|
-
"args": ["-y", "better-web-search-mcp"]
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
That's it. Every search tool works immediately. No `.env`, no API keys, no setup.
|
|
37
|
-
|
|
38
|
-
Or run it directly:
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
npx -y better-web-search-mcp --help
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
## Tools
|
|
45
|
-
|
|
46
|
-
| Tool | What it does |
|
|
47
|
-
|---|---|
|
|
48
|
-
| `web_search` | Aggregated search across providers, deduplicated and reranked |
|
|
49
|
-
| `web_research` | Deep research. Rewrites your question, searches in parallel, extracts top results with citations |
|
|
50
|
-
| `deep_search` | Alias for `web_research` |
|
|
51
|
-
| `web_extract` | Extracts clean content from any URL, using the best method available |
|
|
52
|
-
| `web_find` | Search scoped to a single website |
|
|
53
|
-
| `web_news` | Recent news with timeline and diversity filtering |
|
|
54
|
-
|
|
55
|
-
## How search is better
|
|
56
|
-
|
|
57
|
-
Standard MCP search tools hit one API and return raw results. BetterWebSearch takes a different approach.
|
|
58
|
-
|
|
59
|
-
**Query expansion.** Your single question gets rewritten into 4 to 6 variants, searched in parallel. "Unlimited mobile internet Germany" also searches for "unbegrenztes Datenvolumen Deutschland" and similar phrasings. More coverage without you doing extra work.
|
|
60
|
-
|
|
61
|
-
**Three-tier extraction.** When it fetches a page, it doesn't just grab the HTML and hope for the best:
|
|
62
|
-
|
|
63
|
-
```
|
|
64
|
-
Tier 1: Fast HTTP fetch (< 1s)
|
|
65
|
-
↓ if not enough content
|
|
66
|
-
Tier 2: Structured data (JSON-LD, __NEXT_DATA__, __NUXT__) (1-3s)
|
|
67
|
-
↓ if still not enough
|
|
68
|
-
Tier 3: Playwright browser (full render + API interception) (3-10s)
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
Only escalates when needed. Results carry confidence scores so you know what you're getting.
|
|
72
|
-
|
|
73
|
-
**Self-learning cache.** The first visit to a domain takes the full path. The second visit skips straight to what worked. Domain profiles and API patterns are remembered.
|
|
74
|
-
|
|
75
|
-
## Project structure
|
|
76
|
-
|
|
77
|
-
```
|
|
78
|
-
src/
|
|
79
|
-
providers/ Search backends (DuckDuckGo, Brave, Tavily
|
|
80
|
-
extraction/ 3-tier content pipeline (fetch, structured, browser)
|
|
81
|
-
ranking/ Deduplication, domain scoring, reranking
|
|
82
|
-
tools/ MCP tool definitions (search, research, extract, find, news)
|
|
83
|
-
utils/ Config, caching, retry, query rewriting
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
## Configuration
|
|
87
|
-
|
|
88
|
-
All configuration is optional. DuckDuckGo works with zero setup.
|
|
89
|
-
|
|
90
|
-
```bash
|
|
91
|
-
# Optional: add more providers
|
|
92
|
-
BRAVE_API_KEY=your_key # Better ranking and recency
|
|
93
|
-
TAVILY_API_KEY=your_key # Alternative provider
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
See the [full configuration reference](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) for all options including cache paths, browser settings, and environment variable names.
|
|
97
|
-
|
|
98
|
-
## Documentation
|
|
99
|
-
|
|
100
|
-
Full guides, API reference, and architecture details live on the docs site:
|
|
101
|
-
|
|
102
|
-
**[phantompixeldev.github.io/BetterWebSearch-MCP](https://phantompixeldev.github.io/BetterWebSearch-MCP/)**
|
|
103
|
-
|
|
104
|
-
- [Installation](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/installation/) - npx, npm, or from source
|
|
105
|
-
- [Quick Start](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/quickstart/) - Get running in your first AI client
|
|
106
|
-
- [Configuration](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) - All environment variables and options
|
|
107
|
-
- [Contributing](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/contributing/) - Development setup and PR process
|
|
108
|
-
- [Changelog](CHANGELOG.md) - Release history and version notes
|
|
109
|
-
|
|
110
|
-
## License
|
|
111
|
-
|
|
112
|
-
MIT
|
|
1
|
+
# BetterWebSearch MCP
|
|
2
|
+
|
|
3
|
+
> Keyless-first web search for AI agents. DuckDuckGo works out of the box. No API keys.
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/better-web-search-mcp)
|
|
6
|
+
[](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions)
|
|
7
|
+
[](https://phantompixeldev.github.io/BetterWebSearch-MCP/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
[](https://smithery.ai/server/@PhantomPixelDev/better-web-search-mcp)
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<img src="./images/banner.jpg" alt="BetterWebSearch MCP — Smarter Web Search · Better Data Extraction · Flexible & Extensible · Open Source" width="100%">
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
## What is this?
|
|
16
|
+
|
|
17
|
+
BetterWebSearch MCP is a [Model Context Protocol](https://modelcontextprotocol.io) server that gives AI assistants genuinely better web research. It works without API keys by default (DuckDuckGo), and optionally adds Brave or Tavily for richer ranking and recency.
|
|
18
|
+
|
|
19
|
+
Built for anyone who wants their AI tools to find things on the web reliably, not just when a paid API key is configured.
|
|
20
|
+
|
|
21
|
+
## Quick start (30 seconds)
|
|
22
|
+
|
|
23
|
+
Add this to your MCP client config (Claude Desktop, Cursor, VS Code Copilot, OpenCode, or any MCP client):
|
|
24
|
+
|
|
25
|
+
```json
|
|
26
|
+
{
|
|
27
|
+
"mcpServers": {
|
|
28
|
+
"better-web-search-mcp": {
|
|
29
|
+
"command": "npx",
|
|
30
|
+
"args": ["-y", "better-web-search-mcp"]
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
That's it. Every search tool works immediately. No `.env`, no API keys, no setup.
|
|
37
|
+
|
|
38
|
+
Or run it directly:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx -y better-web-search-mcp --help
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Tools
|
|
45
|
+
|
|
46
|
+
| Tool | What it does |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `web_search` | Aggregated search across providers, deduplicated and reranked |
|
|
49
|
+
| `web_research` | Deep research. Rewrites your question, searches in parallel, extracts top results with citations |
|
|
50
|
+
| `deep_search` | Alias for `web_research` |
|
|
51
|
+
| `web_extract` | Extracts clean content from any URL, using the best method available |
|
|
52
|
+
| `web_find` | Search scoped to a single website |
|
|
53
|
+
| `web_news` | Recent news with timeline and diversity filtering |
|
|
54
|
+
|
|
55
|
+
## How search is better
|
|
56
|
+
|
|
57
|
+
Standard MCP search tools hit one API and return raw results. BetterWebSearch takes a different approach.
|
|
58
|
+
|
|
59
|
+
**Query expansion.** Your single question gets rewritten into 4 to 6 variants, searched in parallel. "Unlimited mobile internet Germany" also searches for "unbegrenztes Datenvolumen Deutschland" and similar phrasings. More coverage without you doing extra work.
|
|
60
|
+
|
|
61
|
+
**Three-tier extraction.** When it fetches a page, it doesn't just grab the HTML and hope for the best:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
Tier 1: Fast HTTP fetch (< 1s)
|
|
65
|
+
↓ if not enough content
|
|
66
|
+
Tier 2: Structured data (JSON-LD, __NEXT_DATA__, __NUXT__) (1-3s)
|
|
67
|
+
↓ if still not enough
|
|
68
|
+
Tier 3: Playwright browser (full render + API interception) (3-10s)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Only escalates when needed. Results carry confidence scores so you know what you're getting.
|
|
72
|
+
|
|
73
|
+
**Self-learning cache.** The first visit to a domain takes the full path. The second visit skips straight to what worked. Domain profiles and API patterns are remembered.
|
|
74
|
+
|
|
75
|
+
## Project structure
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
src/
|
|
79
|
+
providers/ Search backends (DuckDuckGo, Brave, Tavily; SerpApi stubbed)
|
|
80
|
+
extraction/ 3-tier content pipeline (fetch, structured, browser)
|
|
81
|
+
ranking/ Deduplication, domain scoring, reranking
|
|
82
|
+
tools/ MCP tool definitions (search, research, extract, find, news)
|
|
83
|
+
utils/ Config, caching, retry, query rewriting
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Configuration
|
|
87
|
+
|
|
88
|
+
All configuration is optional. DuckDuckGo works with zero setup.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
# Optional: add more providers
|
|
92
|
+
BRAVE_API_KEY=your_key # Better ranking and recency
|
|
93
|
+
TAVILY_API_KEY=your_key # Alternative provider
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
See the [full configuration reference](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) for all options including cache paths, browser settings, and environment variable names.
|
|
97
|
+
|
|
98
|
+
## Documentation
|
|
99
|
+
|
|
100
|
+
Full guides, API reference, and architecture details live on the docs site:
|
|
101
|
+
|
|
102
|
+
**[phantompixeldev.github.io/BetterWebSearch-MCP](https://phantompixeldev.github.io/BetterWebSearch-MCP/)**
|
|
103
|
+
|
|
104
|
+
- [Installation](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/installation/) - npx, npm, or from source
|
|
105
|
+
- [Quick Start](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/quickstart/) - Get running in your first AI client
|
|
106
|
+
- [Configuration](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) - All environment variables and options
|
|
107
|
+
- [Contributing](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/contributing/) - Development setup and PR process
|
|
108
|
+
- [Changelog](CHANGELOG.md) - Release history and version notes
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT
|
|
@@ -82,6 +82,8 @@ export declare const RENDERED_DOM_CONFIDENCE = 0.9;
|
|
|
82
82
|
export declare const BEST_EFFORT_CONFIDENCE = 0.5;
|
|
83
83
|
/** Whether a page has enough visible text to be useful without a browser. */
|
|
84
84
|
export declare function hasEnoughContent(html: string): boolean;
|
|
85
|
+
/** Close and forget the shared browser pool (server shutdown, tests). */
|
|
86
|
+
export declare function closeSharedBrowserPool(): Promise<void>;
|
|
85
87
|
/**
|
|
86
88
|
* Fetch and extract a page, escalating through the three tiers as needed.
|
|
87
89
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/extraction/router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EACL,sBAAsB,EAEvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AACnE,OAAO,EACL,qBAAqB,EAEtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../src/extraction/router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EACL,sBAAsB,EAEvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AACnE,OAAO,EACL,qBAAqB,EAEtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAI9C,kDAAkD;AAClD,MAAM,MAAM,YAAY,GACpB,YAAY,GACZ,gBAAgB,GAChB,uBAAuB,GACvB,cAAc,CAAC;AAEnB,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,UAAU,EAAE;QACV,4CAA4C;QAC5C,MAAM,EAAE,YAAY,CAAC;QACrB,uCAAuC;QACvC,UAAU,EAAE,MAAM,CAAC;QACnB,6CAA6C;QAC7C,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,+DAA+D;IAC/D,eAAe,EAAE,OAAO,CAAC;IACzB,8DAA8D;IAC9D,aAAa,EAAE,OAAO,CAAC;IACvB,uEAAuE;IACvE,QAAQ,EAAE,YAAY,CAAC;IACvB,oFAAoF;IACpF,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;CACvB;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,uEAAuE;IACvE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACnC,oEAAoE;IACpE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iEAAiE;IACjE,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;IAC7B,sBAAsB,CAAC,EAAE,OAAO,sBAAsB,CAAC;IACvD,qBAAqB,CAAC,EAAE,OAAO,qBAAqB,CAAC;IACrD,eAAe,CAAC,EAAE,OAAO,eAAe,CAAC;CAC1C;AAED,6EAA6E;AAC7E,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,2EAA2E;AAC3E,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAC1C,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC,eAAO,MAAM,4BAA4B,OAAO,CAAC;AACjD,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAC3C,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAW1C,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEtD;AAiDD,yEAAyE;AACzE,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAM5D;AAoKD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,MAAM,EACX,IAAI,GAAE,cAAmB,EACzB,IAAI,GAAE,UAAe,GACpB,OAAO,CAAC,UAAU,CAAC,CAoIrB"}
|
|
@@ -21,6 +21,7 @@ import { extractStructuredData, } from "./structured.js";
|
|
|
21
21
|
import { BrowserPool } from "./browser.js";
|
|
22
22
|
import { Cache } from "../utils/cache.js";
|
|
23
23
|
import { recordApiPatterns, updateDomainProfile } from "./learn.js";
|
|
24
|
+
import { loadConfig } from "../utils/config.js";
|
|
24
25
|
/** Minimum visible-text length (after stripping tags) for Level 1 to win. */
|
|
25
26
|
export const ENOUGH_CONTENT_LENGTH = 500;
|
|
26
27
|
/** Minimum structured-content length for Level 2 to win. */
|
|
@@ -59,6 +60,80 @@ function normalizeOptions(opts) {
|
|
|
59
60
|
: (opts.browser_fallback ?? true),
|
|
60
61
|
};
|
|
61
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* The shared browser pool, created on the first Level 3 escalation.
|
|
65
|
+
*
|
|
66
|
+
* Callers do not construct a pool, so without this the browser tier could
|
|
67
|
+
* never run outside tests. Creation is lazy so a server that never escalates
|
|
68
|
+
* past HTTP never launches chromium.
|
|
69
|
+
*/
|
|
70
|
+
let sharedBrowserPool = null;
|
|
71
|
+
/**
|
|
72
|
+
* Pick the pool for Level 3: an injected pool wins (tests), otherwise the
|
|
73
|
+
* lazily created shared one — unless the browser tier is disabled via
|
|
74
|
+
* `BETTER_WEB_SEARCH_DISABLE_BROWSER`.
|
|
75
|
+
*/
|
|
76
|
+
function resolveBrowserPool(injected) {
|
|
77
|
+
if (injected !== undefined) {
|
|
78
|
+
return injected;
|
|
79
|
+
}
|
|
80
|
+
if (!loadConfig().browserEnabled) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
sharedBrowserPool ??= new BrowserPool();
|
|
84
|
+
return sharedBrowserPool;
|
|
85
|
+
}
|
|
86
|
+
/** Close and forget the shared browser pool (server shutdown, tests). */
|
|
87
|
+
export async function closeSharedBrowserPool() {
|
|
88
|
+
const pool = sharedBrowserPool;
|
|
89
|
+
sharedBrowserPool = null;
|
|
90
|
+
if (pool !== null) {
|
|
91
|
+
await pool.close();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Read the cached capability profile for a URL's domain.
|
|
96
|
+
*
|
|
97
|
+
* Returns `null` on a cache miss or a payload that does not carry the fields
|
|
98
|
+
* we branch on — the profile is stored as opaque JSON, so it may predate the
|
|
99
|
+
* current shape.
|
|
100
|
+
*/
|
|
101
|
+
function cachedDomainProfile(url, cache) {
|
|
102
|
+
if (cache === undefined) {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
let domain;
|
|
106
|
+
try {
|
|
107
|
+
domain = new URL(url).hostname;
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
const raw = cache.getDomain(domain);
|
|
113
|
+
if (raw === null || typeof raw !== "object") {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
const profile = raw;
|
|
117
|
+
if (typeof profile.requires_js !== "boolean" ||
|
|
118
|
+
typeof profile.best_method !== "string") {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
return raw;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Whether the browser tier can be skipped for this URL.
|
|
125
|
+
*
|
|
126
|
+
* A domain we have already served without JavaScript does not need a render
|
|
127
|
+
* that costs up to 23s; the profile written by `updateDomainProfile` after
|
|
128
|
+
* every extraction is what tells us so. Explicit `mode: "browser"` always
|
|
129
|
+
* wins over the profile.
|
|
130
|
+
*/
|
|
131
|
+
function canSkipBrowser(profile, mode) {
|
|
132
|
+
return (mode !== "browser" &&
|
|
133
|
+
profile !== null &&
|
|
134
|
+
!profile.requires_js &&
|
|
135
|
+
profile.best_method !== "browser_api_intercept");
|
|
136
|
+
}
|
|
62
137
|
function resolveDeps(deps) {
|
|
63
138
|
return {
|
|
64
139
|
cache: deps.cache,
|
|
@@ -237,9 +312,13 @@ export async function getPage(url, opts = {}, deps = {}) {
|
|
|
237
312
|
}
|
|
238
313
|
}
|
|
239
314
|
// Level 3: browser render + API interception.
|
|
240
|
-
|
|
315
|
+
const profile = cachedDomainProfile(url, d.cache);
|
|
316
|
+
const browserPool = canSkipBrowser(profile, options.mode)
|
|
317
|
+
? undefined
|
|
318
|
+
: resolveBrowserPool(d.browserPool);
|
|
319
|
+
if (options.browser_fallback && browserPool !== undefined) {
|
|
241
320
|
try {
|
|
242
|
-
const render = await
|
|
321
|
+
const render = await browserPool.renderWithBrowser(url);
|
|
243
322
|
const renderedHtml = render.html;
|
|
244
323
|
const captured = render.captured;
|
|
245
324
|
const renderedMetadata = d.extractMetadata(renderedHtml);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.js","sourceRoot":"","sources":["../../src/extraction/router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EACL,sBAAsB,GAEvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAqB,MAAM,eAAe,CAAC;AACnE,OAAO,EACL,qBAAqB,GAEtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../../src/extraction/router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EACL,sBAAsB,GAEvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAqB,MAAM,eAAe,CAAC;AACnE,OAAO,EACL,qBAAqB,GAEtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AA0DhD,6EAA6E;AAC7E,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC,4DAA4D;AAC5D,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C,2EAA2E;AAC3E,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAC1C,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAC3C,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAC3C,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;SAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,qBAAqB,CAAC;AAC3D,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;IACjC,MAAM,aAAa,GAAG,IAAI,KAAK,SAAS,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,KAAK,MAAM,CAAC;IACpC,OAAO;QACL,IAAI;QACJ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;QAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,IAAI,IAAI;QAC7D,gBAAgB,EAAE,aAAa;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,WAAW;gBACX,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;KACtC,CAAC;AACJ,CAAC;AAMD;;;;;;GAMG;AACH,IAAI,iBAAiB,GAAuB,IAAI,CAAC;AAEjD;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,QAAiC;IAEjC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC,cAAc,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,iBAAiB,KAAK,IAAI,WAAW,EAAE,CAAC;IACxC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,iBAAiB,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,SAAS,mBAAmB,CAC1B,GAAW,EACX,KAAwB;IAExB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,GAA6B,CAAC;IAC9C,IACE,OAAO,OAAO,CAAC,WAAW,KAAK,SAAS;QACxC,OAAO,OAAO,CAAC,WAAW,KAAK,QAAQ,EACvC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAoB,CAAC;AAC9B,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CACrB,OAA6B,EAC7B,IAA4B;IAE5B,OAAO,CACL,IAAI,KAAK,SAAS;QAClB,OAAO,KAAK,IAAI;QAChB,CAAC,OAAO,CAAC,WAAW;QACpB,OAAO,CAAC,WAAW,KAAK,uBAAuB,CAChD,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAgB;IACnC,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;QACtC,sBAAsB,EACpB,IAAI,CAAC,sBAAsB,IAAI,sBAAsB;QACvD,qBAAqB,EACnB,IAAI,CAAC,qBAAqB,IAAI,qBAAqB;QACrD,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,eAAe;KACzD,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,UAA0B;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;YACrE,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACrD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAAC,UAA0B;IACrD,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,IAAI,GAAI,KAAiC,CAAC,OAAO,CAAC,CAAC;QACzD,OAAO,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CACjB,GAAW,EACX,IAAY,EACZ,QAAsB,EACtB,IAAkB;IAElB,MAAM,WAAW,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,WAAW,EAAE,eAAe,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,OAAO;QACL,GAAG;QACH,KAAK,EAAE,WAAW,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;QAC3C,OAAO;QACP,UAAU,EAAE;YACV,MAAM,EAAE,YAAY;YACpB,UAAU,EAAE,sBAAsB;YAClC,QAAQ,EAAE,KAAK;SAChB;QACD,eAAe,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC;QAC3E,aAAa,EAAE,SAAS;QACxB,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,SAAS,cAAc,CAAC,OAA2B;IACjD,OAAO,CACL,OAAO,KAAK,IAAI;QAChB,OAAO,CAAC,MAAM,KAAK,GAAG;QACtB,OAAO,CAAC,MAAM,KAAK,GAAG;QACtB,OAAO,CAAC,MAAM,KAAK,GAAG,CACvB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,yBAAyB,CACtC,MAAkB,EAClB,OAA2B,EAC3B,KAAwB;IAExB,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QACpE,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,KAAK,IAAI,IAAI,EACpB,EAAE,KAAK,EAAE,CACV,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,IAAI,GACR,6FAA6F,CAAC;IAChG,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;IAC3C,OAAO;QACL,GAAG,MAAM;QACT,QAAQ;QACR,OAAO,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,OAAO,OAAO,EAAE;KACnE,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAW,EACX,OAAuB,EAAE,EACzB,OAAmB,EAAE;IAErB,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAE5B,gDAAgD;IAChD,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE;oBACV,MAAM,EAAE,MAAM,CAAC,iBAAiC;oBAChD,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,QAAQ,EAAE,KAAK;iBAChB;gBACD,eAAe,EAAE,SAAS;gBAC1B,aAAa,EAAE,SAAS;gBACxB,QAAQ,EAAE;oBACR,KAAK,EAAE,EAAE;oBACT,WAAW,EAAE,EAAE;oBACf,SAAS,EAAE,EAAE;oBACb,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,EAAE;iBACb;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAEzC,gDAAgD;IAChD,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,KAAK,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7E,MAAM,WAAW,GAAG,CAAC,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACxD,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,MAAM,MAAM,GAAe;gBACzB,GAAG;gBACH,KAAK,EAAE,WAAW,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK;gBAC1C,OAAO,EAAE,WAAW,CAAC,eAAe;gBACpC,UAAU,EAAE;oBACV,MAAM,EAAE,YAAY;oBACpB,UAAU,EAAE,qBAAqB;oBACjC,QAAQ,EAAE,KAAK;iBAChB;gBACD,eAAe,EAAE,SAAS;gBAC1B,aAAa,EAAE,SAAS;gBACxB,QAAQ;aACT,CAAC;YACF,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,qBAAqB,CAAC,CAAC;YAC3E,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACtD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,OAAO,CAAC,uBAAuB,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,MAAM,GAAG,yBAAyB,EAAE,CAAC;YAC/C,MAAM,UAAU,GAAG,mBAAmB,CAAC,UAAU,CAAC;gBAChD,CAAC,CAAC,4BAA4B;gBAC9B,CAAC,CAAC,oBAAoB,CAAC;YACzB,MAAM,MAAM,GAAe;gBACzB,GAAG;gBACH,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,OAAO;gBACP,UAAU,EAAE;oBACV,MAAM,EAAE,gBAAgB;oBACxB,UAAU;oBACV,QAAQ,EAAE,KAAK;iBAChB;gBACD,eAAe,EAAE,UAAU;gBAC3B,aAAa,EAAE,SAAS;gBACxB,QAAQ;aACT,CAAC;YACF,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,gBAAgB,EAAE,UAAU,CAAC,CAAC;YACpE,mBAAmB,CAAC,GAAG,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,OAAO,GAAG,mBAAmB,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;QACvD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,gBAAgB,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YACxD,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;YACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACjC,MAAM,gBAAgB,GAAG,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YACzD,MAAM,WAAW,GAAG,CAAC,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;YAChE,MAAM,OAAO,GACX,WAAW,EAAE,eAAe,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAiB,MAAM;gBACjC,CAAC,CAAC,uBAAuB;gBACzB,CAAC,CAAC,cAAc,CAAC;YACnB,MAAM,UAAU,GAAG,MAAM;gBACvB,CAAC,CAAC,sBAAsB;gBACxB,CAAC,CAAC,uBAAuB,CAAC;YAC5B,MAAM,MAAM,GAAe;gBACzB,GAAG;gBACH,KAAK,EAAE,WAAW,EAAE,KAAK,IAAI,gBAAgB,CAAC,KAAK;gBACnD,OAAO;gBACP,UAAU,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAClD,eAAe,EAAE,CAAC,CAAC,qBAAqB,CAAC,YAAY,CAAC;gBACtD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;gBAC5C,QAAQ,EAAE,gBAAgB;aAC3B,CAAC;YACF,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC1D,mBAAmB,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACxD,iBAAiB,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;QACjD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAClD,OAAO,yBAAyB,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;AAC7D,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -4,43 +4,44 @@ import { dirname, join } from "node:path";
|
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6
6
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
7
|
+
import { closeSharedBrowserPool } from "./extraction/router.js";
|
|
7
8
|
import { registerTools } from "./tools/index.js";
|
|
8
9
|
import { formatBanner, loadConfig } from "./utils/config.js";
|
|
9
10
|
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
10
11
|
const pkg = JSON.parse(readFileSync(join(currentDir, "..", "package.json"), "utf8"));
|
|
11
12
|
function printHelp() {
|
|
12
|
-
const help = `${pkg.name} v${pkg.version} — BetterWebSearch MCP
|
|
13
|
-
A self-learning web-access engine for AI agents. Keyless by default via DuckDuckGo.
|
|
14
|
-
|
|
15
|
-
Usage:
|
|
16
|
-
better-web-search-mcp [--help] [--version]
|
|
17
|
-
|
|
18
|
-
Tools (6):
|
|
19
|
-
web_search Fast search (keyless by default, Brave/Tavily if keys present)
|
|
20
|
-
web_research Deep multi-query research (rewrites question, ranks, extracts)
|
|
21
|
-
deep_search Alias for web_research
|
|
22
|
-
web_extract Clean article extraction (HTTP → hydration → browser)
|
|
23
|
-
web_find Site-restricted search (site:example.com query)
|
|
24
|
-
web_news Recent news with timeline & source diversity
|
|
25
|
-
|
|
26
|
-
Env (all optional — keyless works out of the box):
|
|
27
|
-
BRAVE_API_KEY / BETTER_WEB_SEARCH_BRAVE_API_KEY Brave Search API key
|
|
28
|
-
TAVILY_API_KEY / BETTER_WEB_SEARCH_TAVILY_API_KEY Tavily API key
|
|
29
|
-
BETTER_WEB_SEARCH_DISABLE_CACHE=true Disable SQLite cache
|
|
30
|
-
BETTER_WEB_SEARCH_CACHE_PATH=./data/cache.db Custom cache path
|
|
31
|
-
BETTER_WEB_SEARCH_DISABLE_BROWSER=true Disable Playwright fallback
|
|
32
|
-
|
|
33
|
-
Examples:
|
|
34
|
-
npx better-web-search-mcp # keyless (DuckDuckGo)
|
|
35
|
-
BRAVE_API_KEY=... npx better-web-search-mcp # with Brave
|
|
36
|
-
better-web-search-mcp --help
|
|
37
|
-
better-web-search-mcp --version
|
|
38
|
-
|
|
39
|
-
MCP config (Claude Desktop / Cursor / VS Code Copilot):
|
|
40
|
-
{"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"]}}}
|
|
41
|
-
With Brave: {"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"],"env":{"BRAVE_API_KEY":"..."}}}}
|
|
42
|
-
|
|
43
|
-
See README.md and mcp.json for full client examples.
|
|
13
|
+
const help = `${pkg.name} v${pkg.version} — BetterWebSearch MCP
|
|
14
|
+
A self-learning web-access engine for AI agents. Keyless by default via DuckDuckGo.
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
better-web-search-mcp [--help] [--version]
|
|
18
|
+
|
|
19
|
+
Tools (6):
|
|
20
|
+
web_search Fast search (keyless by default, Brave/Tavily if keys present)
|
|
21
|
+
web_research Deep multi-query research (rewrites question, ranks, extracts)
|
|
22
|
+
deep_search Alias for web_research
|
|
23
|
+
web_extract Clean article extraction (HTTP → hydration → browser)
|
|
24
|
+
web_find Site-restricted search (site:example.com query)
|
|
25
|
+
web_news Recent news with timeline & source diversity
|
|
26
|
+
|
|
27
|
+
Env (all optional — keyless works out of the box):
|
|
28
|
+
BRAVE_API_KEY / BETTER_WEB_SEARCH_BRAVE_API_KEY Brave Search API key
|
|
29
|
+
TAVILY_API_KEY / BETTER_WEB_SEARCH_TAVILY_API_KEY Tavily API key
|
|
30
|
+
BETTER_WEB_SEARCH_DISABLE_CACHE=true Disable SQLite cache
|
|
31
|
+
BETTER_WEB_SEARCH_CACHE_PATH=./data/cache.db Custom cache path
|
|
32
|
+
BETTER_WEB_SEARCH_DISABLE_BROWSER=true Disable Playwright fallback
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
npx better-web-search-mcp # keyless (DuckDuckGo)
|
|
36
|
+
BRAVE_API_KEY=... npx better-web-search-mcp # with Brave
|
|
37
|
+
better-web-search-mcp --help
|
|
38
|
+
better-web-search-mcp --version
|
|
39
|
+
|
|
40
|
+
MCP config (Claude Desktop / Cursor / VS Code Copilot):
|
|
41
|
+
{"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"]}}}
|
|
42
|
+
With Brave: {"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"],"env":{"BRAVE_API_KEY":"..."}}}}
|
|
43
|
+
|
|
44
|
+
See README.md and mcp.json for full client examples.
|
|
44
45
|
`;
|
|
45
46
|
process.stdout.write(help);
|
|
46
47
|
}
|
|
@@ -58,9 +59,31 @@ const server = new McpServer({
|
|
|
58
59
|
version: pkg.version,
|
|
59
60
|
});
|
|
60
61
|
registerTools(server);
|
|
62
|
+
/**
|
|
63
|
+
* Shut down cleanly on a signal.
|
|
64
|
+
*
|
|
65
|
+
* The router launches chromium lazily for its Level 3 tier, so an abrupt
|
|
66
|
+
* exit would orphan that browser process. Guarded so a second signal during
|
|
67
|
+
* teardown does not re-enter.
|
|
68
|
+
*/
|
|
69
|
+
let shuttingDown = false;
|
|
70
|
+
function installShutdownHandlers() {
|
|
71
|
+
for (const signal of ["SIGINT", "SIGTERM"]) {
|
|
72
|
+
process.on(signal, () => {
|
|
73
|
+
if (shuttingDown) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
shuttingDown = true;
|
|
77
|
+
void closeSharedBrowserPool()
|
|
78
|
+
.catch(() => { })
|
|
79
|
+
.finally(() => process.exit(0));
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
61
83
|
async function main() {
|
|
62
84
|
const cfg = loadConfig();
|
|
63
85
|
console.error(formatBanner(pkg, cfg));
|
|
86
|
+
installShutdownHandlers();
|
|
64
87
|
const transport = new StdioServerTransport();
|
|
65
88
|
await server.connect(transport);
|
|
66
89
|
}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE7D,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CACxB,CAAC;AAEvC,SAAS,SAAS;IAChB,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCzC,CAAC;IACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IACnD,SAAS,EAAE,CAAC;IACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IACtD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;IACd,OAAO,EAAE,GAAG,CAAC,OAAO;CACrB,CAAC,CAAC;AAEH,aAAa,CAAC,MAAM,CAAC,CAAC;AAEtB,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;IACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE7D,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CACxB,CAAC;AAEvC,SAAS,SAAS;IAChB,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCzC,CAAC;IACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IACnD,SAAS,EAAE,CAAC;IACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IACtD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;IACd,OAAO,EAAE,GAAG,CAAC,OAAO;CACrB,CAAC,CAAC;AAEH,aAAa,CAAC,MAAM,CAAC,CAAC;AAEtB;;;;;;GAMG;AACH,IAAI,YAAY,GAAG,KAAK,CAAC;AACzB,SAAS,uBAAuB;IAC9B,KAAK,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAU,EAAE,CAAC;QACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO;YACT,CAAC;YACD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,sBAAsB,EAAE;iBAC1B,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;iBACf,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACtC,uBAAuB,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;IACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/providers/brave.js
CHANGED
|
@@ -78,7 +78,7 @@ export class BraveProvider {
|
|
|
78
78
|
Accept: "application/json",
|
|
79
79
|
},
|
|
80
80
|
signal: controller.signal,
|
|
81
|
-
}), { retryOn: [429, 503] });
|
|
81
|
+
}), { retryOn: [429, 503], retryNetworkErrors: true });
|
|
82
82
|
if (response.status === 401) {
|
|
83
83
|
console.warn(`[brave] API returned ${response.status}; returning no results.`);
|
|
84
84
|
return [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/providers/brave.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAG9C,MAAM,cAAc,GAAG,gDAAgD,CAAC;AACxE,MAAM,UAAU,GAAG,KAAK,CAAC;AAEzB;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAY;IACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,2EAA2E;IAC3E,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IACtB,MAAM,GAAG,GAAG,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9D,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;AACpC,CAAC;AAgBD,kEAAkE;AAClE,SAAS,YAAY,CAAC,MAAsB;IAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,OAAO,CAAC;IAExB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,IAAmB;QAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,SAAS,GACb,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAC9B,GAAG,EAAE,CACH,KAAK,CAAC,GAAG,cAAc,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE;gBAC9C,OAAO,EAAE;oBACP,sBAAsB,EAAE,MAAM;oBAC9B,MAAM,EAAE,kBAAkB;iBAC3B;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,EACJ,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/providers/brave.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAG9C,MAAM,cAAc,GAAG,gDAAgD,CAAC;AACxE,MAAM,UAAU,GAAG,KAAK,CAAC;AAEzB;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAY;IACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,2EAA2E;IAC3E,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;IACtB,MAAM,GAAG,GAAG,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9D,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;AACpC,CAAC;AAgBD,kEAAkE;AAClE,SAAS,YAAY,CAAC,MAAsB;IAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,OAAO,CAAC;IAExB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,IAAmB;QAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,SAAS,GACb,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAC9B,GAAG,EAAE,CACH,KAAK,CAAC,GAAG,cAAc,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE;gBAC9C,OAAO,EAAE;oBACP,sBAAsB,EAAE,MAAM;oBAC9B,MAAM,EAAE,kBAAkB;iBAC3B;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,EACJ,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAClD,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CACV,wBAAwB,QAAQ,CAAC,MAAM,yBAAyB,CACjE,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CACV,wBAAwB,QAAQ,CAAC,MAAM,yBAAyB,CACjE,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;YACxC,OAAO,OAAO;iBACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC;iBAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;gBACpB,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE;gBAChB,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;gBACxB,SAAS,EAAE,CAAC,CAAC,GAAG;gBAChB,MAAM,EAAE,IAAI,CAAC,IAAI;aAClB,CAAC,CAAC,CAAC;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CACV,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAC3G,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -5,15 +5,21 @@ import type { SearchOptions, SearchProvider, SearchResult } from "./types.js";
|
|
|
5
5
|
* - Brave: enabled when `BRAVE_API_KEY` (or BETTER_WEB_SEARCH_BRAVE_API_KEY) is set.
|
|
6
6
|
* - Tavily: enabled when `TAVILY_API_KEY` (or BETTER_WEB_SEARCH_TAVILY_API_KEY) is set.
|
|
7
7
|
* - DuckDuckGo: always enabled (keyless free fallback) — zero-config.
|
|
8
|
-
*
|
|
8
|
+
*
|
|
9
|
+
* SerpApi is deliberately absent: `SerpApiProvider` is still a stub, so
|
|
10
|
+
* including it only added a no-op call to every aggregation.
|
|
9
11
|
*/
|
|
10
12
|
export declare function enabledProviders(): SearchProvider[];
|
|
13
|
+
/** Wall-clock budget for a single provider, in milliseconds. */
|
|
14
|
+
export declare const PROVIDER_TIMEOUT_MS = 10000;
|
|
11
15
|
/**
|
|
12
16
|
* Aggregate search across all enabled providers.
|
|
13
17
|
*
|
|
14
|
-
* Uses `Promise.allSettled` so one failing provider never aborts the others
|
|
15
|
-
*
|
|
16
|
-
*
|
|
18
|
+
* Uses `Promise.allSettled` so one failing provider never aborts the others,
|
|
19
|
+
* and caps each provider at {@link PROVIDER_TIMEOUT_MS} so one hung provider
|
|
20
|
+
* cannot gate the whole search — the others' results are still returned.
|
|
21
|
+
* Rejected and timed-out providers are logged as warnings and skipped;
|
|
22
|
+
* fulfilled results are flattened into a single array.
|
|
17
23
|
*/
|
|
18
24
|
export declare function aggregateSearch(query: string, opts?: SearchOptions): Promise<SearchResult[]>;
|
|
19
25
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG9E;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,IAAI,cAAc,EAAE,CAanD;AAED,gEAAgE;AAChE,eAAO,MAAM,mBAAmB,QAAS,CAAC;AA2B1C;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,EAAE,CAAC,CAyBzB"}
|