better-web-search-mcp 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -112
- package/dist/index.js +32 -32
- package/package.json +2 -2
- package/dist/extraction/alternative.test.d.ts +0 -2
- package/dist/extraction/alternative.test.d.ts.map +0 -1
- package/dist/extraction/alternative.test.js +0 -145
- package/dist/extraction/alternative.test.js.map +0 -1
- package/dist/extraction/apiIntercept.test.d.ts +0 -2
- package/dist/extraction/apiIntercept.test.d.ts.map +0 -1
- package/dist/extraction/apiIntercept.test.js +0 -144
- package/dist/extraction/apiIntercept.test.js.map +0 -1
- package/dist/extraction/browser.test.d.ts +0 -2
- package/dist/extraction/browser.test.d.ts.map +0 -1
- package/dist/extraction/browser.test.js +0 -137
- package/dist/extraction/browser.test.js.map +0 -1
- package/dist/extraction/fetch.test.d.ts +0 -2
- package/dist/extraction/fetch.test.d.ts.map +0 -1
- package/dist/extraction/fetch.test.js +0 -85
- package/dist/extraction/fetch.test.js.map +0 -1
- package/dist/extraction/fusion.test.d.ts +0 -2
- package/dist/extraction/fusion.test.d.ts.map +0 -1
- package/dist/extraction/fusion.test.js +0 -91
- package/dist/extraction/fusion.test.js.map +0 -1
- package/dist/extraction/metadata.test.d.ts +0 -2
- package/dist/extraction/metadata.test.d.ts.map +0 -1
- package/dist/extraction/metadata.test.js +0 -45
- package/dist/extraction/metadata.test.js.map +0 -1
- package/dist/extraction/readability.test.d.ts +0 -2
- package/dist/extraction/readability.test.d.ts.map +0 -1
- package/dist/extraction/readability.test.js +0 -51
- package/dist/extraction/readability.test.js.map +0 -1
- package/dist/extraction/router.browser.test.d.ts +0 -2
- package/dist/extraction/router.browser.test.d.ts.map +0 -1
- package/dist/extraction/router.browser.test.js +0 -151
- package/dist/extraction/router.browser.test.js.map +0 -1
- package/dist/extraction/router.test.d.ts +0 -2
- package/dist/extraction/router.test.d.ts.map +0 -1
- package/dist/extraction/router.test.js +0 -238
- package/dist/extraction/router.test.js.map +0 -1
- package/dist/extraction/structured.test.d.ts +0 -2
- package/dist/extraction/structured.test.d.ts.map +0 -1
- package/dist/extraction/structured.test.js +0 -85
- package/dist/extraction/structured.test.js.map +0 -1
- package/dist/providers/brave.test.d.ts +0 -2
- package/dist/providers/brave.test.d.ts.map +0 -1
- package/dist/providers/brave.test.js +0 -150
- package/dist/providers/brave.test.js.map +0 -1
- package/dist/providers/duckduckgo.test.d.ts +0 -2
- package/dist/providers/duckduckgo.test.d.ts.map +0 -1
- package/dist/providers/duckduckgo.test.js +0 -102
- package/dist/providers/duckduckgo.test.js.map +0 -1
- package/dist/providers/index.test.d.ts +0 -2
- package/dist/providers/index.test.d.ts.map +0 -1
- package/dist/providers/index.test.js +0 -124
- package/dist/providers/index.test.js.map +0 -1
- package/dist/providers/tavily.test.d.ts +0 -2
- package/dist/providers/tavily.test.d.ts.map +0 -1
- package/dist/providers/tavily.test.js +0 -123
- package/dist/providers/tavily.test.js.map +0 -1
- package/dist/ranking/deduplicate.test.d.ts +0 -2
- package/dist/ranking/deduplicate.test.d.ts.map +0 -1
- package/dist/ranking/deduplicate.test.js +0 -76
- package/dist/ranking/deduplicate.test.js.map +0 -1
- package/dist/ranking/domainScore.test.d.ts +0 -2
- package/dist/ranking/domainScore.test.d.ts.map +0 -1
- package/dist/ranking/domainScore.test.js +0 -37
- package/dist/ranking/domainScore.test.js.map +0 -1
- package/dist/ranking/rerank.test.d.ts +0 -2
- package/dist/ranking/rerank.test.d.ts.map +0 -1
- package/dist/ranking/rerank.test.js +0 -71
- package/dist/ranking/rerank.test.js.map +0 -1
- package/dist/scaffold.test.d.ts +0 -2
- package/dist/scaffold.test.d.ts.map +0 -1
- package/dist/scaffold.test.js +0 -7
- package/dist/scaffold.test.js.map +0 -1
- package/dist/tools/extract.test.d.ts +0 -2
- package/dist/tools/extract.test.d.ts.map +0 -1
- package/dist/tools/extract.test.js +0 -140
- package/dist/tools/extract.test.js.map +0 -1
- package/dist/tools/research.test.d.ts +0 -2
- package/dist/tools/research.test.d.ts.map +0 -1
- package/dist/tools/research.test.js +0 -181
- package/dist/tools/research.test.js.map +0 -1
- package/dist/tools/search.test.d.ts +0 -2
- package/dist/tools/search.test.d.ts.map +0 -1
- package/dist/tools/search.test.js +0 -101
- package/dist/tools/search.test.js.map +0 -1
- package/dist/utils/cache.test.d.ts +0 -2
- package/dist/utils/cache.test.d.ts.map +0 -1
- package/dist/utils/cache.test.js +0 -172
- package/dist/utils/cache.test.js.map +0 -1
- package/dist/utils/config.test.d.ts +0 -2
- package/dist/utils/config.test.d.ts.map +0 -1
- package/dist/utils/config.test.js +0 -52
- package/dist/utils/config.test.js.map +0 -1
- package/dist/utils/domainProfile.test.d.ts +0 -2
- package/dist/utils/domainProfile.test.d.ts.map +0 -1
- package/dist/utils/domainProfile.test.js +0 -74
- package/dist/utils/domainProfile.test.js.map +0 -1
- package/dist/utils/queries.test.d.ts +0 -2
- package/dist/utils/queries.test.d.ts.map +0 -1
- package/dist/utils/queries.test.js +0 -36
- package/dist/utils/queries.test.js.map +0 -1
- package/dist/utils/retry.test.d.ts +0 -2
- package/dist/utils/retry.test.d.ts.map +0 -1
- package/dist/utils/retry.test.js +0 -153
- package/dist/utils/retry.test.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,112 +1,112 @@
|
|
|
1
|
-
# BetterWebSearch MCP
|
|
2
|
-
|
|
3
|
-
> Keyless-first web search for AI agents. DuckDuckGo works out of the box. No API keys.
|
|
4
|
-
|
|
5
|
-
[](https://www.npmjs.com/package/better-web-search-mcp)
|
|
6
|
-
[](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions)
|
|
7
|
-
[](https://phantompixeldev.github.io/BetterWebSearch-MCP/)
|
|
8
|
-
[](LICENSE)
|
|
9
|
-
[](https://smithery.ai/server/@PhantomPixelDev/better-web-search-mcp)
|
|
10
|
-
|
|
11
|
-
<p align="center">
|
|
12
|
-
<img src="./images/banner.jpg" alt="BetterWebSearch MCP — Smarter Web Search · Better Data Extraction · Flexible & Extensible · Open Source" width="100%">
|
|
13
|
-
</p>
|
|
14
|
-
|
|
15
|
-
## What is this?
|
|
16
|
-
|
|
17
|
-
BetterWebSearch MCP is a [Model Context Protocol](https://modelcontextprotocol.io) server that gives AI assistants genuinely better web research. It works without API keys by default (DuckDuckGo), and optionally adds Brave or Tavily for richer ranking and recency.
|
|
18
|
-
|
|
19
|
-
Built for anyone who wants their AI tools to find things on the web reliably, not just when a paid API key is configured.
|
|
20
|
-
|
|
21
|
-
## Quick start (30 seconds)
|
|
22
|
-
|
|
23
|
-
Add this to your MCP client config (Claude Desktop, Cursor, VS Code Copilot, OpenCode, or any MCP client):
|
|
24
|
-
|
|
25
|
-
```json
|
|
26
|
-
{
|
|
27
|
-
"mcpServers": {
|
|
28
|
-
"better-web-search-mcp": {
|
|
29
|
-
"command": "npx",
|
|
30
|
-
"args": ["-y", "better-web-search-mcp"]
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
That's it. Every search tool works immediately. No `.env`, no API keys, no setup.
|
|
37
|
-
|
|
38
|
-
Or run it directly:
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
npx -y better-web-search-mcp --help
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
## Tools
|
|
45
|
-
|
|
46
|
-
| Tool | What it does |
|
|
47
|
-
|---|---|
|
|
48
|
-
| `web_search` | Aggregated search across providers, deduplicated and reranked |
|
|
49
|
-
| `web_research` | Deep research. Rewrites your question, searches in parallel, extracts top results with citations |
|
|
50
|
-
| `deep_search` | Alias for `web_research` |
|
|
51
|
-
| `web_extract` | Extracts clean content from any URL, using the best method available |
|
|
52
|
-
| `web_find` | Search scoped to a single website |
|
|
53
|
-
| `web_news` | Recent news with timeline and diversity filtering |
|
|
54
|
-
|
|
55
|
-
## How search is better
|
|
56
|
-
|
|
57
|
-
Standard MCP search tools hit one API and return raw results. BetterWebSearch takes a different approach.
|
|
58
|
-
|
|
59
|
-
**Query expansion.** Your single question gets rewritten into 4 to 6 variants, searched in parallel. "Unlimited mobile internet Germany" also searches for "unbegrenztes Datenvolumen Deutschland" and similar phrasings. More coverage without you doing extra work.
|
|
60
|
-
|
|
61
|
-
**Three-tier extraction.** When it fetches a page, it doesn't just grab the HTML and hope for the best:
|
|
62
|
-
|
|
63
|
-
```
|
|
64
|
-
Tier 1: Fast HTTP fetch (< 1s)
|
|
65
|
-
↓ if not enough content
|
|
66
|
-
Tier 2: Structured data (JSON-LD, __NEXT_DATA__, __NUXT__) (1-3s)
|
|
67
|
-
↓ if still not enough
|
|
68
|
-
Tier 3: Playwright browser (full render + API interception) (3-10s)
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
Only escalates when needed. Results carry confidence scores so you know what you're getting.
|
|
72
|
-
|
|
73
|
-
**Self-learning cache.** The first visit to a domain takes the full path. The second visit skips straight to what worked. Domain profiles and API patterns are remembered.
|
|
74
|
-
|
|
75
|
-
## Project structure
|
|
76
|
-
|
|
77
|
-
```
|
|
78
|
-
src/
|
|
79
|
-
providers/ Search backends (DuckDuckGo, Brave, Tavily; SerpApi stubbed)
|
|
80
|
-
extraction/ 3-tier content pipeline (fetch, structured, browser)
|
|
81
|
-
ranking/ Deduplication, domain scoring, reranking
|
|
82
|
-
tools/ MCP tool definitions (search, research, extract, find, news)
|
|
83
|
-
utils/ Config, caching, retry, query rewriting
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
## Configuration
|
|
87
|
-
|
|
88
|
-
All configuration is optional. DuckDuckGo works with zero setup.
|
|
89
|
-
|
|
90
|
-
```bash
|
|
91
|
-
# Optional: add more providers
|
|
92
|
-
BRAVE_API_KEY=your_key # Better ranking and recency
|
|
93
|
-
TAVILY_API_KEY=your_key # Alternative provider
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
See the [full configuration reference](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) for all options including cache paths, browser settings, and environment variable names.
|
|
97
|
-
|
|
98
|
-
## Documentation
|
|
99
|
-
|
|
100
|
-
Full guides, API reference, and architecture details live on the docs site:
|
|
101
|
-
|
|
102
|
-
**[phantompixeldev.github.io/BetterWebSearch-MCP](https://phantompixeldev.github.io/BetterWebSearch-MCP/)**
|
|
103
|
-
|
|
104
|
-
- [Installation](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/installation/) - npx, npm, or from source
|
|
105
|
-
- [Quick Start](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/quickstart/) - Get running in your first AI client
|
|
106
|
-
- [Configuration](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) - All environment variables and options
|
|
107
|
-
- [Contributing](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/contributing/) - Development setup and PR process
|
|
108
|
-
- [Changelog](CHANGELOG.md) - Release history and version notes
|
|
109
|
-
|
|
110
|
-
## License
|
|
111
|
-
|
|
112
|
-
MIT
|
|
1
|
+
# BetterWebSearch MCP
|
|
2
|
+
|
|
3
|
+
> Keyless-first web search for AI agents. DuckDuckGo works out of the box. No API keys.
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/better-web-search-mcp)
|
|
6
|
+
[](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions)
|
|
7
|
+
[](https://phantompixeldev.github.io/BetterWebSearch-MCP/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
[](https://smithery.ai/server/@PhantomPixelDev/better-web-search-mcp)
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<img src="./images/banner.jpg" alt="BetterWebSearch MCP — Smarter Web Search · Better Data Extraction · Flexible & Extensible · Open Source" width="100%">
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
## What is this?
|
|
16
|
+
|
|
17
|
+
BetterWebSearch MCP is a [Model Context Protocol](https://modelcontextprotocol.io) server that gives AI assistants genuinely better web research. It works without API keys by default (DuckDuckGo), and optionally adds Brave or Tavily for richer ranking and recency.
|
|
18
|
+
|
|
19
|
+
Built for anyone who wants their AI tools to find things on the web reliably, not just when a paid API key is configured.
|
|
20
|
+
|
|
21
|
+
## Quick start (30 seconds)
|
|
22
|
+
|
|
23
|
+
Add this to your MCP client config (Claude Desktop, Cursor, VS Code Copilot, OpenCode, or any MCP client):
|
|
24
|
+
|
|
25
|
+
```json
|
|
26
|
+
{
|
|
27
|
+
"mcpServers": {
|
|
28
|
+
"better-web-search-mcp": {
|
|
29
|
+
"command": "npx",
|
|
30
|
+
"args": ["-y", "better-web-search-mcp"]
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
That's it. Every search tool works immediately. No `.env`, no API keys, no setup.
|
|
37
|
+
|
|
38
|
+
Or run it directly:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx -y better-web-search-mcp --help
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Tools
|
|
45
|
+
|
|
46
|
+
| Tool | What it does |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `web_search` | Aggregated search across providers, deduplicated and reranked |
|
|
49
|
+
| `web_research` | Deep research. Rewrites your question, searches in parallel, extracts top results with citations |
|
|
50
|
+
| `deep_search` | Alias for `web_research` |
|
|
51
|
+
| `web_extract` | Extracts clean content from any URL, using the best method available |
|
|
52
|
+
| `web_find` | Search scoped to a single website |
|
|
53
|
+
| `web_news` | Recent news with timeline and diversity filtering |
|
|
54
|
+
|
|
55
|
+
## How search is better
|
|
56
|
+
|
|
57
|
+
Standard MCP search tools hit one API and return raw results. BetterWebSearch takes a different approach.
|
|
58
|
+
|
|
59
|
+
**Query expansion.** Your single question gets rewritten into 4 to 6 variants, searched in parallel. "Unlimited mobile internet Germany" also searches for "unbegrenztes Datenvolumen Deutschland" and similar phrasings. More coverage without you doing extra work.
|
|
60
|
+
|
|
61
|
+
**Three-tier extraction.** When it fetches a page, it doesn't just grab the HTML and hope for the best:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
Tier 1: Fast HTTP fetch (< 1s)
|
|
65
|
+
↓ if not enough content
|
|
66
|
+
Tier 2: Structured data (JSON-LD, __NEXT_DATA__, __NUXT__) (1-3s)
|
|
67
|
+
↓ if still not enough
|
|
68
|
+
Tier 3: Playwright browser (full render + API interception) (3-10s)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Only escalates when needed. Results carry confidence scores so you know what you're getting.
|
|
72
|
+
|
|
73
|
+
**Self-learning cache.** The first visit to a domain takes the full path. The second visit skips straight to what worked. Domain profiles and API patterns are remembered.
|
|
74
|
+
|
|
75
|
+
## Project structure
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
src/
|
|
79
|
+
providers/ Search backends (DuckDuckGo, Brave, Tavily; SerpApi stubbed)
|
|
80
|
+
extraction/ 3-tier content pipeline (fetch, structured, browser)
|
|
81
|
+
ranking/ Deduplication, domain scoring, reranking
|
|
82
|
+
tools/ MCP tool definitions (search, research, extract, find, news)
|
|
83
|
+
utils/ Config, caching, retry, query rewriting
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Configuration
|
|
87
|
+
|
|
88
|
+
All configuration is optional. DuckDuckGo works with zero setup.
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
# Optional: add more providers
|
|
92
|
+
BRAVE_API_KEY=your_key # Better ranking and recency
|
|
93
|
+
TAVILY_API_KEY=your_key # Alternative provider
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
See the [full configuration reference](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) for all options including cache paths, browser settings, and environment variable names.
|
|
97
|
+
|
|
98
|
+
## Documentation
|
|
99
|
+
|
|
100
|
+
Full guides, API reference, and architecture details live on the docs site:
|
|
101
|
+
|
|
102
|
+
**[phantompixeldev.github.io/BetterWebSearch-MCP](https://phantompixeldev.github.io/BetterWebSearch-MCP/)**
|
|
103
|
+
|
|
104
|
+
- [Installation](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/installation/) - npx, npm, or from source
|
|
105
|
+
- [Quick Start](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/quickstart/) - Get running in your first AI client
|
|
106
|
+
- [Configuration](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) - All environment variables and options
|
|
107
|
+
- [Contributing](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/contributing/) - Development setup and PR process
|
|
108
|
+
- [Changelog](CHANGELOG.md) - Release history and version notes
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT
|
package/dist/index.js
CHANGED
|
@@ -10,38 +10,38 @@ import { formatBanner, loadConfig } from "./utils/config.js";
|
|
|
10
10
|
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
11
11
|
const pkg = JSON.parse(readFileSync(join(currentDir, "..", "package.json"), "utf8"));
|
|
12
12
|
function printHelp() {
|
|
13
|
-
const help = `${pkg.name} v${pkg.version} — BetterWebSearch MCP
|
|
14
|
-
A self-learning web-access engine for AI agents. Keyless by default via DuckDuckGo.
|
|
15
|
-
|
|
16
|
-
Usage:
|
|
17
|
-
better-web-search-mcp [--help] [--version]
|
|
18
|
-
|
|
19
|
-
Tools (6):
|
|
20
|
-
web_search Fast search (keyless by default, Brave/Tavily if keys present)
|
|
21
|
-
web_research Deep multi-query research (rewrites question, ranks, extracts)
|
|
22
|
-
deep_search Alias for web_research
|
|
23
|
-
web_extract Clean article extraction (HTTP → hydration → browser)
|
|
24
|
-
web_find Site-restricted search (site:example.com query)
|
|
25
|
-
web_news Recent news with timeline & source diversity
|
|
26
|
-
|
|
27
|
-
Env (all optional — keyless works out of the box):
|
|
28
|
-
BRAVE_API_KEY / BETTER_WEB_SEARCH_BRAVE_API_KEY Brave Search API key
|
|
29
|
-
TAVILY_API_KEY / BETTER_WEB_SEARCH_TAVILY_API_KEY Tavily API key
|
|
30
|
-
BETTER_WEB_SEARCH_DISABLE_CACHE=true Disable SQLite cache
|
|
31
|
-
BETTER_WEB_SEARCH_CACHE_PATH=./data/cache.db Custom cache path
|
|
32
|
-
BETTER_WEB_SEARCH_DISABLE_BROWSER=true Disable Playwright fallback
|
|
33
|
-
|
|
34
|
-
Examples:
|
|
35
|
-
npx better-web-search-mcp # keyless (DuckDuckGo)
|
|
36
|
-
BRAVE_API_KEY=... npx better-web-search-mcp # with Brave
|
|
37
|
-
better-web-search-mcp --help
|
|
38
|
-
better-web-search-mcp --version
|
|
39
|
-
|
|
40
|
-
MCP config (Claude Desktop / Cursor / VS Code Copilot):
|
|
41
|
-
{"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"]}}}
|
|
42
|
-
With Brave: {"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"],"env":{"BRAVE_API_KEY":"..."}}}}
|
|
43
|
-
|
|
44
|
-
See README.md and mcp.json for full client examples.
|
|
13
|
+
const help = `${pkg.name} v${pkg.version} — BetterWebSearch MCP
|
|
14
|
+
A self-learning web-access engine for AI agents. Keyless by default via DuckDuckGo.
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
better-web-search-mcp [--help] [--version]
|
|
18
|
+
|
|
19
|
+
Tools (6):
|
|
20
|
+
web_search Fast search (keyless by default, Brave/Tavily if keys present)
|
|
21
|
+
web_research Deep multi-query research (rewrites question, ranks, extracts)
|
|
22
|
+
deep_search Alias for web_research
|
|
23
|
+
web_extract Clean article extraction (HTTP → hydration → browser)
|
|
24
|
+
web_find Site-restricted search (site:example.com query)
|
|
25
|
+
web_news Recent news with timeline & source diversity
|
|
26
|
+
|
|
27
|
+
Env (all optional — keyless works out of the box):
|
|
28
|
+
BRAVE_API_KEY / BETTER_WEB_SEARCH_BRAVE_API_KEY Brave Search API key
|
|
29
|
+
TAVILY_API_KEY / BETTER_WEB_SEARCH_TAVILY_API_KEY Tavily API key
|
|
30
|
+
BETTER_WEB_SEARCH_DISABLE_CACHE=true Disable SQLite cache
|
|
31
|
+
BETTER_WEB_SEARCH_CACHE_PATH=./data/cache.db Custom cache path
|
|
32
|
+
BETTER_WEB_SEARCH_DISABLE_BROWSER=true Disable Playwright fallback
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
npx better-web-search-mcp # keyless (DuckDuckGo)
|
|
36
|
+
BRAVE_API_KEY=... npx better-web-search-mcp # with Brave
|
|
37
|
+
better-web-search-mcp --help
|
|
38
|
+
better-web-search-mcp --version
|
|
39
|
+
|
|
40
|
+
MCP config (Claude Desktop / Cursor / VS Code Copilot):
|
|
41
|
+
{"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"]}}}
|
|
42
|
+
With Brave: {"mcpServers":{"better-web-search-mcp":{"command":"npx","args":["-y","better-web-search-mcp"],"env":{"BRAVE_API_KEY":"..."}}}}
|
|
43
|
+
|
|
44
|
+
See README.md and mcp.json for full client examples.
|
|
45
45
|
`;
|
|
46
46
|
process.stdout.write(help);
|
|
47
47
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "better-web-search-mcp",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "MCP server for genuinely better web research — keyless-first (DuckDuckGo), 3-tier extraction, deep research. Brave/Tavily optional.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"registry": "https://registry.npmjs.org/"
|
|
35
35
|
},
|
|
36
36
|
"scripts": {
|
|
37
|
-
"build": "tsc",
|
|
37
|
+
"build": "tsc -p tsconfig.build.json",
|
|
38
38
|
"dev": "tsx watch src/index.ts",
|
|
39
39
|
"test": "vitest run",
|
|
40
40
|
"lint": "tsc --noEmit",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"alternative.test.d.ts","sourceRoot":"","sources":["../../src/extraction/alternative.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import { buildAlternativeUrls, buildSearchQueries, findAlternativeSources, MAX_SNIPPETS, SNIPPET_CONFIDENCE, } from "./alternative.js";
|
|
3
|
-
import { Cache } from "../utils/cache.js";
|
|
4
|
-
/** A blocked (403) page fixture. */
|
|
5
|
-
function blockedPage(url) {
|
|
6
|
-
return {
|
|
7
|
-
html: "",
|
|
8
|
-
headers: { "content-type": "text/html" },
|
|
9
|
-
status: 403,
|
|
10
|
-
url,
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
|
-
/** A usable (200) page fixture. */
|
|
14
|
-
function okPage(url) {
|
|
15
|
-
return {
|
|
16
|
-
html: "<html><body><article><p>full article text</p></article></body></html>",
|
|
17
|
-
headers: { "content-type": "text/html" },
|
|
18
|
-
status: 200,
|
|
19
|
-
url,
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
/** A search result fixture. */
|
|
23
|
-
const SNIPPET = {
|
|
24
|
-
title: "Syndicated copy",
|
|
25
|
-
url: "https://syndicated.example/article",
|
|
26
|
-
snippet: "The full article text appears here.",
|
|
27
|
-
source: "duckduckgo",
|
|
28
|
-
};
|
|
29
|
-
describe("buildAlternativeUrls", () => {
|
|
30
|
-
it("returns original, ?output=1, /amp, and ?amp variants", () => {
|
|
31
|
-
expect(buildAlternativeUrls("https://example.com/article")).toEqual([
|
|
32
|
-
"https://example.com/article",
|
|
33
|
-
"https://example.com/article?output=1",
|
|
34
|
-
"https://example.com/article/amp",
|
|
35
|
-
"https://example.com/article?amp",
|
|
36
|
-
]);
|
|
37
|
-
});
|
|
38
|
-
it("uses & when the URL already has a query string", () => {
|
|
39
|
-
expect(buildAlternativeUrls("https://example.com/article?id=7")).toEqual([
|
|
40
|
-
"https://example.com/article?id=7",
|
|
41
|
-
"https://example.com/article?id=7&output=1",
|
|
42
|
-
"https://example.com/article/amp?id=7",
|
|
43
|
-
"https://example.com/article?id=7&",
|
|
44
|
-
]);
|
|
45
|
-
});
|
|
46
|
-
it("returns the original URL when it cannot be parsed", () => {
|
|
47
|
-
expect(buildAlternativeUrls("not a url")).toEqual(["not a url"]);
|
|
48
|
-
});
|
|
49
|
-
});
|
|
50
|
-
describe("buildSearchQueries", () => {
|
|
51
|
-
it("quotes the title and adds a site-restricted query", () => {
|
|
52
|
-
expect(buildSearchQueries("https://example.com/article", "My Article")).toEqual([
|
|
53
|
-
'"My Article"',
|
|
54
|
-
'site:example.com "My Article"',
|
|
55
|
-
"My Article example.com",
|
|
56
|
-
]);
|
|
57
|
-
});
|
|
58
|
-
it("falls back to a bare site query when the title is null", () => {
|
|
59
|
-
expect(buildSearchQueries("https://example.com/article", null)).toEqual([
|
|
60
|
-
"site:example.com",
|
|
61
|
-
]);
|
|
62
|
-
});
|
|
63
|
-
});
|
|
64
|
-
describe("findAlternativeSources", () => {
|
|
65
|
-
it("returns snippet evidence when every URL variant is blocked", async () => {
|
|
66
|
-
const fetchPage = vi
|
|
67
|
-
.fn()
|
|
68
|
-
.mockResolvedValue(blockedPage("https://example.com/article"));
|
|
69
|
-
const aggregateSearch = vi.fn().mockResolvedValue([SNIPPET]);
|
|
70
|
-
const deps = { fetchPage, aggregateSearch };
|
|
71
|
-
const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
|
|
72
|
-
expect(fetchPage).toHaveBeenCalledTimes(4);
|
|
73
|
-
expect(evidence).toHaveLength(1);
|
|
74
|
-
expect(evidence[0]).toEqual({
|
|
75
|
-
source: "search_snippet",
|
|
76
|
-
type: "search_snippet",
|
|
77
|
-
confidence: SNIPPET_CONFIDENCE,
|
|
78
|
-
url: "https://syndicated.example/article",
|
|
79
|
-
snippet: "The full article text appears here.",
|
|
80
|
-
});
|
|
81
|
-
});
|
|
82
|
-
it("returns page evidence when an AMP variant succeeds", async () => {
|
|
83
|
-
const fetchPage = vi.fn().mockImplementation(async (url) => {
|
|
84
|
-
return url.includes("/amp") ? okPage(url) : blockedPage(url);
|
|
85
|
-
});
|
|
86
|
-
const aggregateSearch = vi.fn();
|
|
87
|
-
const deps = { fetchPage, aggregateSearch };
|
|
88
|
-
const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
|
|
89
|
-
expect(aggregateSearch).not.toHaveBeenCalled();
|
|
90
|
-
expect(evidence).toHaveLength(1);
|
|
91
|
-
expect(evidence[0]).toMatchObject({
|
|
92
|
-
source: "page",
|
|
93
|
-
type: "page",
|
|
94
|
-
url: "https://example.com/article/amp",
|
|
95
|
-
});
|
|
96
|
-
});
|
|
97
|
-
it("uses a cached variant without fetching it", async () => {
|
|
98
|
-
const cache = new Cache({ memory: true });
|
|
99
|
-
cache.setPage("https://example.com/article/amp", "cached amp content", "http_fetch", 0.85);
|
|
100
|
-
const fetchPage = vi
|
|
101
|
-
.fn()
|
|
102
|
-
.mockResolvedValue(blockedPage("https://example.com/article"));
|
|
103
|
-
const aggregateSearch = vi.fn();
|
|
104
|
-
const deps = { fetchPage, aggregateSearch, cache };
|
|
105
|
-
const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
|
|
106
|
-
expect(fetchPage).toHaveBeenCalledTimes(2);
|
|
107
|
-
expect(aggregateSearch).not.toHaveBeenCalled();
|
|
108
|
-
expect(evidence).toHaveLength(1);
|
|
109
|
-
expect(evidence[0]).toMatchObject({
|
|
110
|
-
source: "page",
|
|
111
|
-
url: "https://example.com/article/amp",
|
|
112
|
-
confidence: 0.85,
|
|
113
|
-
});
|
|
114
|
-
});
|
|
115
|
-
it("generates a site fallback query when the title is null", async () => {
|
|
116
|
-
const fetchPage = vi
|
|
117
|
-
.fn()
|
|
118
|
-
.mockResolvedValue(blockedPage("https://example.com/article"));
|
|
119
|
-
const aggregateSearch = vi.fn().mockResolvedValue([SNIPPET]);
|
|
120
|
-
const deps = { fetchPage, aggregateSearch };
|
|
121
|
-
await findAlternativeSources("https://example.com/article", null, deps);
|
|
122
|
-
expect(aggregateSearch).toHaveBeenCalledWith("site:example.com", {
|
|
123
|
-
count: MAX_SNIPPETS,
|
|
124
|
-
});
|
|
125
|
-
});
|
|
126
|
-
it("returns [] when the search is empty instead of throwing", async () => {
|
|
127
|
-
const fetchPage = vi
|
|
128
|
-
.fn()
|
|
129
|
-
.mockResolvedValue(blockedPage("https://example.com/article"));
|
|
130
|
-
const aggregateSearch = vi.fn().mockResolvedValue([]);
|
|
131
|
-
const deps = { fetchPage, aggregateSearch };
|
|
132
|
-
const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
|
|
133
|
-
expect(evidence).toEqual([]);
|
|
134
|
-
});
|
|
135
|
-
it("returns [] when the search provider throws", async () => {
|
|
136
|
-
const fetchPage = vi
|
|
137
|
-
.fn()
|
|
138
|
-
.mockResolvedValue(blockedPage("https://example.com/article"));
|
|
139
|
-
const aggregateSearch = vi.fn().mockRejectedValue(new Error("provider down"));
|
|
140
|
-
const deps = { fetchPage, aggregateSearch };
|
|
141
|
-
const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
|
|
142
|
-
expect(evidence).toEqual([]);
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
//# sourceMappingURL=alternative.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"alternative.test.js","sourceRoot":"","sources":["../../src/extraction/alternative.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACZ,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,oCAAoC;AACpC,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO;QACL,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;QACxC,MAAM,EAAE,GAAG;QACX,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,mCAAmC;AACnC,SAAS,MAAM,CAAC,GAAW;IACzB,OAAO;QACL,IAAI,EAAE,uEAAuE;QAC7E,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;QACxC,MAAM,EAAE,GAAG;QACX,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,+BAA+B;AAC/B,MAAM,OAAO,GAAiB;IAC5B,KAAK,EAAE,iBAAiB;IACxB,GAAG,EAAE,oCAAoC;IACzC,OAAO,EAAE,qCAAqC;IAC9C,MAAM,EAAE,YAAY;CACrB,CAAC;AAEF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CAAC,oBAAoB,CAAC,6BAA6B,CAAC,CAAC,CAAC,OAAO,CAAC;YAClE,6BAA6B;YAC7B,sCAAsC;YACtC,iCAAiC;YACjC,iCAAiC;SAClC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,oBAAoB,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC;YACvE,kCAAkC;YAClC,2CAA2C;YAC3C,sCAAsC;YACtC,sCAAsC;SACvC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,kBAAkB,CAAC,6BAA6B,EAAE,YAAY,CAAC,CAChE,CAAC,OAAO,CAAC;YACR,cAAc;YACd,+BAA+B;YAC/B,wBAAwB;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,CAAC,kBAAkB,CAAC,6BAA6B,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;YACtE,kBAAkB;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,kBAAkB;YAC9B,GAAG,EAAE,oCAAoC;YACzC,OAAO,EAAE,qCAAqC;SAC/C,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YACjE,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,iCAAiC;SACvC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,KAAK,CAAC,OAAO,CACX,iCAAiC,EACjC,oBAAoB,EACpB,YAAY,EACZ,IAAI,CACL,CAAC;QACF,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QAEpE,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,iCAAiC;YACtC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,sBAAsB,CAAC,6BAA6B,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAExE,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,kBAAkB,EAAE;YAC/D,KAAK,EAAE,YAAY;SACpB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"apiIntercept.test.d.ts","sourceRoot":"","sources":["../../src/extraction/apiIntercept.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import { discoverApiPatterns, installApiCapture, } from "./apiIntercept.js";
|
|
3
|
-
/** Build a fake Playwright response object. */
|
|
4
|
-
function fakeResponse(url, method, contentType, data) {
|
|
5
|
-
return {
|
|
6
|
-
url: () => url,
|
|
7
|
-
request: () => ({ method: () => method }),
|
|
8
|
-
headers: () => ({ "content-type": contentType }),
|
|
9
|
-
json: async () => data,
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
/** Build a fake Playwright page that records its listeners. */
|
|
13
|
-
function fakePage() {
|
|
14
|
-
const listeners = [];
|
|
15
|
-
return {
|
|
16
|
-
listeners,
|
|
17
|
-
on: (_event, listener) => {
|
|
18
|
-
listeners.push(listener);
|
|
19
|
-
},
|
|
20
|
-
off: (_event, listener) => {
|
|
21
|
-
const idx = listeners.indexOf(listener);
|
|
22
|
-
if (idx !== -1) {
|
|
23
|
-
listeners.splice(idx, 1);
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
describe("installApiCapture", () => {
|
|
29
|
-
it("captures JSON responses with url/method/contentType/data", async () => {
|
|
30
|
-
const page = fakePage();
|
|
31
|
-
const handle = installApiCapture(page);
|
|
32
|
-
const listener = page.listeners[0];
|
|
33
|
-
await listener(fakeResponse("https://api.example.com/products/123", "GET", "application/json; charset=utf-8", { id: 123 }));
|
|
34
|
-
expect(handle.captured).toHaveLength(1);
|
|
35
|
-
expect(handle.captured[0]).toMatchObject({
|
|
36
|
-
url: "https://api.example.com/products/123",
|
|
37
|
-
method: "GET",
|
|
38
|
-
contentType: "application/json; charset=utf-8",
|
|
39
|
-
data: { id: 123 },
|
|
40
|
-
});
|
|
41
|
-
});
|
|
42
|
-
it("ignores non-JSON content types", async () => {
|
|
43
|
-
const page = fakePage();
|
|
44
|
-
const handle = installApiCapture(page);
|
|
45
|
-
const listener = page.listeners[0];
|
|
46
|
-
await listener(fakeResponse("https://example.com/style.css", "GET", "text/css", "body{}"));
|
|
47
|
-
expect(handle.captured).toHaveLength(0);
|
|
48
|
-
});
|
|
49
|
-
it("skips JSON responses whose body fails to parse", async () => {
|
|
50
|
-
const page = fakePage();
|
|
51
|
-
const handle = installApiCapture(page);
|
|
52
|
-
const listener = page.listeners[0];
|
|
53
|
-
const bad = fakeResponse("https://api.example.com/broken", "GET", "application/json", { id: 1 });
|
|
54
|
-
bad.json = async () => {
|
|
55
|
-
throw new Error("parse failed");
|
|
56
|
-
};
|
|
57
|
-
await listener(bad);
|
|
58
|
-
expect(handle.captured).toHaveLength(0);
|
|
59
|
-
});
|
|
60
|
-
it("remove() detaches the listener", async () => {
|
|
61
|
-
const page = fakePage();
|
|
62
|
-
const handle = installApiCapture(page);
|
|
63
|
-
expect(page.listeners).toHaveLength(1);
|
|
64
|
-
handle.remove();
|
|
65
|
-
expect(page.listeners).toHaveLength(0);
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
describe("discoverApiPatterns", () => {
|
|
69
|
-
it("normalizes numeric path segments to *", () => {
|
|
70
|
-
const rows = discoverApiPatterns("example.com", [
|
|
71
|
-
{
|
|
72
|
-
url: "https://example.com/api/products/123",
|
|
73
|
-
method: "GET",
|
|
74
|
-
contentType: "application/json",
|
|
75
|
-
data: {},
|
|
76
|
-
},
|
|
77
|
-
]);
|
|
78
|
-
expect(rows).toEqual([
|
|
79
|
-
{
|
|
80
|
-
domain: "example.com",
|
|
81
|
-
endpoint_pattern: "/api/products/*",
|
|
82
|
-
method: "GET",
|
|
83
|
-
content_type: "application/json",
|
|
84
|
-
},
|
|
85
|
-
]);
|
|
86
|
-
});
|
|
87
|
-
it("deduplicates identical method+pattern pairs", () => {
|
|
88
|
-
const rows = discoverApiPatterns("example.com", [
|
|
89
|
-
{
|
|
90
|
-
url: "https://example.com/api/products/123",
|
|
91
|
-
method: "GET",
|
|
92
|
-
contentType: "application/json",
|
|
93
|
-
data: {},
|
|
94
|
-
},
|
|
95
|
-
{
|
|
96
|
-
url: "https://example.com/api/products/456",
|
|
97
|
-
method: "GET",
|
|
98
|
-
contentType: "application/json",
|
|
99
|
-
data: {},
|
|
100
|
-
},
|
|
101
|
-
]);
|
|
102
|
-
expect(rows).toHaveLength(1);
|
|
103
|
-
});
|
|
104
|
-
it("keeps distinct methods as separate patterns", () => {
|
|
105
|
-
const rows = discoverApiPatterns("example.com", [
|
|
106
|
-
{
|
|
107
|
-
url: "https://example.com/api/products/123",
|
|
108
|
-
method: "GET",
|
|
109
|
-
contentType: "application/json",
|
|
110
|
-
data: {},
|
|
111
|
-
},
|
|
112
|
-
{
|
|
113
|
-
url: "https://example.com/api/products/123",
|
|
114
|
-
method: "POST",
|
|
115
|
-
contentType: "application/json",
|
|
116
|
-
data: {},
|
|
117
|
-
},
|
|
118
|
-
]);
|
|
119
|
-
expect(rows).toHaveLength(2);
|
|
120
|
-
});
|
|
121
|
-
it("strips parameters from the content type", () => {
|
|
122
|
-
const rows = discoverApiPatterns("example.com", [
|
|
123
|
-
{
|
|
124
|
-
url: "https://example.com/api/items",
|
|
125
|
-
method: "GET",
|
|
126
|
-
contentType: "application/json; charset=utf-8",
|
|
127
|
-
data: {},
|
|
128
|
-
},
|
|
129
|
-
]);
|
|
130
|
-
expect(rows[0]?.content_type).toBe("application/json");
|
|
131
|
-
});
|
|
132
|
-
it("skips malformed URLs", () => {
|
|
133
|
-
const rows = discoverApiPatterns("example.com", [
|
|
134
|
-
{
|
|
135
|
-
url: "not a url",
|
|
136
|
-
method: "GET",
|
|
137
|
-
contentType: "application/json",
|
|
138
|
-
data: {},
|
|
139
|
-
},
|
|
140
|
-
]);
|
|
141
|
-
expect(rows).toHaveLength(0);
|
|
142
|
-
});
|
|
143
|
-
});
|
|
144
|
-
//# sourceMappingURL=apiIntercept.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"apiIntercept.test.js","sourceRoot":"","sources":["../../src/extraction/apiIntercept.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAElD,OAAO,EACL,mBAAmB,EACnB,iBAAiB,GAGlB,MAAM,mBAAmB,CAAC;AAE3B,+CAA+C;AAC/C,SAAS,YAAY,CACnB,GAAW,EACX,MAAc,EACd,WAAmB,EACnB,IAAa;IAEb,OAAO;QACL,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG;QACd,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QACzC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;QAChD,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI;KACvB,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,SAAS,QAAQ;IAGf,MAAM,SAAS,GAAoC,EAAE,CAAC;IACtD,OAAO;QACL,SAAS;QACT,EAAE,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE;YACvB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QACD,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE;YACxB,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAA6B,CAAC;QAC/D,MAAM,QAAQ,CACZ,YAAY,CACV,sCAAsC,EACtC,KAAK,EACL,iCAAiC,EACjC,EAAE,EAAE,EAAE,GAAG,EAAE,CACZ,CACF,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YACvC,GAAG,EAAE,sCAAsC;YAC3C,MAAM,EAAE,KAAK;YACb,WAAW,EAAE,iCAAiC;YAC9C,IAAI,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE;SAClB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAA6B,CAAC;QAC/D,MAAM,QAAQ,CACZ,YAAY,CAAC,+BAA+B,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAC3E,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAA6B,CAAC;QAC/D,MAAM,GAAG,GAAG,YAAY,CACtB,gCAAgC,EAChC,KAAK,EACL,kBAAkB,EAClB,EAAE,EAAE,EAAE,CAAC,EAAE,CACV,CAAC;QACF,GAAG,CAAC,IAAI,GAAG,KAAK,IAAI,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;QAClC,CAAC,CAAC;QACF,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEpB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,CAAC,MAAM,EAAE,CAAC;QAEhB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,IAAI,GAAG,mBAAmB,CAAC,aAAa,EAAE;YAC9C;gBACE,GAAG,EAAE,sCAAsC;gBAC3C,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;YACnB;gBACE,MAAM,EAAE,aAAa;gBACrB,gBAAgB,EAAE,iBAAiB;gBACnC,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,kBAAkB;aACjC;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,mBAAmB,CAAC,aAAa,EAAE;YAC9C;gBACE,GAAG,EAAE,sCAAsC;gBAC3C,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;YACD;gBACE,GAAG,EAAE,sCAAsC;gBAC3C,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,mBAAmB,CAAC,aAAa,EAAE;YAC9C;gBACE,GAAG,EAAE,sCAAsC;gBAC3C,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;YACD;gBACE,GAAG,EAAE,sCAAsC;gBAC3C,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,IAAI,GAAG,mBAAmB,CAAC,aAAa,EAAE;YAC9C;gBACE,GAAG,EAAE,+BAA+B;gBACpC,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,iCAAiC;gBAC9C,IAAI,EAAE,EAAE;aACT;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,IAAI,GAAG,mBAAmB,CAAC,aAAa,EAAE;YAC9C;gBACE,GAAG,EAAE,WAAW;gBAChB,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,EAAE;aACT;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.test.d.ts","sourceRoot":"","sources":["../../src/extraction/browser.test.ts"],"names":[],"mappings":""}
|