better-web-search-mcp 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/.vscode/mcp.json +9 -0
  2. package/LICENSE +21 -0
  3. package/README.md +112 -0
  4. package/dist/extraction/alternative.d.ts +60 -0
  5. package/dist/extraction/alternative.d.ts.map +1 -0
  6. package/dist/extraction/alternative.js +169 -0
  7. package/dist/extraction/alternative.js.map +1 -0
  8. package/dist/extraction/alternative.test.d.ts +2 -0
  9. package/dist/extraction/alternative.test.d.ts.map +1 -0
  10. package/dist/extraction/alternative.test.js +145 -0
  11. package/dist/extraction/alternative.test.js.map +1 -0
  12. package/dist/extraction/apiIntercept.d.ts +72 -0
  13. package/dist/extraction/apiIntercept.d.ts.map +1 -0
  14. package/dist/extraction/apiIntercept.js +102 -0
  15. package/dist/extraction/apiIntercept.js.map +1 -0
  16. package/dist/extraction/apiIntercept.test.d.ts +2 -0
  17. package/dist/extraction/apiIntercept.test.d.ts.map +1 -0
  18. package/dist/extraction/apiIntercept.test.js +144 -0
  19. package/dist/extraction/apiIntercept.test.js.map +1 -0
  20. package/dist/extraction/browser.d.ts +109 -0
  21. package/dist/extraction/browser.d.ts.map +1 -0
  22. package/dist/extraction/browser.js +203 -0
  23. package/dist/extraction/browser.js.map +1 -0
  24. package/dist/extraction/browser.test.d.ts +2 -0
  25. package/dist/extraction/browser.test.d.ts.map +1 -0
  26. package/dist/extraction/browser.test.js +137 -0
  27. package/dist/extraction/browser.test.js.map +1 -0
  28. package/dist/extraction/evidence.d.ts +63 -0
  29. package/dist/extraction/evidence.d.ts.map +1 -0
  30. package/dist/extraction/evidence.js +10 -0
  31. package/dist/extraction/evidence.js.map +1 -0
  32. package/dist/extraction/fetch.d.ts +53 -0
  33. package/dist/extraction/fetch.d.ts.map +1 -0
  34. package/dist/extraction/fetch.js +77 -0
  35. package/dist/extraction/fetch.js.map +1 -0
  36. package/dist/extraction/fetch.test.d.ts +2 -0
  37. package/dist/extraction/fetch.test.d.ts.map +1 -0
  38. package/dist/extraction/fetch.test.js +85 -0
  39. package/dist/extraction/fetch.test.js.map +1 -0
  40. package/dist/extraction/fusion.d.ts +33 -0
  41. package/dist/extraction/fusion.d.ts.map +1 -0
  42. package/dist/extraction/fusion.js +130 -0
  43. package/dist/extraction/fusion.js.map +1 -0
  44. package/dist/extraction/fusion.test.d.ts +2 -0
  45. package/dist/extraction/fusion.test.d.ts.map +1 -0
  46. package/dist/extraction/fusion.test.js +91 -0
  47. package/dist/extraction/fusion.test.js.map +1 -0
  48. package/dist/extraction/learn.d.ts +20 -0
  49. package/dist/extraction/learn.d.ts.map +1 -0
  50. package/dist/extraction/learn.js +62 -0
  51. package/dist/extraction/learn.js.map +1 -0
  52. package/dist/extraction/metadata.d.ts +28 -0
  53. package/dist/extraction/metadata.d.ts.map +1 -0
  54. package/dist/extraction/metadata.js +32 -0
  55. package/dist/extraction/metadata.js.map +1 -0
  56. package/dist/extraction/metadata.test.d.ts +2 -0
  57. package/dist/extraction/metadata.test.d.ts.map +1 -0
  58. package/dist/extraction/metadata.test.js +45 -0
  59. package/dist/extraction/metadata.test.js.map +1 -0
  60. package/dist/extraction/readability.d.ts +37 -0
  61. package/dist/extraction/readability.d.ts.map +1 -0
  62. package/dist/extraction/readability.js +56 -0
  63. package/dist/extraction/readability.js.map +1 -0
  64. package/dist/extraction/readability.test.d.ts +2 -0
  65. package/dist/extraction/readability.test.d.ts.map +1 -0
  66. package/dist/extraction/readability.test.js +51 -0
  67. package/dist/extraction/readability.test.js.map +1 -0
  68. package/dist/extraction/router.d.ts +93 -0
  69. package/dist/extraction/router.d.ts.map +1 -0
  70. package/dist/extraction/router.js +277 -0
  71. package/dist/extraction/router.js.map +1 -0
  72. package/dist/extraction/router.test.d.ts +2 -0
  73. package/dist/extraction/router.test.d.ts.map +1 -0
  74. package/dist/extraction/router.test.js +238 -0
  75. package/dist/extraction/router.test.js.map +1 -0
  76. package/dist/extraction/structured.d.ts +33 -0
  77. package/dist/extraction/structured.d.ts.map +1 -0
  78. package/dist/extraction/structured.js +168 -0
  79. package/dist/extraction/structured.js.map +1 -0
  80. package/dist/extraction/structured.test.d.ts +2 -0
  81. package/dist/extraction/structured.test.d.ts.map +1 -0
  82. package/dist/extraction/structured.test.js +85 -0
  83. package/dist/extraction/structured.test.js.map +1 -0
  84. package/dist/index.d.ts +3 -0
  85. package/dist/index.d.ts.map +1 -0
  86. package/dist/index.js +71 -0
  87. package/dist/index.js.map +1 -0
  88. package/dist/providers/brave.d.ts +22 -0
  89. package/dist/providers/brave.d.ts.map +1 -0
  90. package/dist/providers/brave.js +111 -0
  91. package/dist/providers/brave.js.map +1 -0
  92. package/dist/providers/brave.test.d.ts +2 -0
  93. package/dist/providers/brave.test.d.ts.map +1 -0
  94. package/dist/providers/brave.test.js +150 -0
  95. package/dist/providers/brave.test.js.map +1 -0
  96. package/dist/providers/duckduckgo.d.ts +15 -0
  97. package/dist/providers/duckduckgo.d.ts.map +1 -0
  98. package/dist/providers/duckduckgo.js +90 -0
  99. package/dist/providers/duckduckgo.js.map +1 -0
  100. package/dist/providers/duckduckgo.test.d.ts +2 -0
  101. package/dist/providers/duckduckgo.test.d.ts.map +1 -0
  102. package/dist/providers/duckduckgo.test.js +102 -0
  103. package/dist/providers/duckduckgo.test.js.map +1 -0
  104. package/dist/providers/index.d.ts +19 -0
  105. package/dist/providers/index.d.ts.map +1 -0
  106. package/dist/providers/index.js +48 -0
  107. package/dist/providers/index.js.map +1 -0
  108. package/dist/providers/index.test.d.ts +2 -0
  109. package/dist/providers/index.test.d.ts.map +1 -0
  110. package/dist/providers/index.test.js +114 -0
  111. package/dist/providers/index.test.js.map +1 -0
  112. package/dist/providers/serpapi.d.ts +13 -0
  113. package/dist/providers/serpapi.d.ts.map +1 -0
  114. package/dist/providers/serpapi.js +14 -0
  115. package/dist/providers/serpapi.js.map +1 -0
  116. package/dist/providers/tavily.d.ts +13 -0
  117. package/dist/providers/tavily.d.ts.map +1 -0
  118. package/dist/providers/tavily.js +65 -0
  119. package/dist/providers/tavily.js.map +1 -0
  120. package/dist/providers/tavily.test.d.ts +2 -0
  121. package/dist/providers/tavily.test.d.ts.map +1 -0
  122. package/dist/providers/tavily.test.js +123 -0
  123. package/dist/providers/tavily.test.js.map +1 -0
  124. package/dist/providers/types.d.ts +48 -0
  125. package/dist/providers/types.d.ts.map +1 -0
  126. package/dist/providers/types.js +10 -0
  127. package/dist/providers/types.js.map +1 -0
  128. package/dist/ranking/deduplicate.d.ts +22 -0
  129. package/dist/ranking/deduplicate.d.ts.map +1 -0
  130. package/dist/ranking/deduplicate.js +64 -0
  131. package/dist/ranking/deduplicate.js.map +1 -0
  132. package/dist/ranking/deduplicate.test.d.ts +2 -0
  133. package/dist/ranking/deduplicate.test.d.ts.map +1 -0
  134. package/dist/ranking/deduplicate.test.js +76 -0
  135. package/dist/ranking/deduplicate.test.js.map +1 -0
  136. package/dist/ranking/domainScore.d.ts +14 -0
  137. package/dist/ranking/domainScore.d.ts.map +1 -0
  138. package/dist/ranking/domainScore.js +102 -0
  139. package/dist/ranking/domainScore.js.map +1 -0
  140. package/dist/ranking/domainScore.test.d.ts +2 -0
  141. package/dist/ranking/domainScore.test.d.ts.map +1 -0
  142. package/dist/ranking/domainScore.test.js +37 -0
  143. package/dist/ranking/domainScore.test.js.map +1 -0
  144. package/dist/ranking/rerank.d.ts +17 -0
  145. package/dist/ranking/rerank.d.ts.map +1 -0
  146. package/dist/ranking/rerank.js +78 -0
  147. package/dist/ranking/rerank.js.map +1 -0
  148. package/dist/ranking/rerank.test.d.ts +2 -0
  149. package/dist/ranking/rerank.test.d.ts.map +1 -0
  150. package/dist/ranking/rerank.test.js +71 -0
  151. package/dist/ranking/rerank.test.js.map +1 -0
  152. package/dist/scaffold.test.d.ts +2 -0
  153. package/dist/scaffold.test.d.ts.map +1 -0
  154. package/dist/scaffold.test.js +7 -0
  155. package/dist/scaffold.test.js.map +1 -0
  156. package/dist/tools/extract.d.ts +52 -0
  157. package/dist/tools/extract.d.ts.map +1 -0
  158. package/dist/tools/extract.js +134 -0
  159. package/dist/tools/extract.js.map +1 -0
  160. package/dist/tools/extract.test.d.ts +2 -0
  161. package/dist/tools/extract.test.d.ts.map +1 -0
  162. package/dist/tools/extract.test.js +110 -0
  163. package/dist/tools/extract.test.js.map +1 -0
  164. package/dist/tools/find.d.ts +38 -0
  165. package/dist/tools/find.d.ts.map +1 -0
  166. package/dist/tools/find.js +74 -0
  167. package/dist/tools/find.js.map +1 -0
  168. package/dist/tools/index.d.ts +11 -0
  169. package/dist/tools/index.d.ts.map +1 -0
  170. package/dist/tools/index.js +22 -0
  171. package/dist/tools/index.js.map +1 -0
  172. package/dist/tools/news.d.ts +39 -0
  173. package/dist/tools/news.d.ts.map +1 -0
  174. package/dist/tools/news.js +131 -0
  175. package/dist/tools/news.js.map +1 -0
  176. package/dist/tools/research.d.ts +47 -0
  177. package/dist/tools/research.d.ts.map +1 -0
  178. package/dist/tools/research.js +204 -0
  179. package/dist/tools/research.js.map +1 -0
  180. package/dist/tools/research.test.d.ts +2 -0
  181. package/dist/tools/research.test.d.ts.map +1 -0
  182. package/dist/tools/research.test.js +181 -0
  183. package/dist/tools/research.test.js.map +1 -0
  184. package/dist/tools/search.d.ts +45 -0
  185. package/dist/tools/search.d.ts.map +1 -0
  186. package/dist/tools/search.js +79 -0
  187. package/dist/tools/search.js.map +1 -0
  188. package/dist/tools/search.test.d.ts +2 -0
  189. package/dist/tools/search.test.d.ts.map +1 -0
  190. package/dist/tools/search.test.js +101 -0
  191. package/dist/tools/search.test.js.map +1 -0
  192. package/dist/utils/cache.d.ts +49 -0
  193. package/dist/utils/cache.d.ts.map +1 -0
  194. package/dist/utils/cache.js +248 -0
  195. package/dist/utils/cache.js.map +1 -0
  196. package/dist/utils/cache.test.d.ts +2 -0
  197. package/dist/utils/cache.test.d.ts.map +1 -0
  198. package/dist/utils/cache.test.js +172 -0
  199. package/dist/utils/cache.test.js.map +1 -0
  200. package/dist/utils/cacheTypes.d.ts +36 -0
  201. package/dist/utils/cacheTypes.d.ts.map +1 -0
  202. package/dist/utils/cacheTypes.js +12 -0
  203. package/dist/utils/cacheTypes.js.map +1 -0
  204. package/dist/utils/config.d.ts +30 -0
  205. package/dist/utils/config.d.ts.map +1 -0
  206. package/dist/utils/config.js +92 -0
  207. package/dist/utils/config.js.map +1 -0
  208. package/dist/utils/config.test.d.ts +2 -0
  209. package/dist/utils/config.test.d.ts.map +1 -0
  210. package/dist/utils/config.test.js +52 -0
  211. package/dist/utils/config.test.js.map +1 -0
  212. package/dist/utils/domainProfile.d.ts +56 -0
  213. package/dist/utils/domainProfile.d.ts.map +1 -0
  214. package/dist/utils/domainProfile.js +58 -0
  215. package/dist/utils/domainProfile.js.map +1 -0
  216. package/dist/utils/domainProfile.test.d.ts +2 -0
  217. package/dist/utils/domainProfile.test.d.ts.map +1 -0
  218. package/dist/utils/domainProfile.test.js +74 -0
  219. package/dist/utils/domainProfile.test.js.map +1 -0
  220. package/dist/utils/memoryCache.d.ts +42 -0
  221. package/dist/utils/memoryCache.d.ts.map +1 -0
  222. package/dist/utils/memoryCache.js +102 -0
  223. package/dist/utils/memoryCache.js.map +1 -0
  224. package/dist/utils/queries.d.ts +18 -0
  225. package/dist/utils/queries.d.ts.map +1 -0
  226. package/dist/utils/queries.js +77 -0
  227. package/dist/utils/queries.js.map +1 -0
  228. package/dist/utils/queries.test.d.ts +2 -0
  229. package/dist/utils/queries.test.d.ts.map +1 -0
  230. package/dist/utils/queries.test.js +36 -0
  231. package/dist/utils/queries.test.js.map +1 -0
  232. package/dist/utils/retry.d.ts +38 -0
  233. package/dist/utils/retry.d.ts.map +1 -0
  234. package/dist/utils/retry.js +115 -0
  235. package/dist/utils/retry.js.map +1 -0
  236. package/dist/utils/retry.test.d.ts +2 -0
  237. package/dist/utils/retry.test.d.ts.map +1 -0
  238. package/dist/utils/retry.test.js +143 -0
  239. package/dist/utils/retry.test.js.map +1 -0
  240. package/mcp.json +12 -0
  241. package/package.json +94 -0
  242. package/scripts/check-native.mjs +22 -0
  243. package/scripts/smoke.mjs +43 -0
  244. package/scripts/test-mcp-tools.mjs +55 -0
  245. package/scripts/test-mcp.mjs +59 -0
  246. package/scripts/typedoc-frontmatter.mjs +26 -0
  247. package/scripts/typedoc-postprocess.mjs +77 -0
  248. package/smithery.yaml +38 -0
@@ -0,0 +1,9 @@
1
+ {
2
+ "servers": {
3
+ "better-web-search-mcp": {
4
+ "command": "npx",
5
+ "args": ["-y", "better-web-search-mcp"],
6
+ "env": {}
7
+ }
8
+ }
9
+ }
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 PhantomPixelDev
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,112 @@
1
+ # BetterWebSearch MCP
2
+
3
+ > Keyless-first web search for AI agents. DuckDuckGo works out of the box. No API keys.
4
+
5
+ [![npm version](https://img.shields.io/npm/v/better-web-search-mcp?color=cb3837)](https://www.npmjs.com/package/better-web-search-mcp)
6
+ [![CI](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions/workflows/ci.yml/badge.svg)](https://github.com/PhantomPixelDev/BetterWebSearch-MCP/actions)
7
+ [![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://phantompixeldev.github.io/BetterWebSearch-MCP/)
8
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
9
+ [![Smithery](https://smithery.ai/badge/@PhantomPixelDev/better-web-search-mcp)](https://smithery.ai/server/@PhantomPixelDev/better-web-search-mcp)
10
+
11
+ <p align="center">
12
+ <img src="./images/banner.jpg" alt="BetterWebSearch MCP — Smarter Web Search · Better Data Extraction · Flexible & Extensible · Open Source" width="100%">
13
+ </p>
14
+
15
+ ## What is this?
16
+
17
+ BetterWebSearch MCP is a [Model Context Protocol](https://modelcontextprotocol.io) server that gives AI assistants genuinely better web research. It works without API keys by default (DuckDuckGo), and optionally adds Brave or Tavily for richer ranking and recency.
18
+
19
+ Built for anyone who wants their AI tools to find things on the web reliably, not just when a paid API key is configured.
20
+
21
+ ## Quick start (30 seconds)
22
+
23
+ Add this to your MCP client config (Claude Desktop, Cursor, VS Code Copilot, OpenCode, or any MCP client):
24
+
25
+ ```json
26
+ {
27
+ "mcpServers": {
28
+ "better-web-search-mcp": {
29
+ "command": "npx",
30
+ "args": ["-y", "better-web-search-mcp"]
31
+ }
32
+ }
33
+ }
34
+ ```
35
+
36
+ That's it. Every search tool works immediately. No `.env`, no API keys, no setup.
37
+
38
+ Or run it directly:
39
+
40
+ ```bash
41
+ npx -y better-web-search-mcp --help
42
+ ```
43
+
44
+ ## Tools
45
+
46
+ | Tool | What it does |
47
+ |---|---|
48
+ | `web_search` | Aggregated search across providers, deduplicated and reranked |
49
+ | `web_research` | Deep research. Rewrites your question, searches in parallel, extracts top results with citations |
50
+ | `deep_search` | Alias for `web_research` |
51
+ | `web_extract` | Extracts clean content from any URL, using the best method available |
52
+ | `web_find` | Search scoped to a single website |
53
+ | `web_news` | Recent news with timeline and diversity filtering |
54
+
55
+ ## How search is better
56
+
57
+ Standard MCP search tools hit one API and return raw results. BetterWebSearch takes a different approach.
58
+
59
+ **Query expansion.** Your single question gets rewritten into 4 to 6 variants, searched in parallel. "Unlimited mobile internet Germany" also searches for "unbegrenztes Datenvolumen Deutschland" and similar phrasings. More coverage without you doing extra work.
60
+
61
+ **Three-tier extraction.** When it fetches a page, it doesn't just grab the HTML and hope for the best:
62
+
63
+ ```
64
+ Tier 1: Fast HTTP fetch (< 1s)
65
+ ↓ if not enough content
66
+ Tier 2: Structured data (JSON-LD, __NEXT_DATA__, __NUXT__) (1-3s)
67
+ ↓ if still not enough
68
+ Tier 3: Playwright browser (full render + API interception) (3-10s)
69
+ ```
70
+
71
+ Only escalates when needed. Results carry confidence scores so you know what you're getting.
72
+
73
+ **Self-learning cache.** The first visit to a domain takes the full path. The second visit skips straight to what worked. Domain profiles and API patterns are remembered.
74
+
75
+ ## Project structure
76
+
77
+ ```
78
+ src/
79
+ providers/ Search backends (DuckDuckGo, Brave, Tavily, SERPApi)
80
+ extraction/ 3-tier content pipeline (fetch, structured, browser)
81
+ ranking/ Deduplication, domain scoring, reranking
82
+ tools/ MCP tool definitions (search, research, extract, find, news)
83
+ utils/ Config, caching, retry, query rewriting
84
+ ```
85
+
86
+ ## Configuration
87
+
88
+ All configuration is optional. DuckDuckGo works with zero setup.
89
+
90
+ ```bash
91
+ # Optional: add more providers
92
+ BRAVE_API_KEY=your_key # Better ranking and recency
93
+ TAVILY_API_KEY=your_key # Alternative provider
94
+ ```
95
+
96
+ See the [full configuration reference](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) for all options including cache paths, browser settings, and environment variable names.
97
+
98
+ ## Documentation
99
+
100
+ Full guides, API reference, and architecture details live on the docs site:
101
+
102
+ **[phantompixeldev.github.io/BetterWebSearch-MCP](https://phantompixeldev.github.io/BetterWebSearch-MCP/)**
103
+
104
+ - [Installation](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/installation/) - npx, npm, or from source
105
+ - [Quick Start](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/quickstart/) - Get running in your first AI client
106
+ - [Configuration](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/configuration/) - All environment variables and options
107
+ - [Contributing](https://phantompixeldev.github.io/BetterWebSearch-MCP/docs/contributing/) - Development setup and PR process
108
+ - [Changelog](CHANGELOG.md) - Release history and version notes
109
+
110
+ ## License
111
+
112
+ MIT
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Alternative source discovery + search-snippet evidence.
3
+ *
4
+ * When the primary extraction is blocked (403 / paywall) or low-confidence,
5
+ * this module tries public URL variants of the original page (AMP,
6
+ * `?output=1`, `?amp`) and, if those are still blocked, falls back to public
7
+ * search snippets as low-confidence evidence. Only public search + AMP
8
+ * variants are used — no paywall bypass.
9
+ */
10
+ import { fetchPage } from "./fetch.js";
11
+ import { aggregateSearch } from "../providers/index.js";
12
+ import type { Evidence } from "./evidence.js";
13
+ import type { Cache } from "../utils/cache.js";
14
+ /** Confidence assigned to search-snippet evidence. */
15
+ export declare const SNIPPET_CONFIDENCE = 0.6;
16
+ /** Confidence assigned to a successfully fetched URL variant. */
17
+ export declare const VARIANT_CONFIDENCE = 0.6;
18
+ /** Maximum number of search snippets kept as evidence. */
19
+ export declare const MAX_SNIPPETS = 3;
20
+ /** Maximum snippet length for page-variant evidence. */
21
+ export declare const MAX_SNIPPET_CHARS = 200;
22
+ /** Injectable dependencies, defaulting to the real modules. */
23
+ export interface AlternativeDeps {
24
+ /** Page fetcher used for URL variants. */
25
+ fetchPage?: typeof fetchPage;
26
+ /** Search aggregator used for snippet fallback. */
27
+ aggregateSearch?: typeof aggregateSearch;
28
+ /** Page cache consulted before fetching each variant. */
29
+ cache?: Cache;
30
+ }
31
+ /**
32
+ * Build the URL variants to try: original, `?output=1`, `/amp`, `?amp`.
33
+ *
34
+ * Query strings are preserved and extended with `&` when present; the `/amp`
35
+ * variant is inserted before the query. Returns the original URL alone when
36
+ * it cannot be parsed.
37
+ */
38
+ export declare function buildAlternativeUrls(url: string): string[];
39
+ /**
40
+ * Build the search queries for snippet fallback.
41
+ *
42
+ * With a title: quoted exact title, site-restricted quoted title, and
43
+ * title + domain keywords. Without a title: a bare `site:` query so the
44
+ * domain is still searched.
45
+ */
46
+ export declare function buildSearchQueries(url: string, title: string | null): string[];
47
+ /**
48
+ * Discover alternative sources for a blocked or low-confidence page.
49
+ *
50
+ * 1. Tries URL variants (`?output=1`, `/amp`, `?amp`) via {@link fetchPage},
51
+ * cache-aware, returning the first usable variant as page evidence.
52
+ * 2. If every variant is blocked, runs {@link aggregateSearch} with the
53
+ * quoted title, a `site:`-restricted query, and title + domain keywords.
54
+ * 3. Treats the top {@link MAX_SNIPPETS} search results as `search_snippet`
55
+ * evidence at confidence {@link SNIPPET_CONFIDENCE}.
56
+ *
57
+ * Never throws on empty or failing searches — it returns `[]`.
58
+ */
59
+ export declare function findAlternativeSources(url: string, title: string | null, deps?: AlternativeDeps): Promise<Evidence[]>;
60
+ //# sourceMappingURL=alternative.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"alternative.d.ts","sourceRoot":"","sources":["../../src/extraction/alternative.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE/C,sDAAsD;AACtD,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,iEAAiE;AACjE,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,0DAA0D;AAC1D,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,+DAA+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;IAC7B,mDAAmD;IACnD,eAAe,CAAC,EAAE,OAAO,eAAe,CAAC;IACzC,yDAAyD;IACzD,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AASD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAgB1D;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GAAG,IAAI,GACnB,MAAM,EAAE,CAgBV;AA0DD;;;;;;;;;;;GAWG;AACH,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAwCrB"}
@@ -0,0 +1,169 @@
1
+ /**
2
+ * Alternative source discovery + search-snippet evidence.
3
+ *
4
+ * When the primary extraction is blocked (403 / paywall) or low-confidence,
5
+ * this module tries public URL variants of the original page (AMP,
6
+ * `?output=1`, `?amp`) and, if those are still blocked, falls back to public
7
+ * search snippets as low-confidence evidence. Only public search + AMP
8
+ * variants are used — no paywall bypass.
9
+ */
10
+ import { fetchPage } from "./fetch.js";
11
+ import { aggregateSearch } from "../providers/index.js";
12
+ /** Confidence assigned to search-snippet evidence. */
13
+ export const SNIPPET_CONFIDENCE = 0.6;
14
+ /** Confidence assigned to a successfully fetched URL variant. */
15
+ export const VARIANT_CONFIDENCE = 0.6;
16
+ /** Maximum number of search snippets kept as evidence. */
17
+ export const MAX_SNIPPETS = 3;
18
+ /** Maximum snippet length for page-variant evidence. */
19
+ export const MAX_SNIPPET_CHARS = 200;
20
+ /**
21
+ * Build the URL variants to try: original, `?output=1`, `/amp`, `?amp`.
22
+ *
23
+ * Query strings are preserved and extended with `&` when present; the `/amp`
24
+ * variant is inserted before the query. Returns the original URL alone when
25
+ * it cannot be parsed.
26
+ */
27
+ export function buildAlternativeUrls(url) {
28
+ let parsed;
29
+ try {
30
+ parsed = new URL(url);
31
+ }
32
+ catch {
33
+ return [url];
34
+ }
35
+ const base = parsed.origin + parsed.pathname;
36
+ const query = parsed.search;
37
+ const sep = query === "" ? "?" : "&";
38
+ return [
39
+ url,
40
+ `${base}${query}${sep}output=1`,
41
+ `${base}/amp${query}`,
42
+ `${base}${query}${sep}amp`,
43
+ ];
44
+ }
45
+ /**
46
+ * Build the search queries for snippet fallback.
47
+ *
48
+ * With a title: quoted exact title, site-restricted quoted title, and
49
+ * title + domain keywords. Without a title: a bare `site:` query so the
50
+ * domain is still searched.
51
+ */
52
+ export function buildSearchQueries(url, title) {
53
+ let domain;
54
+ try {
55
+ domain = new URL(url).hostname;
56
+ }
57
+ catch {
58
+ return [];
59
+ }
60
+ const trimmed = title?.trim() ?? "";
61
+ if (trimmed === "") {
62
+ return [`site:${domain}`];
63
+ }
64
+ return [
65
+ `"${trimmed}"`,
66
+ `site:${domain} "${trimmed}"`,
67
+ `${trimmed} ${domain}`,
68
+ ];
69
+ }
70
+ /** Strip HTML tags to produce a readable snippet. */
71
+ function stripHtml(html) {
72
+ return html
73
+ .replace(/<script[\s\S]*?<\/script>/gi, " ")
74
+ .replace(/<style[\s\S]*?<\/style>/gi, " ")
75
+ .replace(/<[^>]+>/g, " ")
76
+ .replace(/\s+/g, " ")
77
+ .trim();
78
+ }
79
+ /** Whether a fetched page is usable (2xx with non-empty HTML). */
80
+ function isUsable(page) {
81
+ return page.status >= 200 && page.status < 300 && page.html.trim() !== "";
82
+ }
83
+ /**
84
+ * Try each URL variant, returning page evidence on the first success.
85
+ *
86
+ * Each variant is checked against the cache before fetching. A variant is
87
+ * usable when it returns 2xx with non-empty HTML; anything else (403, empty
88
+ * body, timeout, non-HTML) is treated as blocked and the next variant is
89
+ * tried.
90
+ */
91
+ async function tryVariants(url, d) {
92
+ for (const variant of buildAlternativeUrls(url)) {
93
+ const cached = d.cache?.getPage(variant);
94
+ if (cached !== null && cached !== undefined) {
95
+ return {
96
+ source: "page",
97
+ type: "page",
98
+ confidence: cached.confidence,
99
+ url: variant,
100
+ snippet: cached.content.slice(0, MAX_SNIPPET_CHARS),
101
+ };
102
+ }
103
+ try {
104
+ const page = await d.fetchPage(variant);
105
+ if (isUsable(page)) {
106
+ return {
107
+ source: "page",
108
+ type: "page",
109
+ confidence: VARIANT_CONFIDENCE,
110
+ url: variant,
111
+ snippet: stripHtml(page.html).slice(0, MAX_SNIPPET_CHARS),
112
+ };
113
+ }
114
+ }
115
+ catch {
116
+ // Blocked / timeout / non-HTML — try the next variant.
117
+ }
118
+ }
119
+ return null;
120
+ }
121
+ /**
122
+ * Discover alternative sources for a blocked or low-confidence page.
123
+ *
124
+ * 1. Tries URL variants (`?output=1`, `/amp`, `?amp`) via {@link fetchPage},
125
+ * cache-aware, returning the first usable variant as page evidence.
126
+ * 2. If every variant is blocked, runs {@link aggregateSearch} with the
127
+ * quoted title, a `site:`-restricted query, and title + domain keywords.
128
+ * 3. Treats the top {@link MAX_SNIPPETS} search results as `search_snippet`
129
+ * evidence at confidence {@link SNIPPET_CONFIDENCE}.
130
+ *
131
+ * Never throws on empty or failing searches — it returns `[]`.
132
+ */
133
+ export async function findAlternativeSources(url, title, deps = {}) {
134
+ const d = {
135
+ fetchPage: deps.fetchPage ?? fetchPage,
136
+ aggregateSearch: deps.aggregateSearch ?? aggregateSearch,
137
+ cache: deps.cache,
138
+ };
139
+ const variant = await tryVariants(url, d);
140
+ if (variant !== null) {
141
+ return [variant];
142
+ }
143
+ const queries = buildSearchQueries(url, title);
144
+ const collected = [];
145
+ for (const query of queries) {
146
+ try {
147
+ collected.push(...(await d.aggregateSearch(query, { count: MAX_SNIPPETS })));
148
+ }
149
+ catch {
150
+ // Provider failure — keep whatever we already collected.
151
+ }
152
+ }
153
+ const seen = new Set();
154
+ const unique = [];
155
+ for (const result of collected) {
156
+ if (!seen.has(result.url)) {
157
+ seen.add(result.url);
158
+ unique.push(result);
159
+ }
160
+ }
161
+ return unique.slice(0, MAX_SNIPPETS).map((result) => ({
162
+ source: "search_snippet",
163
+ type: "search_snippet",
164
+ confidence: SNIPPET_CONFIDENCE,
165
+ url: result.url,
166
+ snippet: result.snippet,
167
+ }));
168
+ }
169
+ //# sourceMappingURL=alternative.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"alternative.js","sourceRoot":"","sources":["../../src/extraction/alternative.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,SAAS,EAAoB,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAKxD,sDAAsD;AACtD,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,iEAAiE;AACjE,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,0DAA0D;AAC1D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAE9B,wDAAwD;AACxD,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAmBrC;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC9C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACrC,OAAO;QACL,GAAG;QACH,GAAG,IAAI,GAAG,KAAK,GAAG,GAAG,UAAU;QAC/B,GAAG,IAAI,OAAO,KAAK,EAAE;QACrB,GAAG,IAAI,GAAG,KAAK,GAAG,GAAG,KAAK;KAC3B,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,GAAW,EACX,KAAoB;IAEpB,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpC,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACnB,OAAO,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO;QACL,IAAI,OAAO,GAAG;QACd,QAAQ,MAAM,KAAK,OAAO,GAAG;QAC7B,GAAG,OAAO,IAAI,MAAM,EAAE;KACvB,CAAC;AACJ,CAAC;AAED,qDAAqD;AACrD,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;SAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,kEAAkE;AAClE,SAAS,QAAQ,CAAC,IAAiB;IACjC,OAAO,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;AAC5E,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,WAAW,CACxB,GAAW,EACX,CAAe;IAEf,KAAK,MAAM,OAAO,IAAI,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC5C,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC;aACpD,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,OAAO;oBACL,MAAM,EAAE,MAAM;oBACd,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,kBAAkB;oBAC9B,GAAG,EAAE,OAAO;oBACZ,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC;iBAC1D,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAAW,EACX,KAAoB,EACpB,OAAwB,EAAE;IAE1B,MAAM,CAAC,GAAiB;QACtB,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;QACtC,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,eAAe;QACxD,KAAK,EAAE,IAAI,CAAC,KAAK;KAClB,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,SAAS,CAAC,IAAI,CACZ,GAAG,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC,CAC7D,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,EAAE,gBAAgB;QACxB,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,kBAAkB;QAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC,CAAC,CAAC;AACN,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=alternative.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"alternative.test.d.ts","sourceRoot":"","sources":["../../src/extraction/alternative.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,145 @@
1
+ import { describe, expect, it, vi } from "vitest";
2
+ import { buildAlternativeUrls, buildSearchQueries, findAlternativeSources, MAX_SNIPPETS, SNIPPET_CONFIDENCE, } from "./alternative.js";
3
+ import { Cache } from "../utils/cache.js";
4
+ /** A blocked (403) page fixture. */
5
+ function blockedPage(url) {
6
+ return {
7
+ html: "",
8
+ headers: { "content-type": "text/html" },
9
+ status: 403,
10
+ url,
11
+ };
12
+ }
13
+ /** A usable (200) page fixture. */
14
+ function okPage(url) {
15
+ return {
16
+ html: "<html><body><article><p>full article text</p></article></body></html>",
17
+ headers: { "content-type": "text/html" },
18
+ status: 200,
19
+ url,
20
+ };
21
+ }
22
+ /** A search result fixture. */
23
+ const SNIPPET = {
24
+ title: "Syndicated copy",
25
+ url: "https://syndicated.example/article",
26
+ snippet: "The full article text appears here.",
27
+ source: "duckduckgo",
28
+ };
29
+ describe("buildAlternativeUrls", () => {
30
+ it("returns original, ?output=1, /amp, and ?amp variants", () => {
31
+ expect(buildAlternativeUrls("https://example.com/article")).toEqual([
32
+ "https://example.com/article",
33
+ "https://example.com/article?output=1",
34
+ "https://example.com/article/amp",
35
+ "https://example.com/article?amp",
36
+ ]);
37
+ });
38
+ it("uses & when the URL already has a query string", () => {
39
+ expect(buildAlternativeUrls("https://example.com/article?id=7")).toEqual([
40
+ "https://example.com/article?id=7",
41
+ "https://example.com/article?id=7&output=1",
42
+ "https://example.com/article/amp?id=7",
43
+ "https://example.com/article?id=7&amp",
44
+ ]);
45
+ });
46
+ it("returns the original URL when it cannot be parsed", () => {
47
+ expect(buildAlternativeUrls("not a url")).toEqual(["not a url"]);
48
+ });
49
+ });
50
+ describe("buildSearchQueries", () => {
51
+ it("quotes the title and adds a site-restricted query", () => {
52
+ expect(buildSearchQueries("https://example.com/article", "My Article")).toEqual([
53
+ '"My Article"',
54
+ 'site:example.com "My Article"',
55
+ "My Article example.com",
56
+ ]);
57
+ });
58
+ it("falls back to a bare site query when the title is null", () => {
59
+ expect(buildSearchQueries("https://example.com/article", null)).toEqual([
60
+ "site:example.com",
61
+ ]);
62
+ });
63
+ });
64
+ describe("findAlternativeSources", () => {
65
+ it("returns snippet evidence when every URL variant is blocked", async () => {
66
+ const fetchPage = vi
67
+ .fn()
68
+ .mockResolvedValue(blockedPage("https://example.com/article"));
69
+ const aggregateSearch = vi.fn().mockResolvedValue([SNIPPET]);
70
+ const deps = { fetchPage, aggregateSearch };
71
+ const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
72
+ expect(fetchPage).toHaveBeenCalledTimes(4);
73
+ expect(evidence).toHaveLength(1);
74
+ expect(evidence[0]).toEqual({
75
+ source: "search_snippet",
76
+ type: "search_snippet",
77
+ confidence: SNIPPET_CONFIDENCE,
78
+ url: "https://syndicated.example/article",
79
+ snippet: "The full article text appears here.",
80
+ });
81
+ });
82
+ it("returns page evidence when an AMP variant succeeds", async () => {
83
+ const fetchPage = vi.fn().mockImplementation(async (url) => {
84
+ return url.includes("/amp") ? okPage(url) : blockedPage(url);
85
+ });
86
+ const aggregateSearch = vi.fn();
87
+ const deps = { fetchPage, aggregateSearch };
88
+ const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
89
+ expect(aggregateSearch).not.toHaveBeenCalled();
90
+ expect(evidence).toHaveLength(1);
91
+ expect(evidence[0]).toMatchObject({
92
+ source: "page",
93
+ type: "page",
94
+ url: "https://example.com/article/amp",
95
+ });
96
+ });
97
+ it("uses a cached variant without fetching it", async () => {
98
+ const cache = new Cache({ memory: true });
99
+ cache.setPage("https://example.com/article/amp", "cached amp content", "http_fetch", 0.85);
100
+ const fetchPage = vi
101
+ .fn()
102
+ .mockResolvedValue(blockedPage("https://example.com/article"));
103
+ const aggregateSearch = vi.fn();
104
+ const deps = { fetchPage, aggregateSearch, cache };
105
+ const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
106
+ expect(fetchPage).toHaveBeenCalledTimes(2);
107
+ expect(aggregateSearch).not.toHaveBeenCalled();
108
+ expect(evidence).toHaveLength(1);
109
+ expect(evidence[0]).toMatchObject({
110
+ source: "page",
111
+ url: "https://example.com/article/amp",
112
+ confidence: 0.85,
113
+ });
114
+ });
115
+ it("generates a site fallback query when the title is null", async () => {
116
+ const fetchPage = vi
117
+ .fn()
118
+ .mockResolvedValue(blockedPage("https://example.com/article"));
119
+ const aggregateSearch = vi.fn().mockResolvedValue([SNIPPET]);
120
+ const deps = { fetchPage, aggregateSearch };
121
+ await findAlternativeSources("https://example.com/article", null, deps);
122
+ expect(aggregateSearch).toHaveBeenCalledWith("site:example.com", {
123
+ count: MAX_SNIPPETS,
124
+ });
125
+ });
126
+ it("returns [] when the search is empty instead of throwing", async () => {
127
+ const fetchPage = vi
128
+ .fn()
129
+ .mockResolvedValue(blockedPage("https://example.com/article"));
130
+ const aggregateSearch = vi.fn().mockResolvedValue([]);
131
+ const deps = { fetchPage, aggregateSearch };
132
+ const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
133
+ expect(evidence).toEqual([]);
134
+ });
135
+ it("returns [] when the search provider throws", async () => {
136
+ const fetchPage = vi
137
+ .fn()
138
+ .mockResolvedValue(blockedPage("https://example.com/article"));
139
+ const aggregateSearch = vi.fn().mockRejectedValue(new Error("provider down"));
140
+ const deps = { fetchPage, aggregateSearch };
141
+ const evidence = await findAlternativeSources("https://example.com/article", "My Article", deps);
142
+ expect(evidence).toEqual([]);
143
+ });
144
+ });
145
+ //# sourceMappingURL=alternative.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"alternative.test.js","sourceRoot":"","sources":["../../src/extraction/alternative.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACZ,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,oCAAoC;AACpC,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO;QACL,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;QACxC,MAAM,EAAE,GAAG;QACX,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,mCAAmC;AACnC,SAAS,MAAM,CAAC,GAAW;IACzB,OAAO;QACL,IAAI,EAAE,uEAAuE;QAC7E,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;QACxC,MAAM,EAAE,GAAG;QACX,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,+BAA+B;AAC/B,MAAM,OAAO,GAAiB;IAC5B,KAAK,EAAE,iBAAiB;IACxB,GAAG,EAAE,oCAAoC;IACzC,OAAO,EAAE,qCAAqC;IAC9C,MAAM,EAAE,YAAY;CACrB,CAAC;AAEF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CAAC,oBAAoB,CAAC,6BAA6B,CAAC,CAAC,CAAC,OAAO,CAAC;YAClE,6BAA6B;YAC7B,sCAAsC;YACtC,iCAAiC;YACjC,iCAAiC;SAClC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,oBAAoB,CAAC,kCAAkC,CAAC,CAAC,CAAC,OAAO,CAAC;YACvE,kCAAkC;YAClC,2CAA2C;YAC3C,sCAAsC;YACtC,sCAAsC;SACvC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,kBAAkB,CAAC,6BAA6B,EAAE,YAAY,CAAC,CAChE,CAAC,OAAO,CAAC;YACR,cAAc;YACd,+BAA+B;YAC/B,wBAAwB;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,CAAC,kBAAkB,CAAC,6BAA6B,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;YACtE,kBAAkB;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,kBAAkB;YAC9B,GAAG,EAAE,oCAAoC;YACzC,OAAO,EAAE,qCAAqC;SAC/C,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YACjE,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,iCAAiC;SACvC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,KAAK,CAAC,OAAO,CACX,iCAAiC,EACjC,oBAAoB,EACpB,YAAY,EACZ,IAAI,CACL,CAAC;QACF,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QAEpE,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,iCAAiC;YACtC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,sBAAsB,CAAC,6BAA6B,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAExE,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,kBAAkB,EAAE;YAC/D,KAAK,EAAE,YAAY;SACpB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,SAAS,GAAG,EAAE;aACjB,EAAE,EAAE;aACJ,iBAAiB,CAAC,WAAW,CAAC,6BAA6B,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,6BAA6B,EAC7B,YAAY,EACZ,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,72 @@
1
+ /**
2
+ * API interception during browser rendering.
3
+ *
4
+ * Hooks a Playwright page's `response` events to capture JSON API payloads
5
+ * (XHR / fetch responses) that the page loads while rendering. The captured
6
+ * responses are normalized into `api_patterns` rows via
7
+ * {@link discoverApiPatterns}, so the domain profiler can shortcut the next
8
+ * visit with a direct API call.
9
+ */
10
+ /** A single captured JSON API response. */
11
+ export interface CapturedApiResponse {
12
+ /** The absolute URL of the API endpoint. */
13
+ url: string;
14
+ /** The HTTP method used to request it. */
15
+ method: string;
16
+ /** The response content-type (e.g. `application/json`). */
17
+ contentType: string;
18
+ /** The parsed JSON body. */
19
+ data: unknown;
20
+ }
21
+ /** The minimal Playwright `Page` surface this module depends on. */
22
+ export interface CapturePage {
23
+ on(event: "response", listener: (response: ApiResponse) => void): unknown;
24
+ off(event: "response", listener: (response: ApiResponse) => void): unknown;
25
+ }
26
+ /** The minimal Playwright `Response` surface this module depends on. */
27
+ export interface ApiResponse {
28
+ url(): string;
29
+ request(): {
30
+ method(): string;
31
+ };
32
+ headers(): Record<string, string>;
33
+ json(): Promise<unknown>;
34
+ }
35
+ /**
36
+ * Install a JSON API capture listener on a page.
37
+ *
38
+ * Registers a `response` handler that, for every response whose content-type
39
+ * is `application/json`, reads the body and appends `{url, method,
40
+ * contentType, data}` to the returned array. The listener is removed when
41
+ * {@link CapturedApiResponses.remove} is called, so a page can be reused
42
+ * without leaking handlers.
43
+ *
44
+ * @param page The Playwright page to observe.
45
+ * @returns A handle exposing the captured responses and a `remove()` that
46
+ * detaches the listener.
47
+ */
48
+ export declare function installApiCapture(page: CapturePage): CapturedApiResponses;
49
+ /** A handle over the responses captured by {@link installApiCapture}. */
50
+ export interface CapturedApiResponses {
51
+ /** The responses captured so far, in arrival order. */
52
+ readonly captured: readonly CapturedApiResponse[];
53
+ /** Detach the capture listener from the page. */
54
+ remove(): void;
55
+ }
56
+ /**
57
+ * Normalize a captured API URL into an `api_patterns` row.
58
+ *
59
+ * The endpoint pattern replaces numeric path segments with `*` so that
60
+ * `/api/products/123` becomes `/api/products/*`. The domain is the URL's
61
+ * hostname. `content_type` is normalized to the bare media type (e.g.
62
+ * `application/json`).
63
+ */
64
+ export declare function discoverApiPatterns(domain: string, captured: readonly CapturedApiResponse[]): ApiPatternRow[];
65
+ /** An `api_patterns` table row (without the auto-increment id/timestamp). */
66
+ export interface ApiPatternRow {
67
+ domain: string;
68
+ endpoint_pattern: string;
69
+ method: string;
70
+ content_type: string;
71
+ }
72
+ //# sourceMappingURL=apiIntercept.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"apiIntercept.d.ts","sourceRoot":"","sources":["../../src/extraction/apiIntercept.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,2CAA2C;AAC3C,MAAM,WAAW,mBAAmB;IAClC,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,0CAA0C;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,IAAI,EAAE,OAAO,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC;IAC1E,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC;CAC5E;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B,GAAG,IAAI,MAAM,CAAC;IACd,OAAO,IAAI;QAAE,MAAM,IAAI,MAAM,CAAA;KAAE,CAAC;IAChC,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC1B;AAeD;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,WAAW,GAAG,oBAAoB,CAgCzE;AAED,yEAAyE;AACzE,MAAM,WAAW,oBAAoB;IACnC,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAClD,iDAAiD;IACjD,MAAM,IAAI,IAAI,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,SAAS,mBAAmB,EAAE,GACvC,aAAa,EAAE,CA0BjB;AAED,6EAA6E;AAC7E,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB"}