@allansantos-dev/smart-tool 0.9.0 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +50 -23
  2. package/NOTICE +32 -32
  3. package/README.md +117 -116
  4. package/atomic_io.py +94 -94
  5. package/bin/smart-tool.mjs +5 -1
  6. package/client_hooks.py +192 -192
  7. package/code_graph_java.mjs +49 -49
  8. package/code_graph_js.cjs +135 -135
  9. package/duplicates.py +313 -313
  10. package/embedding_cache.py +99 -99
  11. package/endpoint_sync.py +179 -179
  12. package/gateway.py +220 -220
  13. package/hook_decision.py +96 -96
  14. package/index_inventory.py +75 -75
  15. package/indexer.py +878 -878
  16. package/install.py +113 -11
  17. package/install_runtime.py +101 -35
  18. package/integration_metrics.py +82 -82
  19. package/local_embedder.py +172 -172
  20. package/model_defaults.py +70 -70
  21. package/package.json +35 -35
  22. package/project_insights_ui.py +38 -38
  23. package/project_monitor.py +204 -204
  24. package/project_store.py +220 -220
  25. package/projects_ui.py +146 -146
  26. package/requirements.txt +8 -8
  27. package/research_cache.py +171 -171
  28. package/research_metrics.py +110 -110
  29. package/router.py +582 -582
  30. package/secret_store.py +121 -121
  31. package/setup_ui.py +1011 -986
  32. package/smart_tool_daemon.py +3604 -3604
  33. package/version.py +1 -1
  34. package/web_adapters/browser/fetch_camoufox.py +69 -0
  35. package/web_adapters/node/ddg-search.mjs +89 -89
  36. package/web_adapters/node/open-websearch-adapter.mjs +77 -77
  37. package/web_adapters/node/package.json +12 -12
  38. package/web_adapters/node/wikipedia-search.mjs +44 -44
  39. package/web_search_adapters.py +193 -193
  40. package/web_search_external.py +288 -288
  41. package/web_search_health.py +88 -88
@@ -1,288 +1,288 @@
1
- #!/usr/bin/env python3
2
- """Adaptador HTTP dos provedores com índice próprio: plano gratuito sem chave, chave opcional do usuário.
3
-
4
- Mesmo padrão validado por Hermes Agent (plugins/web/keyless_mcp.py) e OpenCode (core/src/tool/websearch.ts):
5
- o endpoint público responde sem credencial; a chave do usuário entra como cabeçalho na mesma chamada.
6
- """
7
- import argparse
8
- import datetime
9
- import email.utils
10
- import json
11
- import math
12
- import re
13
- import sys
14
- import time
15
- import urllib.error
16
- import urllib.parse
17
- import urllib.request
18
- import uuid
19
-
20
- import config
21
-
22
- # Shown in the setup screen: where the key comes from and what it changes. Sources accessed 2026-10-03.
23
- PROVIDERS = {
24
- "exa": {"label": "Exa", "signup": "https://dashboard.exa.ai/api-keys", "key_prefix": "",
25
- "free": "No key: Exa public MCP, up to 3 searches/s and 150 calls per day (each bilingual search uses 2).",
26
- "keyed": "With key: your account limits and credits (new accounts start with free credits)."},
27
- "parallel": {"label": "Parallel", "signup": "https://platform.parallel.ai", "key_prefix": "",
28
- "free": "No key: Parallel public MCP; measured ~60 searches per hour before a 1 h pause.",
29
- "keyed": "With key: your account quota (600 searches/min on the default plan)."},
30
- "keenable": {"label": "Keenable", "signup": "https://app.keenable.ai/signup", "key_prefix": "keen_",
31
- "free": "No key: Keenable public endpoint (own index and ranking), rate limited.",
32
- "keyed": "With key: 100k free requests on signup, then US$ 4 per thousand."},
33
- "tavily": {"label": "Tavily", "signup": "https://app.tavily.com", "key_prefix": "tvly-",
34
- "free": "No key: Tavily keyless mode, rate limited.",
35
- "keyed": "With key: 1,000 free credits per month, no card."},
36
- "firecrawl": {"label": "Firecrawl", "signup": "https://www.firecrawl.dev/app/api-keys", "key_prefix": "fc-",
37
- "free": "No key: Firecrawl public search, rate limited.",
38
- "keyed": "With key: your account credits and limits."},
39
- }
40
- _RATE_LIMIT_MARKERS = ("rate limit", "rate-limit", "ratelimit", "too many requests", "quota exceeded", "slow down",
41
- "limit exceeded", "limit reached", "exhausted")
42
- _RATE_LIMIT_CODE = re.compile(r"(error|http|status|code)\W{0,3}429\b")
43
- _AUTH_MARKERS = ("invalid api key", "unauthorized", "authentication failed", "malformed api key", "invalid token")
44
- # Rate limits signalled inside an MCP result carry no Retry-After; pause briefly instead of the 5 min default.
45
- MCP_RATE_LIMIT_PAUSE_S = 60
46
- _SESSION_ID = uuid.uuid4().hex
47
-
48
-
49
- class ProviderRateLimited(Exception):
50
- """429 persistente; o daemon deve abrir o circuito deste provedor."""
51
-
52
- def __init__(self, retry_after):
53
- self.retry_after = min(3600, max(1, int(retry_after)))
54
- super().__init__("quota temporarily unavailable")
55
-
56
-
57
- class ProviderAuthError(Exception):
58
- """O provedor recusou a chave informada."""
59
-
60
-
61
- class ProviderError(Exception):
62
- """Resposta do provedor fora do contrato; `search` troca a chave por *** antes de propagar."""
63
-
64
-
65
- def _retry_after_seconds(headers):
66
- raw = (headers.get("Retry-After") or "").strip()
67
- if not raw:
68
- return None
69
- try:
70
- return max(0, math.ceil(float(raw)))
71
- except ValueError:
72
- try:
73
- instant = email.utils.parsedate_to_datetime(raw)
74
- return max(0, math.ceil((instant - datetime.datetime.now(datetime.timezone.utc)).total_seconds()))
75
- except (TypeError, ValueError, OverflowError):
76
- return None
77
-
78
-
79
- def _post(url, body, headers, timeout=7):
80
- request = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST", headers={
81
- "Content-Type": "application/json", "Accept": "application/json, text/event-stream",
82
- "User-Agent": "smart-tool", **headers})
83
- for attempt in range(2):
84
- try:
85
- with urllib.request.urlopen(request, timeout=timeout) as response:
86
- return response.read().decode("utf-8", "replace")
87
- except urllib.error.HTTPError as exc:
88
- if exc.code in (401, 403):
89
- exc.close()
90
- raise ProviderAuthError(f"HTTP {exc.code}") from None
91
- if exc.code != 429:
92
- raise
93
- retry_after = _retry_after_seconds(exc.headers)
94
- exc.close()
95
- # One short retry recovers transient limits; a long wait opens the daemon circuit instead.
96
- if attempt == 0 and (retry_after is None or retry_after <= 2):
97
- time.sleep(max(0.5, retry_after or 0))
98
- continue
99
- raise ProviderRateLimited(retry_after or 300) from None
100
-
101
-
102
- def _sse_payloads(body):
103
- events, current = [], []
104
- for line in re.split(r"\r\n|\r|\n", body):
105
- if line.startswith("data:"):
106
- current.append(line[5:].lstrip(" "))
107
- elif not line.strip() and current:
108
- events.append("\n".join(current))
109
- current = []
110
- if current:
111
- events.append("\n".join(current))
112
- return events
113
-
114
-
115
- def _mcp_text(url, tool, arguments, headers):
116
- body = _post(url, {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
117
- "params": {"name": tool, "arguments": arguments}}, headers)
118
- stripped = body.strip()
119
- for candidate in ([stripped] if stripped.startswith("{") else []) + _sse_payloads(body):
120
- try:
121
- data = json.loads(candidate)
122
- except ValueError:
123
- continue
124
- if not isinstance(data, dict):
125
- continue
126
- error, result = data.get("error"), data.get("result")
127
- result = result if isinstance(result, dict) else {}
128
- texts = [str(c.get("text", "")) for c in result.get("content") or [] if isinstance(c, dict)]
129
- if error or result.get("isError"):
130
- if error:
131
- message = str(error.get("message") or error) if isinstance(error, dict) else str(error)
132
- else:
133
- message = " ".join(texts)
134
- lowered = message.lower()
135
- if any(marker in lowered for marker in _RATE_LIMIT_MARKERS) or _RATE_LIMIT_CODE.search(lowered):
136
- raise ProviderRateLimited(MCP_RATE_LIMIT_PAUSE_S)
137
- if headers and any(marker in lowered for marker in _AUTH_MARKERS):
138
- raise ProviderAuthError("key rejected by the provider")
139
- raise ProviderError(message[:300] or "MCP tool failed")
140
- if any(texts):
141
- return next(t for t in texts if t)
142
- raise ProviderError("MCP response without text content")
143
-
144
-
145
- def _exa(query, num, key):
146
- text = _mcp_text("https://mcp.exa.ai/mcp", "web_search_exa", {"query": query, "numResults": num},
147
- {"x-api-key": key} if key else {})
148
- rows = []
149
- for block in text.split("\n---\n"):
150
- title = url = ""
151
- snippet, in_highlights = [], False
152
- for line in map(str.strip, block.splitlines()):
153
- if line.startswith("Title:"):
154
- title = "" if line[6:].strip() == "N/A" else line[6:].strip()
155
- elif line.startswith("URL:"):
156
- url = line[4:].strip()
157
- elif in_highlights and line and not line.startswith(("Title:", "URL:", "Published:", "Author:")):
158
- snippet.append(line)
159
- if line.startswith(("Title:", "URL:", "Highlights:", "Published:", "Author:")):
160
- in_highlights = line.startswith("Highlights:")
161
- if url:
162
- rows.append({"title": title, "url": url, "content": " ".join(snippet)})
163
- return rows
164
-
165
-
166
- def _parallel(query, num, key):
167
- text = _mcp_text("https://search.parallel.ai/mcp", "web_search",
168
- {"objective": query, "search_queries": [query], "session_id": _SESSION_ID},
169
- {"Authorization": "Bearer " + key} if key else {})
170
- try:
171
- results = json.loads(text).get("results") or []
172
- except (ValueError, AttributeError):
173
- raise ProviderError("Parallel returned results in an unexpected format.") from None
174
- return [{"title": r.get("title"), "url": r.get("url"), "content": " ".join(r.get("excerpts") or [])}
175
- for r in results if isinstance(r, dict)]
176
-
177
-
178
- def _keenable(query, num, key):
179
- if key:
180
- url, headers = "https://api.keenable.ai/v1/search", {"X-API-Key": key}
181
- else:
182
- url, headers = "https://api.keenable.ai/v1/search/public", {"X-Keenable-Title": "smart-tool"}
183
- try:
184
- return json.loads(_post(url, {"query": query, "max_results": num}, headers)).get("results") or []
185
- except (ValueError, AttributeError):
186
- raise ProviderError("Keenable returned results in an unexpected format.") from None
187
-
188
-
189
- def _tavily(query, num, key):
190
- headers = {"Authorization": "Bearer " + key} if key else {"X-Tavily-Access-Mode": "keyless"}
191
- payload = json.loads(_post("https://api.tavily.com/search",
192
- {"query": query, "max_results": num, "search_depth": "basic"}, headers, timeout=7))
193
- return payload.get("results") or []
194
-
195
-
196
- def _firecrawl(query, num, key):
197
- headers = {"Authorization": "Bearer " + key} if key else {}
198
- payload = json.loads(_post("https://api.firecrawl.dev/v2/search", {"query": query, "limit": num}, headers,
199
- timeout=7))
200
- if payload.get("success") is not True:
201
- raise ProviderError("Firecrawl did not confirm the search.")
202
- return (payload.get("data") or {}).get("web") or []
203
-
204
-
205
- _SEARCH = {"exa": _exa, "parallel": _parallel, "keenable": _keenable, "tavily": _tavily, "firecrawl": _firecrawl}
206
-
207
-
208
- def search(provider, query, num, key=None):
209
- if provider not in _SEARCH:
210
- raise ValueError("Unknown web provider.")
211
- key = config.get_provider_api_key(provider) if key is None else key
212
- try:
213
- rows = _SEARCH[provider](query, num, key)
214
- except ProviderAuthError:
215
- raise ProviderAuthError("key rejected by the provider") from None
216
- except (ProviderError, ValueError, KeyError, TypeError, AttributeError) as exc:
217
- message = f"{type(exc).__name__}: {exc}"[:300]
218
- raise ProviderError(message.replace(key, "***") if key else message) from None
219
- if not isinstance(rows, list):
220
- raise ProviderError("Provider returned results in an unexpected format.")
221
- results = []
222
- for row in rows:
223
- if not isinstance(row, dict):
224
- continue
225
- link = str(row.get("url") or "")
226
- parsed = urllib.parse.urlparse(link)
227
- if parsed.scheme not in ("http", "https") or not parsed.netloc:
228
- continue
229
- results.append({
230
- "title": str(row.get("title") or "")[:300],
231
- "url": link,
232
- "snippet": str(row.get("content") or row.get("description") or row.get("snippet") or "")[:600],
233
- })
234
- return results[:num]
235
-
236
-
237
- def validate_key(provider, key):
238
- """Real search with the given key: raises ProviderAuthError when the vendor rejects it.
239
-
240
- Measured 2026-10-03: all five vendors reject an invalid key (HTTP 401, or Exa's MCP 'error (401)')."""
241
- config.check_provider_key_format(key)
242
- prefix = PROVIDERS[provider]["key_prefix"]
243
- if prefix and not key.startswith(prefix):
244
- raise ProviderAuthError(f"{PROVIDERS[provider]['label']} keys start with {prefix}")
245
- return len(search(provider, "smart tool web search", 1, key=key))
246
-
247
-
248
- def parse_args(argv):
249
- # Manual parsing: the query is positional and may start with "-" (e.g. "-v flag in pytest").
250
- options, query_parts, i = {"--num": "8", "--provider": None}, [], 0
251
- while i < len(argv):
252
- if argv[i] in options and i + 1 < len(argv):
253
- options[argv[i]] = argv[i + 1]
254
- i += 2
255
- else:
256
- query_parts.append(argv[i])
257
- i += 1
258
- if not query_parts or options["--provider"] not in config.WEB_PROVIDER_NAMES:
259
- raise ValueError('Usage: python web_search_external.py "query" --provider '
260
- + "|".join(config.WEB_PROVIDER_NAMES) + " [--num 8]")
261
- return argparse.Namespace(query=" ".join(query_parts), num=int(options["--num"]), provider=options["--provider"])
262
-
263
-
264
- def main():
265
- args = parse_args(sys.argv[1:])
266
- try:
267
- results = search(args.provider, args.query, args.num)
268
- except ProviderRateLimited as exc:
269
- print(f"{args.provider}: HTTP 429 retry_after={exc.retry_after}", file=sys.stderr)
270
- return 75
271
- except (ProviderAuthError, ProviderError) as exc:
272
- print(f"{args.provider}: {type(exc).__name__}: {exc}", file=sys.stderr)
273
- return 1
274
- except urllib.error.HTTPError as exc:
275
- print(f"{args.provider}: HTTP {exc.code}", file=sys.stderr)
276
- return 1
277
- except Exception as exc:
278
- # The Authorization header never reaches tracebacks or remote error bodies.
279
- print(f"{args.provider}: {type(exc).__name__}", file=sys.stderr)
280
- return 1
281
- print(json.dumps(results, ensure_ascii=False))
282
- return 0
283
-
284
-
285
- if __name__ == "__main__":
286
- sys.stdout.reconfigure(encoding="utf-8")
287
- sys.stderr.reconfigure(encoding="utf-8")
288
- sys.exit(main())
1
+ #!/usr/bin/env python3
2
+ """Adaptador HTTP dos provedores com índice próprio: plano gratuito sem chave, chave opcional do usuário.
3
+
4
+ Mesmo padrão validado por Hermes Agent (plugins/web/keyless_mcp.py) e OpenCode (core/src/tool/websearch.ts):
5
+ o endpoint público responde sem credencial; a chave do usuário entra como cabeçalho na mesma chamada.
6
+ """
7
+ import argparse
8
+ import datetime
9
+ import email.utils
10
+ import json
11
+ import math
12
+ import re
13
+ import sys
14
+ import time
15
+ import urllib.error
16
+ import urllib.parse
17
+ import urllib.request
18
+ import uuid
19
+
20
+ import config
21
+
22
+ # Shown in the setup screen: where the key comes from and what it changes. Sources accessed 2026-10-03.
23
+ PROVIDERS = {
24
+ "exa": {"label": "Exa", "signup": "https://dashboard.exa.ai/api-keys", "key_prefix": "",
25
+ "free": "No key: Exa public MCP, up to 3 searches/s and 150 calls per day (each bilingual search uses 2).",
26
+ "keyed": "With key: your account limits and credits (new accounts start with free credits)."},
27
+ "parallel": {"label": "Parallel", "signup": "https://platform.parallel.ai", "key_prefix": "",
28
+ "free": "No key: Parallel public MCP; measured ~60 searches per hour before a 1 h pause.",
29
+ "keyed": "With key: your account quota (600 searches/min on the default plan)."},
30
+ "keenable": {"label": "Keenable", "signup": "https://app.keenable.ai/signup", "key_prefix": "keen_",
31
+ "free": "No key: Keenable public endpoint (own index and ranking), rate limited.",
32
+ "keyed": "With key: 100k free requests on signup, then US$ 4 per thousand."},
33
+ "tavily": {"label": "Tavily", "signup": "https://app.tavily.com", "key_prefix": "tvly-",
34
+ "free": "No key: Tavily keyless mode, rate limited.",
35
+ "keyed": "With key: 1,000 free credits per month, no card."},
36
+ "firecrawl": {"label": "Firecrawl", "signup": "https://www.firecrawl.dev/app/api-keys", "key_prefix": "fc-",
37
+ "free": "No key: Firecrawl public search, rate limited.",
38
+ "keyed": "With key: your account credits and limits."},
39
+ }
40
+ _RATE_LIMIT_MARKERS = ("rate limit", "rate-limit", "ratelimit", "too many requests", "quota exceeded", "slow down",
41
+ "limit exceeded", "limit reached", "exhausted")
42
+ _RATE_LIMIT_CODE = re.compile(r"(error|http|status|code)\W{0,3}429\b")
43
+ _AUTH_MARKERS = ("invalid api key", "unauthorized", "authentication failed", "malformed api key", "invalid token")
44
+ # Rate limits signalled inside an MCP result carry no Retry-After; pause briefly instead of the 5 min default.
45
+ MCP_RATE_LIMIT_PAUSE_S = 60
46
+ _SESSION_ID = uuid.uuid4().hex
47
+
48
+
49
+ class ProviderRateLimited(Exception):
50
+ """429 persistente; o daemon deve abrir o circuito deste provedor."""
51
+
52
+ def __init__(self, retry_after):
53
+ self.retry_after = min(3600, max(1, int(retry_after)))
54
+ super().__init__("quota temporarily unavailable")
55
+
56
+
57
+ class ProviderAuthError(Exception):
58
+ """O provedor recusou a chave informada."""
59
+
60
+
61
+ class ProviderError(Exception):
62
+ """Resposta do provedor fora do contrato; `search` troca a chave por *** antes de propagar."""
63
+
64
+
65
+ def _retry_after_seconds(headers):
66
+ raw = (headers.get("Retry-After") or "").strip()
67
+ if not raw:
68
+ return None
69
+ try:
70
+ return max(0, math.ceil(float(raw)))
71
+ except ValueError:
72
+ try:
73
+ instant = email.utils.parsedate_to_datetime(raw)
74
+ return max(0, math.ceil((instant - datetime.datetime.now(datetime.timezone.utc)).total_seconds()))
75
+ except (TypeError, ValueError, OverflowError):
76
+ return None
77
+
78
+
79
+ def _post(url, body, headers, timeout=7):
80
+ request = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST", headers={
81
+ "Content-Type": "application/json", "Accept": "application/json, text/event-stream",
82
+ "User-Agent": "smart-tool", **headers})
83
+ for attempt in range(2):
84
+ try:
85
+ with urllib.request.urlopen(request, timeout=timeout) as response:
86
+ return response.read().decode("utf-8", "replace")
87
+ except urllib.error.HTTPError as exc:
88
+ if exc.code in (401, 403):
89
+ exc.close()
90
+ raise ProviderAuthError(f"HTTP {exc.code}") from None
91
+ if exc.code != 429:
92
+ raise
93
+ retry_after = _retry_after_seconds(exc.headers)
94
+ exc.close()
95
+ # One short retry recovers transient limits; a long wait opens the daemon circuit instead.
96
+ if attempt == 0 and (retry_after is None or retry_after <= 2):
97
+ time.sleep(max(0.5, retry_after or 0))
98
+ continue
99
+ raise ProviderRateLimited(retry_after or 300) from None
100
+
101
+
102
+ def _sse_payloads(body):
103
+ events, current = [], []
104
+ for line in re.split(r"\r\n|\r|\n", body):
105
+ if line.startswith("data:"):
106
+ current.append(line[5:].lstrip(" "))
107
+ elif not line.strip() and current:
108
+ events.append("\n".join(current))
109
+ current = []
110
+ if current:
111
+ events.append("\n".join(current))
112
+ return events
113
+
114
+
115
+ def _mcp_text(url, tool, arguments, headers):
116
+ body = _post(url, {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
117
+ "params": {"name": tool, "arguments": arguments}}, headers)
118
+ stripped = body.strip()
119
+ for candidate in ([stripped] if stripped.startswith("{") else []) + _sse_payloads(body):
120
+ try:
121
+ data = json.loads(candidate)
122
+ except ValueError:
123
+ continue
124
+ if not isinstance(data, dict):
125
+ continue
126
+ error, result = data.get("error"), data.get("result")
127
+ result = result if isinstance(result, dict) else {}
128
+ texts = [str(c.get("text", "")) for c in result.get("content") or [] if isinstance(c, dict)]
129
+ if error or result.get("isError"):
130
+ if error:
131
+ message = str(error.get("message") or error) if isinstance(error, dict) else str(error)
132
+ else:
133
+ message = " ".join(texts)
134
+ lowered = message.lower()
135
+ if any(marker in lowered for marker in _RATE_LIMIT_MARKERS) or _RATE_LIMIT_CODE.search(lowered):
136
+ raise ProviderRateLimited(MCP_RATE_LIMIT_PAUSE_S)
137
+ if headers and any(marker in lowered for marker in _AUTH_MARKERS):
138
+ raise ProviderAuthError("key rejected by the provider")
139
+ raise ProviderError(message[:300] or "MCP tool failed")
140
+ if any(texts):
141
+ return next(t for t in texts if t)
142
+ raise ProviderError("MCP response without text content")
143
+
144
+
145
+ def _exa(query, num, key):
146
+ text = _mcp_text("https://mcp.exa.ai/mcp", "web_search_exa", {"query": query, "numResults": num},
147
+ {"x-api-key": key} if key else {})
148
+ rows = []
149
+ for block in text.split("\n---\n"):
150
+ title = url = ""
151
+ snippet, in_highlights = [], False
152
+ for line in map(str.strip, block.splitlines()):
153
+ if line.startswith("Title:"):
154
+ title = "" if line[6:].strip() == "N/A" else line[6:].strip()
155
+ elif line.startswith("URL:"):
156
+ url = line[4:].strip()
157
+ elif in_highlights and line and not line.startswith(("Title:", "URL:", "Published:", "Author:")):
158
+ snippet.append(line)
159
+ if line.startswith(("Title:", "URL:", "Highlights:", "Published:", "Author:")):
160
+ in_highlights = line.startswith("Highlights:")
161
+ if url:
162
+ rows.append({"title": title, "url": url, "content": " ".join(snippet)})
163
+ return rows
164
+
165
+
166
+ def _parallel(query, num, key):
167
+ text = _mcp_text("https://search.parallel.ai/mcp", "web_search",
168
+ {"objective": query, "search_queries": [query], "session_id": _SESSION_ID},
169
+ {"Authorization": "Bearer " + key} if key else {})
170
+ try:
171
+ results = json.loads(text).get("results") or []
172
+ except (ValueError, AttributeError):
173
+ raise ProviderError("Parallel returned results in an unexpected format.") from None
174
+ return [{"title": r.get("title"), "url": r.get("url"), "content": " ".join(r.get("excerpts") or [])}
175
+ for r in results if isinstance(r, dict)]
176
+
177
+
178
+ def _keenable(query, num, key):
179
+ if key:
180
+ url, headers = "https://api.keenable.ai/v1/search", {"X-API-Key": key}
181
+ else:
182
+ url, headers = "https://api.keenable.ai/v1/search/public", {"X-Keenable-Title": "smart-tool"}
183
+ try:
184
+ return json.loads(_post(url, {"query": query, "max_results": num}, headers)).get("results") or []
185
+ except (ValueError, AttributeError):
186
+ raise ProviderError("Keenable returned results in an unexpected format.") from None
187
+
188
+
189
+ def _tavily(query, num, key):
190
+ headers = {"Authorization": "Bearer " + key} if key else {"X-Tavily-Access-Mode": "keyless"}
191
+ payload = json.loads(_post("https://api.tavily.com/search",
192
+ {"query": query, "max_results": num, "search_depth": "basic"}, headers, timeout=7))
193
+ return payload.get("results") or []
194
+
195
+
196
+ def _firecrawl(query, num, key):
197
+ headers = {"Authorization": "Bearer " + key} if key else {}
198
+ payload = json.loads(_post("https://api.firecrawl.dev/v2/search", {"query": query, "limit": num}, headers,
199
+ timeout=7))
200
+ if payload.get("success") is not True:
201
+ raise ProviderError("Firecrawl did not confirm the search.")
202
+ return (payload.get("data") or {}).get("web") or []
203
+
204
+
205
+ _SEARCH = {"exa": _exa, "parallel": _parallel, "keenable": _keenable, "tavily": _tavily, "firecrawl": _firecrawl}
206
+
207
+
208
+ def search(provider, query, num, key=None):
209
+ if provider not in _SEARCH:
210
+ raise ValueError("Unknown web provider.")
211
+ key = config.get_provider_api_key(provider) if key is None else key
212
+ try:
213
+ rows = _SEARCH[provider](query, num, key)
214
+ except ProviderAuthError:
215
+ raise ProviderAuthError("key rejected by the provider") from None
216
+ except (ProviderError, ValueError, KeyError, TypeError, AttributeError) as exc:
217
+ message = f"{type(exc).__name__}: {exc}"[:300]
218
+ raise ProviderError(message.replace(key, "***") if key else message) from None
219
+ if not isinstance(rows, list):
220
+ raise ProviderError("Provider returned results in an unexpected format.")
221
+ results = []
222
+ for row in rows:
223
+ if not isinstance(row, dict):
224
+ continue
225
+ link = str(row.get("url") or "")
226
+ parsed = urllib.parse.urlparse(link)
227
+ if parsed.scheme not in ("http", "https") or not parsed.netloc:
228
+ continue
229
+ results.append({
230
+ "title": str(row.get("title") or "")[:300],
231
+ "url": link,
232
+ "snippet": str(row.get("content") or row.get("description") or row.get("snippet") or "")[:600],
233
+ })
234
+ return results[:num]
235
+
236
+
237
+ def validate_key(provider, key):
238
+ """Real search with the given key: raises ProviderAuthError when the vendor rejects it.
239
+
240
+ Measured 2026-10-03: all five vendors reject an invalid key (HTTP 401, or Exa's MCP 'error (401)')."""
241
+ config.check_provider_key_format(key)
242
+ prefix = PROVIDERS[provider]["key_prefix"]
243
+ if prefix and not key.startswith(prefix):
244
+ raise ProviderAuthError(f"{PROVIDERS[provider]['label']} keys start with {prefix}")
245
+ return len(search(provider, "smart tool web search", 1, key=key))
246
+
247
+
248
+ def parse_args(argv):
249
+ # Manual parsing: the query is positional and may start with "-" (e.g. "-v flag in pytest").
250
+ options, query_parts, i = {"--num": "8", "--provider": None}, [], 0
251
+ while i < len(argv):
252
+ if argv[i] in options and i + 1 < len(argv):
253
+ options[argv[i]] = argv[i + 1]
254
+ i += 2
255
+ else:
256
+ query_parts.append(argv[i])
257
+ i += 1
258
+ if not query_parts or options["--provider"] not in config.WEB_PROVIDER_NAMES:
259
+ raise ValueError('Usage: python web_search_external.py "query" --provider '
260
+ + "|".join(config.WEB_PROVIDER_NAMES) + " [--num 8]")
261
+ return argparse.Namespace(query=" ".join(query_parts), num=int(options["--num"]), provider=options["--provider"])
262
+
263
+
264
+ def main():
265
+ args = parse_args(sys.argv[1:])
266
+ try:
267
+ results = search(args.provider, args.query, args.num)
268
+ except ProviderRateLimited as exc:
269
+ print(f"{args.provider}: HTTP 429 retry_after={exc.retry_after}", file=sys.stderr)
270
+ return 75
271
+ except (ProviderAuthError, ProviderError) as exc:
272
+ print(f"{args.provider}: {type(exc).__name__}: {exc}", file=sys.stderr)
273
+ return 1
274
+ except urllib.error.HTTPError as exc:
275
+ print(f"{args.provider}: HTTP {exc.code}", file=sys.stderr)
276
+ return 1
277
+ except Exception as exc:
278
+ # The Authorization header never reaches tracebacks or remote error bodies.
279
+ print(f"{args.provider}: {type(exc).__name__}", file=sys.stderr)
280
+ return 1
281
+ print(json.dumps(results, ensure_ascii=False))
282
+ return 0
283
+
284
+
285
+ if __name__ == "__main__":
286
+ sys.stdout.reconfigure(encoding="utf-8")
287
+ sys.stderr.reconfigure(encoding="utf-8")
288
+ sys.exit(main())