@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,329 +1,329 @@
1
- """
2
- Web search tool for the agent harness, using DuckDuckGo exclusively via the
3
- `ddgs` package.
4
- """
5
-
6
- from __future__ import annotations
7
-
8
- import logging
9
- from typing import Optional
10
-
11
- from ddgs import DDGS
12
- from pydantic import BaseModel, Field, field_validator
13
-
14
- from safety.policy_enforcement import is_url_permitted
15
- from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
16
-
17
- logger = logging.getLogger(__name__)
18
-
19
-
20
- # ---- Schema exposed to the LLM -------------------------------------------
21
-
22
- class WebSearchParams(BaseModel):
23
- query: str = Field(..., description="Search query", min_length=1, max_length=400)
24
- num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
25
- page: int = Field(
26
- 1,
27
- ge=1,
28
- description=(
29
- "Which page of results to return (1-based). Use the page named "
30
- "in a result's message to see further results for the same "
31
- "query."
32
- ),
33
- )
34
-
35
- @field_validator("query")
36
- @classmethod
37
- def strip_query(cls, value: str) -> str:
38
- value = value.strip()
39
- if not value:
40
- raise ValueError("query cannot be empty")
41
- return value
42
-
43
-
44
- TOOL_SCHEMA = {
45
- "name": "web_search",
46
- "description": (
47
- "Search the web for current information. Use this for facts that may "
48
- "have changed recently, current events, or anything you're not "
49
- "fully confident about."
50
- ),
51
- "input_schema": WebSearchParams.model_json_schema(),
52
- }
53
-
54
-
55
- # ---- Normalized result shape ---------------------------------------------
56
-
57
- class SearchResult(BaseModel):
58
- title: str
59
- url: str
60
- snippet: str
61
- published: Optional[str] = None # ddgs's text search doesn't return a
62
- # publish date, so this stays None here
63
- # ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
64
- # where the rest is. Before §51 a cut snippet and a short one were
65
- # indistinguishable, so "the source does not mention X" and "the first
66
- # 300 characters do not mention X" read the same to the model.
67
- snippet_truncated: bool = False
68
- # Batch 156 (PG8's shape, one field over). The snippet had a cap and a
69
- # flag and the title beside it had neither, so a 5,000-character title
70
- # went to the model whole while the snippet under it was cut to 300 --
71
- # measured, in the same result. FLAG AND NO ROUTE, which is the
72
- # asymmetry `arxiv.py` already argues for the same field: a title past
73
- # the cap is a malformed result rather than a long one, and the route
74
- # to the page is the `url` sitting next to it either way.
75
- title_truncated: bool = False
76
-
77
-
78
- class WebSearchError(Exception):
79
- """No longer raised by this module -- exhausted retries return an
80
- error dict instead, so a transient search outage cannot fail a
81
- ten-pass research run. Kept as part of this module's public surface."""
82
-
83
-
84
- # ---- Policy / tuning knobs -------------------------------------------------
85
-
86
- MAX_SNIPPET_CHARS = 300
87
-
88
- # Batch 156. 300 is the number `arxiv.py` settled for the same kind of
89
- # value, measured there over sixty real entries whose longest title was
90
- # 188 characters. NOT imported from it: two constants with one reason each
91
- # is the choice `_MAX_AUTHORS` already argues for one file over, and these
92
- # two bound titles from two different providers.
93
- #
94
- # No equivalent measurement was taken for web titles in this batch, and
95
- # saying so is the honest version. What justifies the number here is that
96
- # it is the one already settled for a title, and a title is the same kind
97
- # of value whichever provider hands it over; the HTML spec puts no bound on
98
- # `<title>` at all.
99
- MAX_TITLE_CHARS = 300
100
-
101
- REQUEST_TIMEOUT_S = 8.0
102
- MAX_RETRIES = 2
103
- CACHE_TTL_S = 300
104
-
105
- # One instance per tool, never shared: an equal key string from an
106
- # arXiv query and a web query would otherwise collide.
107
- _cache = TTLCache(CACHE_TTL_S)
108
-
109
-
110
- # ---- Provider call ----------------------------------------------------------
111
-
112
- def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
113
- """
114
- The one function that actually talks to DuckDuckGo. DDGS() is
115
- instantiated fresh per call (cheap) and used as a context manager so
116
- its underlying HTTP session gets cleaned up automatically afterward.
117
-
118
- `page` is ROADMAP_v3 §51 (PG3's sibling). The record assumed this tool
119
- had no paging to expose and only a snippet cap to fix; MEASURED (batch
120
- 105), `ddgs 9.14.4` takes `page: int = 1` on its search path and
121
- accepted it live. So the provider implemented paging here too, and this
122
- tool was spending the parameter on a default.
123
-
124
- ALSO MEASURED, and the reason the message this feeds is worded the way
125
- it is: page 2 of a live query OVERLAPPED page 1 by one result. DuckDuckGo's
126
- paging is not a clean partition the way arXiv's `start` is (measured the
127
- same day: zero overlap), so a page here is "more results", not "the next
128
- five distinct results", and nothing downstream may assume otherwise.
129
- """
130
- with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
131
- return ddgs_client.text(query, max_results=num_results, page=page)
132
-
133
-
134
- def _normalize(raw: list[dict]) -> list[SearchResult]:
135
- """DDGS's rows as `SearchResult`s, redacted and bounded (§73).
136
-
137
- EVERY STRING IN HERE WAS WRITTEN BY SOMEBODY ELSE, which is the rule
138
- PG8/#131 states and the reason all of it goes through one helper now.
139
- Two things were wrong at once and they interact, so fixing either alone
140
- would have been worse than fixing neither:
141
-
142
- REDACT ABOVE THE CUT (RB1, A59). The snippet was cut to 300 characters
143
- and `registry.dispatch` redacted the piece that was kept. Measured
144
- (batch 156), a `ghp_` token straddling that cut survived post-hoc
145
- redaction at 36 of the 39 offsets it could straddle -- the kept prefix
146
- is not the token, the dropped suffix is gone, and `param_digest`'s
147
- docstring had already written down why ("truncating first cuts a
148
- credential below the 20 characters its pattern needs").
149
-
150
- AND THE CUTS THAT DID NOT EXIST YET. The title had no bound at all, so
151
- adding one without the ordering above would have MANUFACTURED a second
152
- straddle site in the field being repaired. Every new cap here is a new
153
- cut, and every cut needs the redaction above it.
154
-
155
- TOTAL OVER WHATEVER DDGS RETURNED (A61), which is the half no report
156
- named. `r.get("title", "(untitled)")` substitutes only for an ABSENT
157
- key, and `{"title": null}` is a present one -- the lesson
158
- `tools/base.py` states for tool arguments, arriving from the provider
159
- end. Measured, a single row with `title: null` raised `ValidationError`
160
- out of here, which `run` caught as a failed attempt, RETRIED TWICE,
161
- and then reported as "search failed after 3 attempts" -- ten good
162
- results lost to one malformed one, plus two pointless re-queries.
163
- `arxiv._parse_atom_feed` had the identical defect and fixed it in batch
164
- 117 ("SKIPPED, NOT RAISED ... Raising here aborted the whole search
165
- over one malformed entry"); the cure was one file over the whole time.
166
- """
167
- third = ThirdPartyText("web_search")
168
- out: list[SearchResult] = []
169
- for r in raw:
170
- if not isinstance(r, dict):
171
- logger.warning(
172
- "web_search skipped a result: the provider returned a %s "
173
- "where a row was expected.", type(r).__name__)
174
- continue
175
-
176
- url, withheld = third.url(r.get("href"))
177
- if withheld:
178
- # NOT SILENT (#49). The length is the fact; the URL is not
179
- # echoed, because a URL is one of the things the redactor is
180
- # looking for userinfo inside.
181
- logger.warning(
182
- "web_search dropped a result: its URL is longer than the "
183
- "%d characters fetch_url accepts, so it is a route this "
184
- "harness would refuse to take.", MAX_URL_CHARS)
185
- if url is None:
186
- # A result is a (title, url, snippet) triple whose url is how
187
- # the model reaches it. With the url gone there is nothing to
188
- # cite and nothing to fetch, so this is the same call the
189
- # policy filter below already makes -- and it is the call
190
- # `_parse_atom_feed` makes for an entry with no id.
191
- continue
192
-
193
- # `resolve=False` on purpose. This filters results; it does not
194
- # fetch them, and a DNS lookup per hit is real latency for a check
195
- # whose only failure mode is showing the model a URL that fetch_url
196
- # will refuse anyway. An IP LITERAL is still checked, in every
197
- # spelling the platform's resolver reads as a number -- a result
198
- # pointing straight at 169.254.169.254 is dropped here rather than
199
- # offered and then denied (#54).
200
- #
201
- # "EVERY SPELLING" SINCE BATCH 213 (#207, register item 53). Batch
202
- # 156 narrowed this comment to dotted and bracketed literals,
203
- # because `http://2130706433/`, `http://0x7f.0.0.1/` and
204
- # `http://0x7F000001/` -- all 127.0.0.1 -- passed: `ipaddress`
205
- # parses none of them. is_url_permitted now asks the resolver with
206
- # AI_NUMERICHOST, which reads numbers and looks nothing up, so they
207
- # are dropped here as fetch_url would refuse them.
208
- #
209
- # ABOVE THE POLICY CHECK AND BELOW THE BOUND, deliberately: the
210
- # string checked here is the string the model is offered, so a
211
- # redaction that rewrote the host cannot slip past the check that
212
- # was run on the host before it.
213
- if is_url_permitted(url, resolve=False):
214
- continue
215
-
216
- title, title_truncated = third.text(r.get("title"), MAX_TITLE_CHARS)
217
- if title is None:
218
- title = "(untitled)"
219
- snippet, snippet_truncated = third.text(
220
- r.get("body"), MAX_SNIPPET_CHARS)
221
- if snippet is None:
222
- snippet = ""
223
-
224
- out.append(
225
- SearchResult(
226
- title=title,
227
- url=url,
228
- snippet=snippet,
229
- published=None,
230
- snippet_truncated=snippet_truncated,
231
- title_truncated=title_truncated,
232
- )
233
- )
234
- third.report()
235
- return out
236
-
237
-
238
- def _kept(results: list, asked: int) -> list:
239
- """`results`, cut to the number the caller asked for (A62).
240
-
241
- THE SCHEMA ALREADY PROMISED THIS and nothing enforced it.
242
- `num_results` is advertised `le=10` and passed to the provider as
243
- `max_results`, and a provider is free to ignore it -- measured (batch
244
- 156), a call asking for 5 returned every one of 5,000 rows it was
245
- handed. So this is a documented bound being kept rather than a new
246
- policy, and it is the LIST axis of the same question batch 156 asked
247
- of each value: per-value caps multiplied by an unbounded row count are
248
- not a bound.
249
-
250
- `_call_ddgs` is where the finding wanted a byte ceiling, and it cannot
251
- have one: the library buffers the response and owns the socket, so
252
- nothing above it sees bytes. What this layer CAN bound is what it
253
- keeps, which is the quantity that reaches the model.
254
-
255
- NO FLAG, deliberately, where every other cut in this batch gets one.
256
- Nothing is being withheld from the caller: it asked for `asked`
257
- results and it is getting `asked` results. The over-delivery is a
258
- provider fact rather than a truncation, so it goes to the log, where
259
- the person debugging a provider can see it.
260
- """
261
- if len(results) > asked:
262
- logger.warning(
263
- "web_search kept %d of %d results: the provider returned more "
264
- "than the %d asked for.", asked, len(results), asked)
265
- return results[:asked]
266
-
267
-
268
- def run(params: dict) -> dict:
269
- parsed = WebSearchParams(**params)
270
- # §51 (PG9). `page` is part of the key for the reason `start` is on the
271
- # arXiv side: it changes the response, so a key without it serves page 1
272
- # to every request for page 2 and does it silently.
273
- cache_key = f"{parsed.query}|{parsed.num_results}|{parsed.page}"
274
-
275
- results = _cache.get(cache_key)
276
- if results is not None:
277
- logger.info("web_search cache hit for %r", parsed.query)
278
- else:
279
- last_exc: Optional[Exception] = None
280
- results = []
281
- for attempt in range(MAX_RETRIES + 1):
282
- try:
283
- raw = _call_ddgs(parsed.query, parsed.num_results, parsed.page)
284
- results = _kept(_normalize(raw), parsed.num_results)
285
- _cache.set(cache_key, results)
286
- break
287
- except Exception as e: # Simpler to try again then elaborate on every error type
288
- last_exc = e
289
- logger.warning(
290
- "web_search attempt %s/%s failed for %r: %s",
291
- attempt + 1, MAX_RETRIES + 1, parsed.query, e)
292
- else:
293
- # RETURNED, not raised -- same change and same reason as
294
- # arxiv_search. A search provider being unreachable is a
295
- # result the model can work around; raising made one
296
- # transient failure in one pass fail a whole ten-pass
297
- # research run. fetch_url was already the odd one out for
298
- # doing this correctly.
299
- return {
300
- "error": f"search failed after {MAX_RETRIES + 1} "
301
- f"attempts: {last_exc}"
302
- }
303
-
304
- if not results:
305
- if parsed.page > 1:
306
- return {
307
- "results": [], "result_count": 0, "page": parsed.page,
308
- "message": (
309
- f"No results on page {parsed.page}. The provider "
310
- f"returned nothing further for this query."
311
- ),
312
- }
313
- return {"results": [], "result_count": 0, "message": "No results found."}
314
-
315
- out = {
316
- "results": [r.model_dump() for r in results],
317
- "result_count": len(results),
318
- "page": parsed.page,
319
- }
320
- # PG4. DuckDuckGo reports no total, so the honest message names the next
321
- # page without claiming how many there are -- and says the pages overlap,
322
- # because measured they do. A full page is the only signal that more may
323
- # exist; a short one is where the provider stopped.
324
- if len(results) >= parsed.num_results:
325
- out["message"] = (
326
- f"Page {parsed.page}. Use page={parsed.page + 1} for more "
327
- f"results; pages may overlap by a result or two."
328
- )
1
+ """
2
+ Web search tool for the agent harness, using DuckDuckGo exclusively via the
3
+ `ddgs` package.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import logging
9
+ from typing import Optional
10
+
11
+ from ddgs import DDGS
12
+ from pydantic import BaseModel, Field, field_validator
13
+
14
+ from safety.policy_enforcement import is_url_permitted
15
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ # ---- Schema exposed to the LLM -------------------------------------------
21
+
22
+ class WebSearchParams(BaseModel):
23
+ query: str = Field(..., description="Search query", min_length=1, max_length=400)
24
+ num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
25
+ page: int = Field(
26
+ 1,
27
+ ge=1,
28
+ description=(
29
+ "Which page of results to return (1-based). Use the page named "
30
+ "in a result's message to see further results for the same "
31
+ "query."
32
+ ),
33
+ )
34
+
35
+ @field_validator("query")
36
+ @classmethod
37
+ def strip_query(cls, value: str) -> str:
38
+ value = value.strip()
39
+ if not value:
40
+ raise ValueError("query cannot be empty")
41
+ return value
42
+
43
+
44
+ TOOL_SCHEMA = {
45
+ "name": "web_search",
46
+ "description": (
47
+ "Search the web for current information. Use this for facts that may "
48
+ "have changed recently, current events, or anything you're not "
49
+ "fully confident about."
50
+ ),
51
+ "input_schema": WebSearchParams.model_json_schema(),
52
+ }
53
+
54
+
55
+ # ---- Normalized result shape ---------------------------------------------
56
+
57
+ class SearchResult(BaseModel):
58
+ title: str
59
+ url: str
60
+ snippet: str
61
+ published: Optional[str] = None # ddgs's text search doesn't return a
62
+ # publish date, so this stays None here
63
+ # ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
64
+ # where the rest is. Before §51 a cut snippet and a short one were
65
+ # indistinguishable, so "the source does not mention X" and "the first
66
+ # 300 characters do not mention X" read the same to the model.
67
+ snippet_truncated: bool = False
68
+ # Batch 156 (PG8's shape, one field over). The snippet had a cap and a
69
+ # flag and the title beside it had neither, so a 5,000-character title
70
+ # went to the model whole while the snippet under it was cut to 300 --
71
+ # measured, in the same result. FLAG AND NO ROUTE, which is the
72
+ # asymmetry `arxiv.py` already argues for the same field: a title past
73
+ # the cap is a malformed result rather than a long one, and the route
74
+ # to the page is the `url` sitting next to it either way.
75
+ title_truncated: bool = False
76
+
77
+
78
+ class WebSearchError(Exception):
79
+ """No longer raised by this module -- exhausted retries return an
80
+ error dict instead, so a transient search outage cannot fail a
81
+ ten-pass research run. Kept as part of this module's public surface."""
82
+
83
+
84
+ # ---- Policy / tuning knobs -------------------------------------------------
85
+
86
+ MAX_SNIPPET_CHARS = 300
87
+
88
+ # Batch 156. 300 is the number `arxiv.py` settled for the same kind of
89
+ # value, measured there over sixty real entries whose longest title was
90
+ # 188 characters. NOT imported from it: two constants with one reason each
91
+ # is the choice `_MAX_AUTHORS` already argues for one file over, and these
92
+ # two bound titles from two different providers.
93
+ #
94
+ # No equivalent measurement was taken for web titles in this batch, and
95
+ # saying so is the honest version. What justifies the number here is that
96
+ # it is the one already settled for a title, and a title is the same kind
97
+ # of value whichever provider hands it over; the HTML spec puts no bound on
98
+ # `<title>` at all.
99
+ MAX_TITLE_CHARS = 300
100
+
101
+ REQUEST_TIMEOUT_S = 8.0
102
+ MAX_RETRIES = 2
103
+ CACHE_TTL_S = 300
104
+
105
+ # One instance per tool, never shared: an equal key string from an
106
+ # arXiv query and a web query would otherwise collide.
107
+ _cache = TTLCache(CACHE_TTL_S)
108
+
109
+
110
+ # ---- Provider call ----------------------------------------------------------
111
+
112
+ def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
113
+ """
114
+ The one function that actually talks to DuckDuckGo. DDGS() is
115
+ instantiated fresh per call (cheap) and used as a context manager so
116
+ its underlying HTTP session gets cleaned up automatically afterward.
117
+
118
+ `page` is ROADMAP_v3 §51 (PG3's sibling). The record assumed this tool
119
+ had no paging to expose and only a snippet cap to fix; MEASURED (batch
120
+ 105), `ddgs 9.14.4` takes `page: int = 1` on its search path and
121
+ accepted it live. So the provider implemented paging here too, and this
122
+ tool was spending the parameter on a default.
123
+
124
+ ALSO MEASURED, and the reason the message this feeds is worded the way
125
+ it is: page 2 of a live query OVERLAPPED page 1 by one result. DuckDuckGo's
126
+ paging is not a clean partition the way arXiv's `start` is (measured the
127
+ same day: zero overlap), so a page here is "more results", not "the next
128
+ five distinct results", and nothing downstream may assume otherwise.
129
+ """
130
+ with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
131
+ return ddgs_client.text(query, max_results=num_results, page=page)
132
+
133
+
134
+ def _normalize(raw: list[dict]) -> list[SearchResult]:
135
+ """DDGS's rows as `SearchResult`s, redacted and bounded (§73).
136
+
137
+ EVERY STRING IN HERE WAS WRITTEN BY SOMEBODY ELSE, which is the rule
138
+ PG8/#131 states and the reason all of it goes through one helper now.
139
+ Two things were wrong at once and they interact, so fixing either alone
140
+ would have been worse than fixing neither:
141
+
142
+ REDACT ABOVE THE CUT (RB1, A59). The snippet was cut to 300 characters
143
+ and `registry.dispatch` redacted the piece that was kept. Measured
144
+ (batch 156), a `ghp_` token straddling that cut survived post-hoc
145
+ redaction at 36 of the 39 offsets it could straddle -- the kept prefix
146
+ is not the token, the dropped suffix is gone, and `param_digest`'s
147
+ docstring had already written down why ("truncating first cuts a
148
+ credential below the 20 characters its pattern needs").
149
+
150
+ AND THE CUTS THAT DID NOT EXIST YET. The title had no bound at all, so
151
+ adding one without the ordering above would have MANUFACTURED a second
152
+ straddle site in the field being repaired. Every new cap here is a new
153
+ cut, and every cut needs the redaction above it.
154
+
155
+ TOTAL OVER WHATEVER DDGS RETURNED (A61), which is the half no report
156
+ named. `r.get("title", "(untitled)")` substitutes only for an ABSENT
157
+ key, and `{"title": null}` is a present one -- the lesson
158
+ `tools/base.py` states for tool arguments, arriving from the provider
159
+ end. Measured, a single row with `title: null` raised `ValidationError`
160
+ out of here, which `run` caught as a failed attempt, RETRIED TWICE,
161
+ and then reported as "search failed after 3 attempts" -- ten good
162
+ results lost to one malformed one, plus two pointless re-queries.
163
+ `arxiv._parse_atom_feed` had the identical defect and fixed it in batch
164
+ 117 ("SKIPPED, NOT RAISED ... Raising here aborted the whole search
165
+ over one malformed entry"); the cure was one file over the whole time.
166
+ """
167
+ third = ThirdPartyText("web_search")
168
+ out: list[SearchResult] = []
169
+ for r in raw:
170
+ if not isinstance(r, dict):
171
+ logger.warning(
172
+ "web_search skipped a result: the provider returned a %s "
173
+ "where a row was expected.", type(r).__name__)
174
+ continue
175
+
176
+ url, withheld = third.url(r.get("href"))
177
+ if withheld:
178
+ # NOT SILENT (#49). The length is the fact; the URL is not
179
+ # echoed, because a URL is one of the things the redactor is
180
+ # looking for userinfo inside.
181
+ logger.warning(
182
+ "web_search dropped a result: its URL is longer than the "
183
+ "%d characters fetch_url accepts, so it is a route this "
184
+ "harness would refuse to take.", MAX_URL_CHARS)
185
+ if url is None:
186
+ # A result is a (title, url, snippet) triple whose url is how
187
+ # the model reaches it. With the url gone there is nothing to
188
+ # cite and nothing to fetch, so this is the same call the
189
+ # policy filter below already makes -- and it is the call
190
+ # `_parse_atom_feed` makes for an entry with no id.
191
+ continue
192
+
193
+ # `resolve=False` on purpose. This filters results; it does not
194
+ # fetch them, and a DNS lookup per hit is real latency for a check
195
+ # whose only failure mode is showing the model a URL that fetch_url
196
+ # will refuse anyway. An IP LITERAL is still checked, in every
197
+ # spelling the platform's resolver reads as a number -- a result
198
+ # pointing straight at 169.254.169.254 is dropped here rather than
199
+ # offered and then denied (#54).
200
+ #
201
+ # "EVERY SPELLING" SINCE BATCH 213 (#207, register item 53). Batch
202
+ # 156 narrowed this comment to dotted and bracketed literals,
203
+ # because `http://2130706433/`, `http://0x7f.0.0.1/` and
204
+ # `http://0x7F000001/` -- all 127.0.0.1 -- passed: `ipaddress`
205
+ # parses none of them. is_url_permitted now asks the resolver with
206
+ # AI_NUMERICHOST, which reads numbers and looks nothing up, so they
207
+ # are dropped here as fetch_url would refuse them.
208
+ #
209
+ # ABOVE THE POLICY CHECK AND BELOW THE BOUND, deliberately: the
210
+ # string checked here is the string the model is offered, so a
211
+ # redaction that rewrote the host cannot slip past the check that
212
+ # was run on the host before it.
213
+ if is_url_permitted(url, resolve=False):
214
+ continue
215
+
216
+ title, title_truncated = third.text(r.get("title"), MAX_TITLE_CHARS)
217
+ if title is None:
218
+ title = "(untitled)"
219
+ snippet, snippet_truncated = third.text(
220
+ r.get("body"), MAX_SNIPPET_CHARS)
221
+ if snippet is None:
222
+ snippet = ""
223
+
224
+ out.append(
225
+ SearchResult(
226
+ title=title,
227
+ url=url,
228
+ snippet=snippet,
229
+ published=None,
230
+ snippet_truncated=snippet_truncated,
231
+ title_truncated=title_truncated,
232
+ )
233
+ )
234
+ third.report()
235
+ return out
236
+
237
+
238
+ def _kept(results: list, asked: int) -> list:
239
+ """`results`, cut to the number the caller asked for (A62).
240
+
241
+ THE SCHEMA ALREADY PROMISED THIS and nothing enforced it.
242
+ `num_results` is advertised `le=10` and passed to the provider as
243
+ `max_results`, and a provider is free to ignore it -- measured (batch
244
+ 156), a call asking for 5 returned every one of 5,000 rows it was
245
+ handed. So this is a documented bound being kept rather than a new
246
+ policy, and it is the LIST axis of the same question batch 156 asked
247
+ of each value: per-value caps multiplied by an unbounded row count are
248
+ not a bound.
249
+
250
+ `_call_ddgs` is where the finding wanted a byte ceiling, and it cannot
251
+ have one: the library buffers the response and owns the socket, so
252
+ nothing above it sees bytes. What this layer CAN bound is what it
253
+ keeps, which is the quantity that reaches the model.
254
+
255
+ NO FLAG, deliberately, where every other cut in this batch gets one.
256
+ Nothing is being withheld from the caller: it asked for `asked`
257
+ results and it is getting `asked` results. The over-delivery is a
258
+ provider fact rather than a truncation, so it goes to the log, where
259
+ the person debugging a provider can see it.
260
+ """
261
+ if len(results) > asked:
262
+ logger.warning(
263
+ "web_search kept %d of %d results: the provider returned more "
264
+ "than the %d asked for.", asked, len(results), asked)
265
+ return results[:asked]
266
+
267
+
268
+ def run(params: dict) -> dict:
269
+ parsed = WebSearchParams(**params)
270
+ # §51 (PG9). `page` is part of the key for the reason `start` is on the
271
+ # arXiv side: it changes the response, so a key without it serves page 1
272
+ # to every request for page 2 and does it silently.
273
+ cache_key = f"{parsed.query}|{parsed.num_results}|{parsed.page}"
274
+
275
+ results = _cache.get(cache_key)
276
+ if results is not None:
277
+ logger.info("web_search cache hit for %r", parsed.query)
278
+ else:
279
+ last_exc: Optional[Exception] = None
280
+ results = []
281
+ for attempt in range(MAX_RETRIES + 1):
282
+ try:
283
+ raw = _call_ddgs(parsed.query, parsed.num_results, parsed.page)
284
+ results = _kept(_normalize(raw), parsed.num_results)
285
+ _cache.set(cache_key, results)
286
+ break
287
+ except Exception as e: # Simpler to try again then elaborate on every error type
288
+ last_exc = e
289
+ logger.warning(
290
+ "web_search attempt %s/%s failed for %r: %s",
291
+ attempt + 1, MAX_RETRIES + 1, parsed.query, e)
292
+ else:
293
+ # RETURNED, not raised -- same change and same reason as
294
+ # arxiv_search. A search provider being unreachable is a
295
+ # result the model can work around; raising made one
296
+ # transient failure in one pass fail a whole ten-pass
297
+ # research run. fetch_url was already the odd one out for
298
+ # doing this correctly.
299
+ return {
300
+ "error": f"search failed after {MAX_RETRIES + 1} "
301
+ f"attempts: {last_exc}"
302
+ }
303
+
304
+ if not results:
305
+ if parsed.page > 1:
306
+ return {
307
+ "results": [], "result_count": 0, "page": parsed.page,
308
+ "message": (
309
+ f"No results on page {parsed.page}. The provider "
310
+ f"returned nothing further for this query."
311
+ ),
312
+ }
313
+ return {"results": [], "result_count": 0, "message": "No results found."}
314
+
315
+ out = {
316
+ "results": [r.model_dump() for r in results],
317
+ "result_count": len(results),
318
+ "page": parsed.page,
319
+ }
320
+ # PG4. DuckDuckGo reports no total, so the honest message names the next
321
+ # page without claiming how many there are -- and says the pages overlap,
322
+ # because measured they do. A full page is the only signal that more may
323
+ # exist; a short one is where the provider stopped.
324
+ if len(results) >= parsed.num_results:
325
+ out["message"] = (
326
+ f"Page {parsed.page}. Use page={parsed.page + 1} for more "
327
+ f"results; pages may overlap by a result or two."
328
+ )
329
329
  return out