@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -15,7 +15,13 @@ from typing import Literal, Optional
15
15
  import sympy
16
16
  from pydantic import BaseModel, Field
17
17
 
18
- from tools.builtin._math_common import safe_parse, serialize, MathParseError
18
+ from tools.builtin._math_common import (
19
+ MathParseError,
20
+ _split_warnings,
21
+ parse_params,
22
+ safe_parse,
23
+ serialize,
24
+ )
19
25
 
20
26
  _OPERATIONS = Literal[
21
27
  "evaluate", "simplify", "expand", "factor",
@@ -32,7 +38,42 @@ class SymbolicMathParams(BaseModel):
32
38
  "'x**2 + 3*x - 4', 'sin(x)*cos(x)', or 'x**2 - 4 = 0' for solve."
33
39
  ),
34
40
  )
35
- variable: str = Field("x", description="The variable to solve for / differentiate / integrate / expand around")
41
+ variable: str = Field(
42
+ "x",
43
+ pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
44
+ description=(
45
+ "The ONE variable to solve for / differentiate / integrate / "
46
+ "expand around. A single name, not a list: for a system, put "
47
+ "the extra equations in `equations` and the solver takes its "
48
+ "symbols from the system itself."
49
+ ),
50
+ )
51
+ # `sympy.Symbol` accepts any string, so an unvalidated value
52
+ # became a symbol that CANNOT occur in the parsed expression, and every
53
+ # operation taking a variable then had a well-defined wrong answer:
54
+ # `solve('x**2 - 4 = 0', variable='x, y')` returned `[]` and
55
+ # `differentiate('x**3', variable='x, y')` returned `'0'`, neither
56
+ # distinguishable in the result from a correct empty solution set or a
57
+ # correct zero derivative. The schema is where its neighbours are
58
+ # already validated (`order` ge=1, `precision` le=50), and the trigger
59
+ # was the schema's own invitation: the sibling field `equations` IS a
60
+ # list, and the solve-a-system path ignores `variable` entirely -- so
61
+ # the one operation where a caller has a reason to name two variables
62
+ # is the one where this field is unused (batch 143).
63
+ #
64
+ # `pattern` is a field the pinned `google-genai` `types.Schema`
65
+ # carries, checked rather than assumed, so this does not widen the
66
+ # GOOGLE-branch problem that is still open.
67
+ symbols: Optional[list[str]] = Field(
68
+ None,
69
+ description=(
70
+ "Other multi-character symbol names in the expression, e.g. "
71
+ "['force', 'mass']. `variable` is always protected; any OTHER "
72
+ "multi-character name is read as a product of single letters "
73
+ "unless listed here ('mass' becomes a*m*s**2). Single letters "
74
+ "and names containing an underscore are safe without this."
75
+ ),
76
+ )
36
77
  equations: Optional[list[str]] = Field(
37
78
  None, description="For 'solve' with a system: additional equations alongside `expression`"
38
79
  )
@@ -59,11 +100,23 @@ TOOL_SCHEMA = {
59
100
 
60
101
 
61
102
  def run(params: dict) -> dict:
62
- parsed = SymbolicMathParams(**params)
103
+ try:
104
+ parsed = parse_params(SymbolicMathParams, params)
105
+ except MathParseError as e:
106
+ return {"error": str(e)}
107
+ # `variable` PLUS whatever the caller declared, on EVERY parsed field.
108
+ # Before batch 131 this tool passed `symbols=` on two of its six parses,
109
+ # the one name a caller could protect survived and any other name in the
110
+ # same expression was rewritten into a product of single letters --
111
+ # `integrate('force*distance', variable='distance')` answered about
112
+ # `c*e*f*o*r` without saying so.
113
+ rewritten: set = set()
114
+ syms = [parsed.variable] + [s for s in (parsed.symbols or [])
115
+ if s != parsed.variable]
63
116
 
64
117
  try:
65
118
  var = sympy.Symbol(parsed.variable)
66
- expr = safe_parse(parsed.expression, symbols=[parsed.variable])
119
+ expr = safe_parse(parsed.expression, symbols=syms, rewritten=rewritten)
67
120
  op = parsed.operation
68
121
 
69
122
  if op == "evaluate":
@@ -83,23 +136,28 @@ def run(params: dict) -> dict:
83
136
 
84
137
  elif op == "integrate":
85
138
  if parsed.lower_bound is not None and parsed.upper_bound is not None:
86
- lo = safe_parse(parsed.lower_bound)
87
- hi = safe_parse(parsed.upper_bound)
139
+ lo = safe_parse(parsed.lower_bound, symbols=syms,
140
+ rewritten=rewritten)
141
+ hi = safe_parse(parsed.upper_bound, symbols=syms,
142
+ rewritten=rewritten)
88
143
  result = sympy.integrate(expr, (var, lo, hi))
89
144
  else:
90
145
  result = sympy.integrate(expr, var)
91
146
 
92
147
  elif op == "limit":
93
- point = safe_parse(parsed.point) if parsed.point is not None else 0
148
+ point = (safe_parse(parsed.point, symbols=syms, rewritten=rewritten)
149
+ if parsed.point is not None else 0)
94
150
  result = sympy.limit(expr, var, point, dir=parsed.direction)
95
151
 
96
152
  elif op == "series":
97
- point = safe_parse(parsed.point) if parsed.point is not None else 0
153
+ point = (safe_parse(parsed.point, symbols=syms, rewritten=rewritten)
154
+ if parsed.point is not None else 0)
98
155
  result = expr.series(var, point, parsed.order).removeO()
99
156
 
100
157
  elif op == "solve":
101
158
  if parsed.equations:
102
- extra = [safe_parse(e, symbols=[parsed.variable]) for e in parsed.equations]
159
+ extra = [safe_parse(e, symbols=syms, rewritten=rewritten)
160
+ for e in parsed.equations]
103
161
  all_exprs = [expr] + extra
104
162
  symbols_found = sorted(
105
163
  set().union(*[e.free_symbols for e in all_exprs]), key=str
@@ -116,4 +174,16 @@ def run(params: dict) -> dict:
116
174
  except Exception as e:
117
175
  return {"error": f"Computation failed: {e}"}
118
176
 
119
- return {"result": serialize(result), "operation": op}
177
+ # H6, one batch late (batch 143). `serialize` is the SECOND place this function can fail
178
+ # and it used to sit outside every try, so a correct answer merely too
179
+ # large to print left `run()` as a bare ValueError and was reported as
180
+ # a harness bug with a traceback. A tail handler rather than one try
181
+ # round the whole body, which is the shape `discrete_math` -- the one
182
+ # tool H6 reached -- already uses.
183
+ try:
184
+ out: dict = {"result": serialize(result, what=op), "operation": op}
185
+ except MathParseError as e:
186
+ return {"error": str(e)}
187
+ if rewritten:
188
+ out["warnings"] = _split_warnings(rewritten)
189
+ return out
@@ -29,7 +29,6 @@ conversation, not a person.
29
29
 
30
30
  import config
31
31
 
32
-
33
32
  TOOL_SCHEMA = {
34
33
  "name": "todo_write",
35
34
  "description": (
@@ -76,8 +75,11 @@ def _normalise(raw) -> tuple:
76
75
  """Validate one list. Returns (items, error) -- exactly one is truthy.
77
76
 
78
77
  Every failure is an error result the model can correct, never an
79
- exception: a bare KeyError or TypeError here escapes the loop's
80
- ToolCallDenied handler and takes down the whole turn. pin.py's rule.
78
+ exception -- tools/base.py carries the one copy of why, under "WHAT A
79
+ CALLABLE REGISTERED BELOW MAY RAISE". The short version is not the one
80
+ this used to give: a raise here is CONTAINED, and what an error result
81
+ buys is a message the model can act on instead of a generic wrapper
82
+ (batch 146).
81
83
  """
82
84
  if not isinstance(raw, (list, tuple)):
83
85
  return None, ("todo_write requires 'items' to be a list of "
@@ -1,151 +1,329 @@
1
- """
2
- Web search tool for the agent harness, using DuckDuckGo exclusively via the
3
- `ddgs` package.
4
- """
5
-
6
- from __future__ import annotations
7
-
8
- import logging
9
- from typing import Optional
10
-
11
- from pydantic import BaseModel, Field, field_validator
12
- from ddgs import DDGS
13
-
14
- from safety.policy_enforcement import is_url_permitted
15
- from tools.builtin._net_common import TTLCache
16
-
17
- logger = logging.getLogger(__name__)
18
-
19
-
20
- # ---- Schema exposed to the LLM -------------------------------------------
21
-
22
- class WebSearchParams(BaseModel):
23
- query: str = Field(..., description="Search query", min_length=1, max_length=400)
24
- num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
25
-
26
- @field_validator("query")
27
- @classmethod
28
- def strip_query(cls, value: str) -> str:
29
- value = value.strip()
30
- if not value:
31
- raise ValueError("query cannot be empty")
32
- return value
33
-
34
-
35
- TOOL_SCHEMA = {
36
- "name": "web_search",
37
- "description": (
38
- "Search the web for current information. Use this for facts that may "
39
- "have changed recently, current events, or anything you're not "
40
- "fully confident about."
41
- ),
42
- "input_schema": WebSearchParams.model_json_schema(),
43
- }
44
-
45
-
46
- # ---- Normalized result shape ---------------------------------------------
47
-
48
- class SearchResult(BaseModel):
49
- title: str
50
- url: str
51
- snippet: str
52
- published: Optional[str] = None # ddgs's text search doesn't return a
53
- # publish date, so this stays None here
54
-
55
-
56
- class WebSearchError(Exception):
57
- """No longer raised by this module -- exhausted retries return an
58
- error dict instead, so a transient search outage cannot fail a
59
- ten-pass research run. Kept as part of this module's public surface."""
60
-
61
-
62
- # ---- Policy / tuning knobs -------------------------------------------------
63
-
64
- MAX_SNIPPET_CHARS = 300
65
- REQUEST_TIMEOUT_S = 8.0
66
- MAX_RETRIES = 2
67
- CACHE_TTL_S = 300
68
-
69
- # One instance per tool, never shared: an equal key string from an
70
- # arXiv query and a web query would otherwise collide.
71
- _cache = TTLCache(CACHE_TTL_S)
72
-
73
-
74
- # ---- Provider call ----------------------------------------------------------
75
-
76
- def _call_ddgs(query: str, num_results: int) -> list[dict]:
77
- """
78
- The one function that actually talks to DuckDuckGo. DDGS() is
79
- instantiated fresh per call (cheap) and used as a context manager so
80
- its underlying HTTP session gets cleaned up automatically afterward.
81
- """
82
- with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
83
- return ddgs_client.text(query, max_results=num_results)
84
-
85
-
86
- def _normalize(raw: list[dict]) -> list[SearchResult]:
87
- out: list[SearchResult] = []
88
- for r in raw:
89
- url = r.get("href", "")
90
- # `resolve=False` on purpose. This filters results; it does not
91
- # fetch them, and a DNS lookup per hit is real latency for a check
92
- # whose only failure mode is showing the model a URL that fetch_url
93
- # will refuse anyway. An IP LITERAL is still checked -- a result
94
- # pointing straight at 169.254.169.254 is dropped here rather than
95
- # offered and then denied (#54).
96
- if is_url_permitted(url, resolve=False):
97
- continue
98
-
99
- snippet = (r.get("body") or "")[:MAX_SNIPPET_CHARS]
100
-
101
- out.append(
102
- SearchResult(
103
- title=r.get("title", "(untitled)"),
104
- url=url,
105
- snippet=snippet,
106
- published=None,
107
- )
108
- )
109
- return out
110
-
111
-
112
- def run(params: dict) -> dict:
113
- parsed = WebSearchParams(**params)
114
- cache_key = f"{parsed.query}|{parsed.num_results}"
115
-
116
- results = _cache.get(cache_key)
117
- if results is not None:
118
- logger.info("web_search cache hit for %r", parsed.query)
119
- else:
120
- last_exc: Optional[Exception] = None
121
- results = []
122
- for attempt in range(MAX_RETRIES + 1):
123
- try:
124
- raw = _call_ddgs(parsed.query, parsed.num_results)
125
- results = _normalize(raw)
126
- _cache.set(cache_key, results)
127
- break
128
- except Exception as e: # Simpler to try again then elaborate on every error type
129
- last_exc = e
130
- logger.warning(
131
- "web_search attempt %s/%s failed for %r: %s",
132
- attempt + 1, MAX_RETRIES + 1, parsed.query, e)
133
- else:
134
- # RETURNED, not raised -- same change and same reason as
135
- # arxiv_search. A search provider being unreachable is a
136
- # result the model can work around; raising made one
137
- # transient failure in one pass fail a whole ten-pass
138
- # research run. fetch_url was already the odd one out for
139
- # doing this correctly.
140
- return {
141
- "error": f"search failed after {MAX_RETRIES + 1} "
142
- f"attempts: {last_exc}"
143
- }
144
-
145
- if not results:
146
- return {"results": [], "result_count": 0, "message": "No results found."}
147
-
148
- return {
149
- "results": [r.model_dump() for r in results],
150
- "result_count": len(results),
151
- }
1
+ """
2
+ Web search tool for the agent harness, using DuckDuckGo exclusively via the
3
+ `ddgs` package.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import logging
9
+ from typing import Optional
10
+
11
+ from ddgs import DDGS
12
+ from pydantic import BaseModel, Field, field_validator
13
+
14
+ from safety.policy_enforcement import is_url_permitted
15
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ # ---- Schema exposed to the LLM -------------------------------------------
21
+
22
+ class WebSearchParams(BaseModel):
23
+ query: str = Field(..., description="Search query", min_length=1, max_length=400)
24
+ num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
25
+ page: int = Field(
26
+ 1,
27
+ ge=1,
28
+ description=(
29
+ "Which page of results to return (1-based). Use the page named "
30
+ "in a result's message to see further results for the same "
31
+ "query."
32
+ ),
33
+ )
34
+
35
+ @field_validator("query")
36
+ @classmethod
37
+ def strip_query(cls, value: str) -> str:
38
+ value = value.strip()
39
+ if not value:
40
+ raise ValueError("query cannot be empty")
41
+ return value
42
+
43
+
44
+ TOOL_SCHEMA = {
45
+ "name": "web_search",
46
+ "description": (
47
+ "Search the web for current information. Use this for facts that may "
48
+ "have changed recently, current events, or anything you're not "
49
+ "fully confident about."
50
+ ),
51
+ "input_schema": WebSearchParams.model_json_schema(),
52
+ }
53
+
54
+
55
+ # ---- Normalized result shape ---------------------------------------------
56
+
57
+ class SearchResult(BaseModel):
58
+ title: str
59
+ url: str
60
+ snippet: str
61
+ published: Optional[str] = None # ddgs's text search doesn't return a
62
+ # publish date, so this stays None here
63
+ # ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
64
+ # where the rest is. Before §51 a cut snippet and a short one were
65
+ # indistinguishable, so "the source does not mention X" and "the first
66
+ # 300 characters do not mention X" read the same to the model.
67
+ snippet_truncated: bool = False
68
+ # Batch 156 (PG8's shape, one field over). The snippet had a cap and a
69
+ # flag and the title beside it had neither, so a 5,000-character title
70
+ # went to the model whole while the snippet under it was cut to 300 --
71
+ # measured, in the same result. FLAG AND NO ROUTE, which is the
72
+ # asymmetry `arxiv.py` already argues for the same field: a title past
73
+ # the cap is a malformed result rather than a long one, and the route
74
+ # to the page is the `url` sitting next to it either way.
75
+ title_truncated: bool = False
76
+
77
+
78
+ class WebSearchError(Exception):
79
+ """No longer raised by this module -- exhausted retries return an
80
+ error dict instead, so a transient search outage cannot fail a
81
+ ten-pass research run. Kept as part of this module's public surface."""
82
+
83
+
84
+ # ---- Policy / tuning knobs -------------------------------------------------
85
+
86
+ MAX_SNIPPET_CHARS = 300
87
+
88
+ # Batch 156. 300 is the number `arxiv.py` settled for the same kind of
89
+ # value, measured there over sixty real entries whose longest title was
90
+ # 188 characters. NOT imported from it: two constants with one reason each
91
+ # is the choice `_MAX_AUTHORS` already argues for one file over, and these
92
+ # two bound titles from two different providers.
93
+ #
94
+ # No equivalent measurement was taken for web titles in this batch, and
95
+ # saying so is the honest version. What justifies the number here is that
96
+ # it is the one already settled for a title, and a title is the same kind
97
+ # of value whichever provider hands it over; the HTML spec puts no bound on
98
+ # `<title>` at all.
99
+ MAX_TITLE_CHARS = 300
100
+
101
+ REQUEST_TIMEOUT_S = 8.0
102
+ MAX_RETRIES = 2
103
+ CACHE_TTL_S = 300
104
+
105
+ # One instance per tool, never shared: an equal key string from an
106
+ # arXiv query and a web query would otherwise collide.
107
+ _cache = TTLCache(CACHE_TTL_S)
108
+
109
+
110
+ # ---- Provider call ----------------------------------------------------------
111
+
112
+ def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
113
+ """
114
+ The one function that actually talks to DuckDuckGo. DDGS() is
115
+ instantiated fresh per call (cheap) and used as a context manager so
116
+ its underlying HTTP session gets cleaned up automatically afterward.
117
+
118
+ `page` is ROADMAP_v3 §51 (PG3's sibling). The record assumed this tool
119
+ had no paging to expose and only a snippet cap to fix; MEASURED (batch
120
+ 105), `ddgs 9.14.4` takes `page: int = 1` on its search path and
121
+ accepted it live. So the provider implemented paging here too, and this
122
+ tool was spending the parameter on a default.
123
+
124
+ ALSO MEASURED, and the reason the message this feeds is worded the way
125
+ it is: page 2 of a live query OVERLAPPED page 1 by one result. DuckDuckGo's
126
+ paging is not a clean partition the way arXiv's `start` is (measured the
127
+ same day: zero overlap), so a page here is "more results", not "the next
128
+ five distinct results", and nothing downstream may assume otherwise.
129
+ """
130
+ with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
131
+ return ddgs_client.text(query, max_results=num_results, page=page)
132
+
133
+
134
+ def _normalize(raw: list[dict]) -> list[SearchResult]:
135
+ """DDGS's rows as `SearchResult`s, redacted and bounded (§73).
136
+
137
+ EVERY STRING IN HERE WAS WRITTEN BY SOMEBODY ELSE, which is the rule
138
+ PG8/#131 states and the reason all of it goes through one helper now.
139
+ Two things were wrong at once and they interact, so fixing either alone
140
+ would have been worse than fixing neither:
141
+
142
+ REDACT ABOVE THE CUT (RB1, A59). The snippet was cut to 300 characters
143
+ and `registry.dispatch` redacted the piece that was kept. Measured
144
+ (batch 156), a `ghp_` token straddling that cut survived post-hoc
145
+ redaction at 36 of the 39 offsets it could straddle -- the kept prefix
146
+ is not the token, the dropped suffix is gone, and `param_digest`'s
147
+ docstring had already written down why ("truncating first cuts a
148
+ credential below the 20 characters its pattern needs").
149
+
150
+ AND THE CUTS THAT DID NOT EXIST YET. The title had no bound at all, so
151
+ adding one without the ordering above would have MANUFACTURED a second
152
+ straddle site in the field being repaired. Every new cap here is a new
153
+ cut, and every cut needs the redaction above it.
154
+
155
+ TOTAL OVER WHATEVER DDGS RETURNED (A61), which is the half no report
156
+ named. `r.get("title", "(untitled)")` substitutes only for an ABSENT
157
+ key, and `{"title": null}` is a present one -- the lesson
158
+ `tools/base.py` states for tool arguments, arriving from the provider
159
+ end. Measured, a single row with `title: null` raised `ValidationError`
160
+ out of here, which `run` caught as a failed attempt, RETRIED TWICE,
161
+ and then reported as "search failed after 3 attempts" -- ten good
162
+ results lost to one malformed one, plus two pointless re-queries.
163
+ `arxiv._parse_atom_feed` had the identical defect and fixed it in batch
164
+ 117 ("SKIPPED, NOT RAISED ... Raising here aborted the whole search
165
+ over one malformed entry"); the cure was one file over the whole time.
166
+ """
167
+ third = ThirdPartyText("web_search")
168
+ out: list[SearchResult] = []
169
+ for r in raw:
170
+ if not isinstance(r, dict):
171
+ logger.warning(
172
+ "web_search skipped a result: the provider returned a %s "
173
+ "where a row was expected.", type(r).__name__)
174
+ continue
175
+
176
+ url, withheld = third.url(r.get("href"))
177
+ if withheld:
178
+ # NOT SILENT (#49). The length is the fact; the URL is not
179
+ # echoed, because a URL is one of the things the redactor is
180
+ # looking for userinfo inside.
181
+ logger.warning(
182
+ "web_search dropped a result: its URL is longer than the "
183
+ "%d characters fetch_url accepts, so it is a route this "
184
+ "harness would refuse to take.", MAX_URL_CHARS)
185
+ if url is None:
186
+ # A result is a (title, url, snippet) triple whose url is how
187
+ # the model reaches it. With the url gone there is nothing to
188
+ # cite and nothing to fetch, so this is the same call the
189
+ # policy filter below already makes -- and it is the call
190
+ # `_parse_atom_feed` makes for an entry with no id.
191
+ continue
192
+
193
+ # `resolve=False` on purpose. This filters results; it does not
194
+ # fetch them, and a DNS lookup per hit is real latency for a check
195
+ # whose only failure mode is showing the model a URL that fetch_url
196
+ # will refuse anyway. An IP LITERAL is still checked, in every
197
+ # spelling the platform's resolver reads as a number -- a result
198
+ # pointing straight at 169.254.169.254 is dropped here rather than
199
+ # offered and then denied (#54).
200
+ #
201
+ # "EVERY SPELLING" SINCE BATCH 213 (#207, register item 53). Batch
202
+ # 156 narrowed this comment to dotted and bracketed literals,
203
+ # because `http://2130706433/`, `http://0x7f.0.0.1/` and
204
+ # `http://0x7F000001/` -- all 127.0.0.1 -- passed: `ipaddress`
205
+ # parses none of them. is_url_permitted now asks the resolver with
206
+ # AI_NUMERICHOST, which reads numbers and looks nothing up, so they
207
+ # are dropped here as fetch_url would refuse them.
208
+ #
209
+ # ABOVE THE POLICY CHECK AND BELOW THE BOUND, deliberately: the
210
+ # string checked here is the string the model is offered, so a
211
+ # redaction that rewrote the host cannot slip past the check that
212
+ # was run on the host before it.
213
+ if is_url_permitted(url, resolve=False):
214
+ continue
215
+
216
+ title, title_truncated = third.text(r.get("title"), MAX_TITLE_CHARS)
217
+ if title is None:
218
+ title = "(untitled)"
219
+ snippet, snippet_truncated = third.text(
220
+ r.get("body"), MAX_SNIPPET_CHARS)
221
+ if snippet is None:
222
+ snippet = ""
223
+
224
+ out.append(
225
+ SearchResult(
226
+ title=title,
227
+ url=url,
228
+ snippet=snippet,
229
+ published=None,
230
+ snippet_truncated=snippet_truncated,
231
+ title_truncated=title_truncated,
232
+ )
233
+ )
234
+ third.report()
235
+ return out
236
+
237
+
238
+ def _kept(results: list, asked: int) -> list:
239
+ """`results`, cut to the number the caller asked for (A62).
240
+
241
+ THE SCHEMA ALREADY PROMISED THIS and nothing enforced it.
242
+ `num_results` is advertised `le=10` and passed to the provider as
243
+ `max_results`, and a provider is free to ignore it -- measured (batch
244
+ 156), a call asking for 5 returned every one of 5,000 rows it was
245
+ handed. So this is a documented bound being kept rather than a new
246
+ policy, and it is the LIST axis of the same question batch 156 asked
247
+ of each value: per-value caps multiplied by an unbounded row count are
248
+ not a bound.
249
+
250
+ `_call_ddgs` is where the finding wanted a byte ceiling, and it cannot
251
+ have one: the library buffers the response and owns the socket, so
252
+ nothing above it sees bytes. What this layer CAN bound is what it
253
+ keeps, which is the quantity that reaches the model.
254
+
255
+ NO FLAG, deliberately, where every other cut in this batch gets one.
256
+ Nothing is being withheld from the caller: it asked for `asked`
257
+ results and it is getting `asked` results. The over-delivery is a
258
+ provider fact rather than a truncation, so it goes to the log, where
259
+ the person debugging a provider can see it.
260
+ """
261
+ if len(results) > asked:
262
+ logger.warning(
263
+ "web_search kept %d of %d results: the provider returned more "
264
+ "than the %d asked for.", asked, len(results), asked)
265
+ return results[:asked]
266
+
267
+
268
+ def run(params: dict) -> dict:
269
+ parsed = WebSearchParams(**params)
270
+ # §51 (PG9). `page` is part of the key for the reason `start` is on the
271
+ # arXiv side: it changes the response, so a key without it serves page 1
272
+ # to every request for page 2 and does it silently.
273
+ cache_key = f"{parsed.query}|{parsed.num_results}|{parsed.page}"
274
+
275
+ results = _cache.get(cache_key)
276
+ if results is not None:
277
+ logger.info("web_search cache hit for %r", parsed.query)
278
+ else:
279
+ last_exc: Optional[Exception] = None
280
+ results = []
281
+ for attempt in range(MAX_RETRIES + 1):
282
+ try:
283
+ raw = _call_ddgs(parsed.query, parsed.num_results, parsed.page)
284
+ results = _kept(_normalize(raw), parsed.num_results)
285
+ _cache.set(cache_key, results)
286
+ break
287
+ except Exception as e: # Simpler to try again then elaborate on every error type
288
+ last_exc = e
289
+ logger.warning(
290
+ "web_search attempt %s/%s failed for %r: %s",
291
+ attempt + 1, MAX_RETRIES + 1, parsed.query, e)
292
+ else:
293
+ # RETURNED, not raised -- same change and same reason as
294
+ # arxiv_search. A search provider being unreachable is a
295
+ # result the model can work around; raising made one
296
+ # transient failure in one pass fail a whole ten-pass
297
+ # research run. fetch_url was already the odd one out for
298
+ # doing this correctly.
299
+ return {
300
+ "error": f"search failed after {MAX_RETRIES + 1} "
301
+ f"attempts: {last_exc}"
302
+ }
303
+
304
+ if not results:
305
+ if parsed.page > 1:
306
+ return {
307
+ "results": [], "result_count": 0, "page": parsed.page,
308
+ "message": (
309
+ f"No results on page {parsed.page}. The provider "
310
+ f"returned nothing further for this query."
311
+ ),
312
+ }
313
+ return {"results": [], "result_count": 0, "message": "No results found."}
314
+
315
+ out = {
316
+ "results": [r.model_dump() for r in results],
317
+ "result_count": len(results),
318
+ "page": parsed.page,
319
+ }
320
+ # PG4. DuckDuckGo reports no total, so the honest message names the next
321
+ # page without claiming how many there are -- and says the pages overlap,
322
+ # because measured they do. A full page is the only signal that more may
323
+ # exist; a short one is where the provider stopped.
324
+ if len(results) >= parsed.num_results:
325
+ out["message"] = (
326
+ f"Page {parsed.page}. Use page={parsed.page + 1} for more "
327
+ f"results; pages may overlap by a result or two."
328
+ )
329
+ return out