@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,329 +1,403 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Web search tool for the agent harness, using DuckDuckGo exclusively via the
|
|
3
|
-
`ddgs` package.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from __future__ import annotations
|
|
7
|
-
|
|
8
|
-
import logging
|
|
9
|
-
from typing import Optional
|
|
10
|
-
|
|
11
|
-
from ddgs import DDGS
|
|
12
|
-
from pydantic import BaseModel, Field, field_validator
|
|
13
|
-
|
|
14
|
-
from safety.policy_enforcement import is_url_permitted
|
|
15
|
-
from tools.builtin
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
#
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
#
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
#
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
#
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
is not
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
if
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
if
|
|
222
|
-
snippet
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
if len(results)
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
1
|
+
"""
|
|
2
|
+
Web search tool for the agent harness, using DuckDuckGo exclusively via the
|
|
3
|
+
`ddgs` package.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
from ddgs import DDGS
|
|
12
|
+
from pydantic import BaseModel, Field, field_validator
|
|
13
|
+
|
|
14
|
+
from safety.policy_enforcement import is_url_permitted
|
|
15
|
+
from tools.builtin import _net_common
|
|
16
|
+
from tools.builtin._net_common import (
|
|
17
|
+
MAX_URL_CHARS,
|
|
18
|
+
NetworkLimitReached,
|
|
19
|
+
ThirdPartyText,
|
|
20
|
+
TTLCache,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# ---- Schema exposed to the LLM -------------------------------------------
|
|
27
|
+
|
|
28
|
+
class WebSearchParams(BaseModel):
|
|
29
|
+
query: str = Field(..., description="Search query", min_length=1, max_length=400)
|
|
30
|
+
num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
|
|
31
|
+
page: int = Field(
|
|
32
|
+
1,
|
|
33
|
+
ge=1,
|
|
34
|
+
description=(
|
|
35
|
+
"Which page of results to return (1-based). Use the page named "
|
|
36
|
+
"in a result's message to see further results for the same "
|
|
37
|
+
"query."
|
|
38
|
+
),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
@field_validator("query")
|
|
42
|
+
@classmethod
|
|
43
|
+
def strip_query(cls, value: str) -> str:
|
|
44
|
+
value = value.strip()
|
|
45
|
+
if not value:
|
|
46
|
+
raise ValueError("query cannot be empty")
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
TOOL_SCHEMA = {
|
|
51
|
+
"name": "web_search",
|
|
52
|
+
"description": (
|
|
53
|
+
"Search the web for current information. Use this for facts that may "
|
|
54
|
+
"have changed recently, current events, or anything you're not "
|
|
55
|
+
"fully confident about."
|
|
56
|
+
),
|
|
57
|
+
"input_schema": WebSearchParams.model_json_schema(),
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# ---- Normalized result shape ---------------------------------------------
|
|
62
|
+
|
|
63
|
+
class SearchResult(BaseModel):
|
|
64
|
+
"""One result as the model reads it.
|
|
65
|
+
|
|
66
|
+
BATCH 240 TOOK OUT WHAT CARRIED NOTHING. `published` was always None --
|
|
67
|
+
ddgs's text search returns no date, as its own comment here said -- and
|
|
68
|
+
both truncation flags were False on almost every row. Measured on a live
|
|
69
|
+
search, those three keys were 361 of the result's 2,651 characters. The
|
|
70
|
+
flags are now present only when TRUE (`model_dump(exclude_defaults=
|
|
71
|
+
True)`), the shape `arxiv_search` already uses for its own.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
title: str
|
|
75
|
+
url: str
|
|
76
|
+
snippet: str
|
|
77
|
+
# ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
|
|
78
|
+
# where the rest is. Before §51 a cut snippet and a short one were
|
|
79
|
+
# indistinguishable, so "the source does not mention X" and "the first
|
|
80
|
+
# 300 characters do not mention X" read the same to the model.
|
|
81
|
+
snippet_truncated: bool = False
|
|
82
|
+
# Batch 156 (PG8's shape, one field over). The snippet had a cap and a
|
|
83
|
+
# flag and the title beside it had neither, so a 5,000-character title
|
|
84
|
+
# went to the model whole while the snippet under it was cut to 300 --
|
|
85
|
+
# measured, in the same result. FLAG AND NO ROUTE, which is the
|
|
86
|
+
# asymmetry `arxiv.py` already argues for the same field: a title past
|
|
87
|
+
# the cap is a malformed result rather than a long one, and the route
|
|
88
|
+
# to the page is the `url` sitting next to it either way.
|
|
89
|
+
title_truncated: bool = False
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class WebSearchError(Exception):
|
|
93
|
+
"""No longer raised by this module -- exhausted retries return an
|
|
94
|
+
error dict instead, so a transient search outage cannot fail a
|
|
95
|
+
ten-pass research run. Kept as part of this module's public surface."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ---- Policy / tuning knobs -------------------------------------------------
|
|
99
|
+
|
|
100
|
+
MAX_SNIPPET_CHARS = 300
|
|
101
|
+
|
|
102
|
+
# A cut snippet ends at a word, with this after it, rather than in the
|
|
103
|
+
# middle of one ("...into a vector th"): the flag says it was cut, and the
|
|
104
|
+
# mark says where. Not before 60% of the cap, so a snippet with one long
|
|
105
|
+
# word is still cut near the cap rather than to almost nothing.
|
|
106
|
+
_ELLIPSIS = "\u2026"
|
|
107
|
+
|
|
108
|
+
# Batch 156. 300 is the number `arxiv.py` settled for the same kind of
|
|
109
|
+
# value, measured there over sixty real entries whose longest title was
|
|
110
|
+
# 188 characters. NOT imported from it: two constants with one reason each
|
|
111
|
+
# is the choice `_MAX_AUTHORS` already argues for one file over, and these
|
|
112
|
+
# two bound titles from two different providers.
|
|
113
|
+
#
|
|
114
|
+
# No equivalent measurement was taken for web titles in this batch, and
|
|
115
|
+
# saying so is the honest version. What justifies the number here is that
|
|
116
|
+
# it is the one already settled for a title, and a title is the same kind
|
|
117
|
+
# of value whichever provider hands it over; the HTML spec puts no bound on
|
|
118
|
+
# `<title>` at all.
|
|
119
|
+
MAX_TITLE_CHARS = 300
|
|
120
|
+
|
|
121
|
+
REQUEST_TIMEOUT_S = 8.0
|
|
122
|
+
MAX_RETRIES = 2
|
|
123
|
+
CACHE_TTL_S = 300
|
|
124
|
+
|
|
125
|
+
# One instance per tool, never shared: an equal key string from an
|
|
126
|
+
# arXiv query and a web query would otherwise collide.
|
|
127
|
+
_cache = TTLCache(CACHE_TTL_S)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ---- Provider call ----------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
|
|
133
|
+
"""
|
|
134
|
+
The one function that actually talks to DuckDuckGo. DDGS() is
|
|
135
|
+
instantiated fresh per call (cheap) and used as a context manager so
|
|
136
|
+
its underlying HTTP session gets cleaned up automatically afterward.
|
|
137
|
+
|
|
138
|
+
`page` is ROADMAP_v3 §51 (PG3's sibling). The record assumed this tool
|
|
139
|
+
had no paging to expose and only a snippet cap to fix; MEASURED (batch
|
|
140
|
+
105), `ddgs 9.14.4` takes `page: int = 1` on its search path and
|
|
141
|
+
accepted it live. So the provider implemented paging here too, and this
|
|
142
|
+
tool was spending the parameter on a default.
|
|
143
|
+
|
|
144
|
+
ALSO MEASURED, and the reason the message this feeds is worded the way
|
|
145
|
+
it is: page 2 of a live query OVERLAPPED page 1 by one result. DuckDuckGo's
|
|
146
|
+
paging is not a clean partition the way arXiv's `start` is (measured the
|
|
147
|
+
same day: zero overlap), so a page here is "more results", not "the next
|
|
148
|
+
five distinct results", and nothing downstream may assume otherwise.
|
|
149
|
+
"""
|
|
150
|
+
# ROADMAP_v3 §53. Admitted against the tool meter before the search,
|
|
151
|
+
# and charged IO15's measured estimate after it, success or not: a
|
|
152
|
+
# failed attempt still spent most of what a good one does. `ddgs` talks
|
|
153
|
+
# through primp, which has no hook, so this is the one tool whose figure
|
|
154
|
+
# is not counted at the wire.
|
|
155
|
+
_net_common.TOOLS.admit()
|
|
156
|
+
try:
|
|
157
|
+
with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
|
|
158
|
+
return ddgs_client.text(query, max_results=num_results, page=page)
|
|
159
|
+
except Exception as exc:
|
|
160
|
+
# An empty search is an ANSWER. Retried as a failure, it cost two
|
|
161
|
+
# more full searches and then told the model "search failed after
|
|
162
|
+
# 3 attempts: No results found." -- measured, the same day.
|
|
163
|
+
if _empty(exc):
|
|
164
|
+
return []
|
|
165
|
+
raise
|
|
166
|
+
finally:
|
|
167
|
+
_net_common.record_web_search(num_results)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _normalize(raw: list[dict]) -> list[SearchResult]:
|
|
171
|
+
"""DDGS's rows as `SearchResult`s, redacted and bounded (§73).
|
|
172
|
+
|
|
173
|
+
EVERY STRING IN HERE WAS WRITTEN BY SOMEBODY ELSE, which is the rule
|
|
174
|
+
PG8/#131 states and the reason all of it goes through one helper now.
|
|
175
|
+
Two things were wrong at once and they interact, so fixing either alone
|
|
176
|
+
would have been worse than fixing neither:
|
|
177
|
+
|
|
178
|
+
REDACT ABOVE THE CUT (RB1, A59). The snippet was cut to 300 characters
|
|
179
|
+
and `registry.dispatch` redacted the piece that was kept. Measured
|
|
180
|
+
(batch 156), a `ghp_` token straddling that cut survived post-hoc
|
|
181
|
+
redaction at 36 of the 39 offsets it could straddle -- the kept prefix
|
|
182
|
+
is not the token, the dropped suffix is gone, and `param_digest`'s
|
|
183
|
+
docstring had already written down why ("truncating first cuts a
|
|
184
|
+
credential below the 20 characters its pattern needs").
|
|
185
|
+
|
|
186
|
+
AND THE CUTS THAT DID NOT EXIST YET. The title had no bound at all, so
|
|
187
|
+
adding one without the ordering above would have MANUFACTURED a second
|
|
188
|
+
straddle site in the field being repaired. Every new cap here is a new
|
|
189
|
+
cut, and every cut needs the redaction above it.
|
|
190
|
+
|
|
191
|
+
TOTAL OVER WHATEVER DDGS RETURNED (A61), which is the half no report
|
|
192
|
+
named. `r.get("title", "(untitled)")` substitutes only for an ABSENT
|
|
193
|
+
key, and `{"title": null}` is a present one -- the lesson
|
|
194
|
+
`tools/base.py` states for tool arguments, arriving from the provider
|
|
195
|
+
end. Measured, a single row with `title: null` raised `ValidationError`
|
|
196
|
+
out of here, which `run` caught as a failed attempt, RETRIED TWICE,
|
|
197
|
+
and then reported as "search failed after 3 attempts" -- ten good
|
|
198
|
+
results lost to one malformed one, plus two pointless re-queries.
|
|
199
|
+
`arxiv._parse_atom_feed` had the identical defect and fixed it in batch
|
|
200
|
+
117 ("SKIPPED, NOT RAISED ... Raising here aborted the whole search
|
|
201
|
+
over one malformed entry"); the cure was one file over the whole time.
|
|
202
|
+
"""
|
|
203
|
+
third = ThirdPartyText("web_search")
|
|
204
|
+
out: list[SearchResult] = []
|
|
205
|
+
for r in raw:
|
|
206
|
+
if not isinstance(r, dict):
|
|
207
|
+
logger.warning(
|
|
208
|
+
"web_search skipped a result: the provider returned a %s "
|
|
209
|
+
"where a row was expected.", type(r).__name__)
|
|
210
|
+
continue
|
|
211
|
+
|
|
212
|
+
url, withheld = third.url(r.get("href"))
|
|
213
|
+
if withheld:
|
|
214
|
+
# NOT SILENT (#49). The length is the fact; the URL is not
|
|
215
|
+
# echoed, because a URL is one of the things the redactor is
|
|
216
|
+
# looking for userinfo inside.
|
|
217
|
+
logger.warning(
|
|
218
|
+
"web_search dropped a result: its URL is longer than the "
|
|
219
|
+
"%d characters fetch_url accepts, so it is a route this "
|
|
220
|
+
"harness would refuse to take.", MAX_URL_CHARS)
|
|
221
|
+
if url is None:
|
|
222
|
+
# A result is a (title, url, snippet) triple whose url is how
|
|
223
|
+
# the model reaches it. With the url gone there is nothing to
|
|
224
|
+
# cite and nothing to fetch, so this is the same call the
|
|
225
|
+
# policy filter below already makes -- and it is the call
|
|
226
|
+
# `_parse_atom_feed` makes for an entry with no id.
|
|
227
|
+
continue
|
|
228
|
+
|
|
229
|
+
# `resolve=False` on purpose. This filters results; it does not
|
|
230
|
+
# fetch them, and a DNS lookup per hit is real latency for a check
|
|
231
|
+
# whose only failure mode is showing the model a URL that fetch_url
|
|
232
|
+
# will refuse anyway. An IP LITERAL is still checked, in every
|
|
233
|
+
# spelling the platform's resolver reads as a number -- a result
|
|
234
|
+
# pointing straight at 169.254.169.254 is dropped here rather than
|
|
235
|
+
# offered and then denied (#54).
|
|
236
|
+
#
|
|
237
|
+
# "EVERY SPELLING" SINCE BATCH 213 (#207, register item 53). Batch
|
|
238
|
+
# 156 narrowed this comment to dotted and bracketed literals,
|
|
239
|
+
# because `http://2130706433/`, `http://0x7f.0.0.1/` and
|
|
240
|
+
# `http://0x7F000001/` -- all 127.0.0.1 -- passed: `ipaddress`
|
|
241
|
+
# parses none of them. is_url_permitted now asks the resolver with
|
|
242
|
+
# AI_NUMERICHOST, which reads numbers and looks nothing up, so they
|
|
243
|
+
# are dropped here as fetch_url would refuse them.
|
|
244
|
+
#
|
|
245
|
+
# ABOVE THE POLICY CHECK AND BELOW THE BOUND, deliberately: the
|
|
246
|
+
# string checked here is the string the model is offered, so a
|
|
247
|
+
# redaction that rewrote the host cannot slip past the check that
|
|
248
|
+
# was run on the host before it.
|
|
249
|
+
if is_url_permitted(url, resolve=False):
|
|
250
|
+
continue
|
|
251
|
+
|
|
252
|
+
title, title_truncated = third.text(r.get("title"), MAX_TITLE_CHARS)
|
|
253
|
+
if title is None:
|
|
254
|
+
title = "(untitled)"
|
|
255
|
+
snippet, snippet_truncated = third.text(
|
|
256
|
+
r.get("body"), MAX_SNIPPET_CHARS)
|
|
257
|
+
if snippet is None:
|
|
258
|
+
snippet = ""
|
|
259
|
+
elif snippet_truncated:
|
|
260
|
+
snippet = _at_a_word(snippet, MAX_SNIPPET_CHARS)
|
|
261
|
+
|
|
262
|
+
out.append(
|
|
263
|
+
SearchResult(
|
|
264
|
+
title=title,
|
|
265
|
+
url=url,
|
|
266
|
+
snippet=snippet,
|
|
267
|
+
snippet_truncated=snippet_truncated,
|
|
268
|
+
title_truncated=title_truncated,
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
third.report()
|
|
272
|
+
return out
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _at_a_word(text: str, cap: int) -> str:
|
|
276
|
+
"""`text`, already cut to `cap`, cut back to a word and marked."""
|
|
277
|
+
room = text[:cap - len(_ELLIPSIS)]
|
|
278
|
+
space = room.rfind(" ")
|
|
279
|
+
if space >= cap * 0.6:
|
|
280
|
+
room = room[:space]
|
|
281
|
+
return room.rstrip(" ,;:-") + _ELLIPSIS
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _empty(exc: BaseException) -> bool:
|
|
285
|
+
"""ddgs's way of saying a search found nothing: it RAISES.
|
|
286
|
+
|
|
287
|
+
Measured (batch 240): `DDGS.text` raises `DDGSException("No results
|
|
288
|
+
found.")` when every backend answered and none had a row -- and only
|
|
289
|
+
then; a backend's own failure is raised under its own message. Read by
|
|
290
|
+
class NAME, as `core/provider_errors` reads an SDK's, because the suite
|
|
291
|
+
installs a fake `ddgs`.
|
|
292
|
+
"""
|
|
293
|
+
return (type(exc).__name__ == "DDGSException"
|
|
294
|
+
and str(exc).strip().rstrip(".").lower() == "no results found")
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _rate_limited(exc: BaseException) -> bool:
|
|
298
|
+
return type(exc).__name__ == "RatelimitException"
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _kept(results: list, asked: int) -> list:
|
|
302
|
+
"""`results`, cut to the number the caller asked for (A62).
|
|
303
|
+
|
|
304
|
+
THE SCHEMA ALREADY PROMISED THIS and nothing enforced it.
|
|
305
|
+
`num_results` is advertised `le=10` and passed to the provider as
|
|
306
|
+
`max_results`, and a provider is free to ignore it -- measured (batch
|
|
307
|
+
156), a call asking for 5 returned every one of 5,000 rows it was
|
|
308
|
+
handed. So this is a documented bound being kept rather than a new
|
|
309
|
+
policy, and it is the LIST axis of the same question batch 156 asked
|
|
310
|
+
of each value: per-value caps multiplied by an unbounded row count are
|
|
311
|
+
not a bound.
|
|
312
|
+
|
|
313
|
+
`_call_ddgs` is where the finding wanted a byte ceiling, and it cannot
|
|
314
|
+
have one: the library buffers the response and owns the socket, so
|
|
315
|
+
nothing above it sees bytes. What this layer CAN bound is what it
|
|
316
|
+
keeps, which is the quantity that reaches the model.
|
|
317
|
+
|
|
318
|
+
NO FLAG, deliberately, where every other cut in this batch gets one.
|
|
319
|
+
Nothing is being withheld from the caller: it asked for `asked`
|
|
320
|
+
results and it is getting `asked` results. The over-delivery is a
|
|
321
|
+
provider fact rather than a truncation, so it goes to the log, where
|
|
322
|
+
the person debugging a provider can see it.
|
|
323
|
+
"""
|
|
324
|
+
if len(results) > asked:
|
|
325
|
+
logger.warning(
|
|
326
|
+
"web_search kept %d of %d results: the provider returned more "
|
|
327
|
+
"than the %d asked for.", asked, len(results), asked)
|
|
328
|
+
return results[:asked]
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def run(params: dict) -> dict:
|
|
332
|
+
parsed = WebSearchParams(**params)
|
|
333
|
+
# §51 (PG9). `page` is part of the key for the reason `start` is on the
|
|
334
|
+
# arXiv side: it changes the response, so a key without it serves page 1
|
|
335
|
+
# to every request for page 2 and does it silently.
|
|
336
|
+
cache_key = f"{parsed.query}|{parsed.num_results}|{parsed.page}"
|
|
337
|
+
|
|
338
|
+
results = _cache.get(cache_key)
|
|
339
|
+
if results is not None:
|
|
340
|
+
logger.info("web_search cache hit for %r", parsed.query)
|
|
341
|
+
else:
|
|
342
|
+
last_exc: Optional[Exception] = None
|
|
343
|
+
results = []
|
|
344
|
+
for attempt in range(MAX_RETRIES + 1):
|
|
345
|
+
try:
|
|
346
|
+
raw = _call_ddgs(parsed.query, parsed.num_results, parsed.page)
|
|
347
|
+
results = _kept(_normalize(raw), parsed.num_results)
|
|
348
|
+
_cache.set(cache_key, results)
|
|
349
|
+
break
|
|
350
|
+
except NetworkLimitReached:
|
|
351
|
+
# §53 (IO6). A full meter is not weather: another attempt
|
|
352
|
+
# is refused the same way. Raised on, for dispatch to turn
|
|
353
|
+
# into the result the model reads.
|
|
354
|
+
raise
|
|
355
|
+
except Exception as e: # Simpler to try again then elaborate on every error type
|
|
356
|
+
if _rate_limited(e):
|
|
357
|
+
# Retrying a rate limit at once is asking again into
|
|
358
|
+
# the limit. Said once, plainly, so the model waits.
|
|
359
|
+
return {"error": (
|
|
360
|
+
"The search provider is rate-limiting this harness. "
|
|
361
|
+
"Wait a minute before searching again.")}
|
|
362
|
+
last_exc = e
|
|
363
|
+
logger.warning(
|
|
364
|
+
"web_search attempt %s/%s failed for %r: %s",
|
|
365
|
+
attempt + 1, MAX_RETRIES + 1, parsed.query, e)
|
|
366
|
+
else:
|
|
367
|
+
# RETURNED, not raised -- same change and same reason as
|
|
368
|
+
# arxiv_search. A search provider being unreachable is a
|
|
369
|
+
# result the model can work around; raising made one
|
|
370
|
+
# transient failure in one pass fail a whole ten-pass
|
|
371
|
+
# research run. fetch_url was already the odd one out for
|
|
372
|
+
# doing this correctly.
|
|
373
|
+
return {
|
|
374
|
+
"error": f"search failed after {MAX_RETRIES + 1} "
|
|
375
|
+
f"attempts: {last_exc}"
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if not results:
|
|
379
|
+
if parsed.page > 1:
|
|
380
|
+
return {
|
|
381
|
+
"results": [], "result_count": 0, "page": parsed.page,
|
|
382
|
+
"message": (
|
|
383
|
+
f"No results on page {parsed.page}. The provider "
|
|
384
|
+
f"returned nothing further for this query."
|
|
385
|
+
),
|
|
386
|
+
}
|
|
387
|
+
return {"results": [], "result_count": 0, "message": "No results found."}
|
|
388
|
+
|
|
389
|
+
out = {
|
|
390
|
+
"results": [r.model_dump(exclude_defaults=True) for r in results],
|
|
391
|
+
"result_count": len(results),
|
|
392
|
+
"page": parsed.page,
|
|
393
|
+
}
|
|
394
|
+
# PG4. DuckDuckGo reports no total, so the honest message names the next
|
|
395
|
+
# page without claiming how many there are -- and says the pages overlap,
|
|
396
|
+
# because measured they do. A full page is the only signal that more may
|
|
397
|
+
# exist; a short one is where the provider stopped.
|
|
398
|
+
if len(results) >= parsed.num_results:
|
|
399
|
+
out["message"] = (
|
|
400
|
+
f"Page {parsed.page}. Use page={parsed.page + 1} for more "
|
|
401
|
+
f"results; pages may overlap by a result or two."
|
|
402
|
+
)
|
|
329
403
|
return out
|