@yahyahammad/venastine 0.2.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +233 -94
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +277 -43
- package/agents/tui_commands.py +7 -12
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +158 -20
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1238 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +830 -66
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +1516 -440
- package/core/memory.py +120 -8
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +158 -26
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1671 -199
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +7 -12
- package/package.json +234 -4
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +51 -2
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +353 -17
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +368 -35
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +347 -30
- package/tui/app.py +4645 -438
- package/tui/app.tcss +185 -2
- package/tui/commands.py +10 -154
- package/tui/diffs.py +286 -265
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +60 -10
- package/tui/meters.py +103 -7
- package/tui/screens.py +346 -92
- package/tui/themes.py +59 -19
- package/tui/widgets.py +1575 -111
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -15,7 +15,13 @@ from typing import Literal, Optional
|
|
|
15
15
|
import sympy
|
|
16
16
|
from pydantic import BaseModel, Field
|
|
17
17
|
|
|
18
|
-
from tools.builtin._math_common import
|
|
18
|
+
from tools.builtin._math_common import (
|
|
19
|
+
MathParseError,
|
|
20
|
+
_split_warnings,
|
|
21
|
+
parse_params,
|
|
22
|
+
safe_parse,
|
|
23
|
+
serialize,
|
|
24
|
+
)
|
|
19
25
|
|
|
20
26
|
_OPERATIONS = Literal[
|
|
21
27
|
"evaluate", "simplify", "expand", "factor",
|
|
@@ -32,7 +38,42 @@ class SymbolicMathParams(BaseModel):
|
|
|
32
38
|
"'x**2 + 3*x - 4', 'sin(x)*cos(x)', or 'x**2 - 4 = 0' for solve."
|
|
33
39
|
),
|
|
34
40
|
)
|
|
35
|
-
variable: str = Field(
|
|
41
|
+
variable: str = Field(
|
|
42
|
+
"x",
|
|
43
|
+
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
|
44
|
+
description=(
|
|
45
|
+
"The ONE variable to solve for / differentiate / integrate / "
|
|
46
|
+
"expand around. A single name, not a list: for a system, put "
|
|
47
|
+
"the extra equations in `equations` and the solver takes its "
|
|
48
|
+
"symbols from the system itself."
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
# `sympy.Symbol` accepts any string, so an unvalidated value
|
|
52
|
+
# became a symbol that CANNOT occur in the parsed expression, and every
|
|
53
|
+
# operation taking a variable then had a well-defined wrong answer:
|
|
54
|
+
# `solve('x**2 - 4 = 0', variable='x, y')` returned `[]` and
|
|
55
|
+
# `differentiate('x**3', variable='x, y')` returned `'0'`, neither
|
|
56
|
+
# distinguishable in the result from a correct empty solution set or a
|
|
57
|
+
# correct zero derivative. The schema is where its neighbours are
|
|
58
|
+
# already validated (`order` ge=1, `precision` le=50), and the trigger
|
|
59
|
+
# was the schema's own invitation: the sibling field `equations` IS a
|
|
60
|
+
# list, and the solve-a-system path ignores `variable` entirely -- so
|
|
61
|
+
# the one operation where a caller has a reason to name two variables
|
|
62
|
+
# is the one where this field is unused (batch 143).
|
|
63
|
+
#
|
|
64
|
+
# `pattern` is a field the pinned `google-genai` `types.Schema`
|
|
65
|
+
# carries, checked rather than assumed, so this does not widen the
|
|
66
|
+
# GOOGLE-branch problem that is still open.
|
|
67
|
+
symbols: Optional[list[str]] = Field(
|
|
68
|
+
None,
|
|
69
|
+
description=(
|
|
70
|
+
"Other multi-character symbol names in the expression, e.g. "
|
|
71
|
+
"['force', 'mass']. `variable` is always protected; any OTHER "
|
|
72
|
+
"multi-character name is read as a product of single letters "
|
|
73
|
+
"unless listed here ('mass' becomes a*m*s**2). Single letters "
|
|
74
|
+
"and names containing an underscore are safe without this."
|
|
75
|
+
),
|
|
76
|
+
)
|
|
36
77
|
equations: Optional[list[str]] = Field(
|
|
37
78
|
None, description="For 'solve' with a system: additional equations alongside `expression`"
|
|
38
79
|
)
|
|
@@ -59,11 +100,23 @@ TOOL_SCHEMA = {
|
|
|
59
100
|
|
|
60
101
|
|
|
61
102
|
def run(params: dict) -> dict:
|
|
62
|
-
|
|
103
|
+
try:
|
|
104
|
+
parsed = parse_params(SymbolicMathParams, params)
|
|
105
|
+
except MathParseError as e:
|
|
106
|
+
return {"error": str(e)}
|
|
107
|
+
# `variable` PLUS whatever the caller declared, on EVERY parsed field.
|
|
108
|
+
# Before batch 131 this tool passed `symbols=` on two of its six parses,
|
|
109
|
+
# the one name a caller could protect survived and any other name in the
|
|
110
|
+
# same expression was rewritten into a product of single letters --
|
|
111
|
+
# `integrate('force*distance', variable='distance')` answered about
|
|
112
|
+
# `c*e*f*o*r` without saying so.
|
|
113
|
+
rewritten: set = set()
|
|
114
|
+
syms = [parsed.variable] + [s for s in (parsed.symbols or [])
|
|
115
|
+
if s != parsed.variable]
|
|
63
116
|
|
|
64
117
|
try:
|
|
65
118
|
var = sympy.Symbol(parsed.variable)
|
|
66
|
-
expr = safe_parse(parsed.expression, symbols=
|
|
119
|
+
expr = safe_parse(parsed.expression, symbols=syms, rewritten=rewritten)
|
|
67
120
|
op = parsed.operation
|
|
68
121
|
|
|
69
122
|
if op == "evaluate":
|
|
@@ -83,23 +136,28 @@ def run(params: dict) -> dict:
|
|
|
83
136
|
|
|
84
137
|
elif op == "integrate":
|
|
85
138
|
if parsed.lower_bound is not None and parsed.upper_bound is not None:
|
|
86
|
-
lo = safe_parse(parsed.lower_bound
|
|
87
|
-
|
|
139
|
+
lo = safe_parse(parsed.lower_bound, symbols=syms,
|
|
140
|
+
rewritten=rewritten)
|
|
141
|
+
hi = safe_parse(parsed.upper_bound, symbols=syms,
|
|
142
|
+
rewritten=rewritten)
|
|
88
143
|
result = sympy.integrate(expr, (var, lo, hi))
|
|
89
144
|
else:
|
|
90
145
|
result = sympy.integrate(expr, var)
|
|
91
146
|
|
|
92
147
|
elif op == "limit":
|
|
93
|
-
point = safe_parse(parsed.point
|
|
148
|
+
point = (safe_parse(parsed.point, symbols=syms, rewritten=rewritten)
|
|
149
|
+
if parsed.point is not None else 0)
|
|
94
150
|
result = sympy.limit(expr, var, point, dir=parsed.direction)
|
|
95
151
|
|
|
96
152
|
elif op == "series":
|
|
97
|
-
point = safe_parse(parsed.point
|
|
153
|
+
point = (safe_parse(parsed.point, symbols=syms, rewritten=rewritten)
|
|
154
|
+
if parsed.point is not None else 0)
|
|
98
155
|
result = expr.series(var, point, parsed.order).removeO()
|
|
99
156
|
|
|
100
157
|
elif op == "solve":
|
|
101
158
|
if parsed.equations:
|
|
102
|
-
extra = [safe_parse(e, symbols=
|
|
159
|
+
extra = [safe_parse(e, symbols=syms, rewritten=rewritten)
|
|
160
|
+
for e in parsed.equations]
|
|
103
161
|
all_exprs = [expr] + extra
|
|
104
162
|
symbols_found = sorted(
|
|
105
163
|
set().union(*[e.free_symbols for e in all_exprs]), key=str
|
|
@@ -116,4 +174,16 @@ def run(params: dict) -> dict:
|
|
|
116
174
|
except Exception as e:
|
|
117
175
|
return {"error": f"Computation failed: {e}"}
|
|
118
176
|
|
|
119
|
-
|
|
177
|
+
# H6, one batch late (batch 143). `serialize` is the SECOND place this function can fail
|
|
178
|
+
# and it used to sit outside every try, so a correct answer merely too
|
|
179
|
+
# large to print left `run()` as a bare ValueError and was reported as
|
|
180
|
+
# a harness bug with a traceback. A tail handler rather than one try
|
|
181
|
+
# round the whole body, which is the shape `discrete_math` -- the one
|
|
182
|
+
# tool H6 reached -- already uses.
|
|
183
|
+
try:
|
|
184
|
+
out: dict = {"result": serialize(result, what=op), "operation": op}
|
|
185
|
+
except MathParseError as e:
|
|
186
|
+
return {"error": str(e)}
|
|
187
|
+
if rewritten:
|
|
188
|
+
out["warnings"] = _split_warnings(rewritten)
|
|
189
|
+
return out
|
package/tools/builtin/todo.py
CHANGED
|
@@ -29,7 +29,6 @@ conversation, not a person.
|
|
|
29
29
|
|
|
30
30
|
import config
|
|
31
31
|
|
|
32
|
-
|
|
33
32
|
TOOL_SCHEMA = {
|
|
34
33
|
"name": "todo_write",
|
|
35
34
|
"description": (
|
|
@@ -76,8 +75,11 @@ def _normalise(raw) -> tuple:
|
|
|
76
75
|
"""Validate one list. Returns (items, error) -- exactly one is truthy.
|
|
77
76
|
|
|
78
77
|
Every failure is an error result the model can correct, never an
|
|
79
|
-
exception
|
|
80
|
-
|
|
78
|
+
exception -- tools/base.py carries the one copy of why, under "WHAT A
|
|
79
|
+
CALLABLE REGISTERED BELOW MAY RAISE". The short version is not the one
|
|
80
|
+
this used to give: a raise here is CONTAINED, and what an error result
|
|
81
|
+
buys is a message the model can act on instead of a generic wrapper
|
|
82
|
+
(batch 146).
|
|
81
83
|
"""
|
|
82
84
|
if not isinstance(raw, (list, tuple)):
|
|
83
85
|
return None, ("todo_write requires 'items' to be a list of "
|
|
@@ -1,151 +1,329 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Web search tool for the agent harness, using DuckDuckGo exclusively via the
|
|
3
|
-
`ddgs` package.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from __future__ import annotations
|
|
7
|
-
|
|
8
|
-
import logging
|
|
9
|
-
from typing import Optional
|
|
10
|
-
|
|
11
|
-
from
|
|
12
|
-
from
|
|
13
|
-
|
|
14
|
-
from safety.policy_enforcement import is_url_permitted
|
|
15
|
-
from tools.builtin._net_common import TTLCache
|
|
16
|
-
|
|
17
|
-
logger = logging.getLogger(__name__)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
# ---- Schema exposed to the LLM -------------------------------------------
|
|
21
|
-
|
|
22
|
-
class WebSearchParams(BaseModel):
|
|
23
|
-
query: str = Field(..., description="Search query", min_length=1, max_length=400)
|
|
24
|
-
num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
#
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
#
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
"""
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
1
|
+
"""
|
|
2
|
+
Web search tool for the agent harness, using DuckDuckGo exclusively via the
|
|
3
|
+
`ddgs` package.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
from ddgs import DDGS
|
|
12
|
+
from pydantic import BaseModel, Field, field_validator
|
|
13
|
+
|
|
14
|
+
from safety.policy_enforcement import is_url_permitted
|
|
15
|
+
from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# ---- Schema exposed to the LLM -------------------------------------------
|
|
21
|
+
|
|
22
|
+
class WebSearchParams(BaseModel):
|
|
23
|
+
query: str = Field(..., description="Search query", min_length=1, max_length=400)
|
|
24
|
+
num_results: int = Field(5, ge=1, le=10, description="Number of results to return")
|
|
25
|
+
page: int = Field(
|
|
26
|
+
1,
|
|
27
|
+
ge=1,
|
|
28
|
+
description=(
|
|
29
|
+
"Which page of results to return (1-based). Use the page named "
|
|
30
|
+
"in a result's message to see further results for the same "
|
|
31
|
+
"query."
|
|
32
|
+
),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
@field_validator("query")
|
|
36
|
+
@classmethod
|
|
37
|
+
def strip_query(cls, value: str) -> str:
|
|
38
|
+
value = value.strip()
|
|
39
|
+
if not value:
|
|
40
|
+
raise ValueError("query cannot be empty")
|
|
41
|
+
return value
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
TOOL_SCHEMA = {
|
|
45
|
+
"name": "web_search",
|
|
46
|
+
"description": (
|
|
47
|
+
"Search the web for current information. Use this for facts that may "
|
|
48
|
+
"have changed recently, current events, or anything you're not "
|
|
49
|
+
"fully confident about."
|
|
50
|
+
),
|
|
51
|
+
"input_schema": WebSearchParams.model_json_schema(),
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ---- Normalized result shape ---------------------------------------------
|
|
56
|
+
|
|
57
|
+
class SearchResult(BaseModel):
|
|
58
|
+
title: str
|
|
59
|
+
url: str
|
|
60
|
+
snippet: str
|
|
61
|
+
published: Optional[str] = None # ddgs's text search doesn't return a
|
|
62
|
+
# publish date, so this stays None here
|
|
63
|
+
# ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
|
|
64
|
+
# where the rest is. Before §51 a cut snippet and a short one were
|
|
65
|
+
# indistinguishable, so "the source does not mention X" and "the first
|
|
66
|
+
# 300 characters do not mention X" read the same to the model.
|
|
67
|
+
snippet_truncated: bool = False
|
|
68
|
+
# Batch 156 (PG8's shape, one field over). The snippet had a cap and a
|
|
69
|
+
# flag and the title beside it had neither, so a 5,000-character title
|
|
70
|
+
# went to the model whole while the snippet under it was cut to 300 --
|
|
71
|
+
# measured, in the same result. FLAG AND NO ROUTE, which is the
|
|
72
|
+
# asymmetry `arxiv.py` already argues for the same field: a title past
|
|
73
|
+
# the cap is a malformed result rather than a long one, and the route
|
|
74
|
+
# to the page is the `url` sitting next to it either way.
|
|
75
|
+
title_truncated: bool = False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class WebSearchError(Exception):
|
|
79
|
+
"""No longer raised by this module -- exhausted retries return an
|
|
80
|
+
error dict instead, so a transient search outage cannot fail a
|
|
81
|
+
ten-pass research run. Kept as part of this module's public surface."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---- Policy / tuning knobs -------------------------------------------------
|
|
85
|
+
|
|
86
|
+
MAX_SNIPPET_CHARS = 300
|
|
87
|
+
|
|
88
|
+
# Batch 156. 300 is the number `arxiv.py` settled for the same kind of
|
|
89
|
+
# value, measured there over sixty real entries whose longest title was
|
|
90
|
+
# 188 characters. NOT imported from it: two constants with one reason each
|
|
91
|
+
# is the choice `_MAX_AUTHORS` already argues for one file over, and these
|
|
92
|
+
# two bound titles from two different providers.
|
|
93
|
+
#
|
|
94
|
+
# No equivalent measurement was taken for web titles in this batch, and
|
|
95
|
+
# saying so is the honest version. What justifies the number here is that
|
|
96
|
+
# it is the one already settled for a title, and a title is the same kind
|
|
97
|
+
# of value whichever provider hands it over; the HTML spec puts no bound on
|
|
98
|
+
# `<title>` at all.
|
|
99
|
+
MAX_TITLE_CHARS = 300
|
|
100
|
+
|
|
101
|
+
REQUEST_TIMEOUT_S = 8.0
|
|
102
|
+
MAX_RETRIES = 2
|
|
103
|
+
CACHE_TTL_S = 300
|
|
104
|
+
|
|
105
|
+
# One instance per tool, never shared: an equal key string from an
|
|
106
|
+
# arXiv query and a web query would otherwise collide.
|
|
107
|
+
_cache = TTLCache(CACHE_TTL_S)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# ---- Provider call ----------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
|
|
113
|
+
"""
|
|
114
|
+
The one function that actually talks to DuckDuckGo. DDGS() is
|
|
115
|
+
instantiated fresh per call (cheap) and used as a context manager so
|
|
116
|
+
its underlying HTTP session gets cleaned up automatically afterward.
|
|
117
|
+
|
|
118
|
+
`page` is ROADMAP_v3 §51 (PG3's sibling). The record assumed this tool
|
|
119
|
+
had no paging to expose and only a snippet cap to fix; MEASURED (batch
|
|
120
|
+
105), `ddgs 9.14.4` takes `page: int = 1` on its search path and
|
|
121
|
+
accepted it live. So the provider implemented paging here too, and this
|
|
122
|
+
tool was spending the parameter on a default.
|
|
123
|
+
|
|
124
|
+
ALSO MEASURED, and the reason the message this feeds is worded the way
|
|
125
|
+
it is: page 2 of a live query OVERLAPPED page 1 by one result. DuckDuckGo's
|
|
126
|
+
paging is not a clean partition the way arXiv's `start` is (measured the
|
|
127
|
+
same day: zero overlap), so a page here is "more results", not "the next
|
|
128
|
+
five distinct results", and nothing downstream may assume otherwise.
|
|
129
|
+
"""
|
|
130
|
+
with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
|
|
131
|
+
return ddgs_client.text(query, max_results=num_results, page=page)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _normalize(raw: list[dict]) -> list[SearchResult]:
|
|
135
|
+
"""DDGS's rows as `SearchResult`s, redacted and bounded (§73).
|
|
136
|
+
|
|
137
|
+
EVERY STRING IN HERE WAS WRITTEN BY SOMEBODY ELSE, which is the rule
|
|
138
|
+
PG8/#131 states and the reason all of it goes through one helper now.
|
|
139
|
+
Two things were wrong at once and they interact, so fixing either alone
|
|
140
|
+
would have been worse than fixing neither:
|
|
141
|
+
|
|
142
|
+
REDACT ABOVE THE CUT (RB1, A59). The snippet was cut to 300 characters
|
|
143
|
+
and `registry.dispatch` redacted the piece that was kept. Measured
|
|
144
|
+
(batch 156), a `ghp_` token straddling that cut survived post-hoc
|
|
145
|
+
redaction at 36 of the 39 offsets it could straddle -- the kept prefix
|
|
146
|
+
is not the token, the dropped suffix is gone, and `param_digest`'s
|
|
147
|
+
docstring had already written down why ("truncating first cuts a
|
|
148
|
+
credential below the 20 characters its pattern needs").
|
|
149
|
+
|
|
150
|
+
AND THE CUTS THAT DID NOT EXIST YET. The title had no bound at all, so
|
|
151
|
+
adding one without the ordering above would have MANUFACTURED a second
|
|
152
|
+
straddle site in the field being repaired. Every new cap here is a new
|
|
153
|
+
cut, and every cut needs the redaction above it.
|
|
154
|
+
|
|
155
|
+
TOTAL OVER WHATEVER DDGS RETURNED (A61), which is the half no report
|
|
156
|
+
named. `r.get("title", "(untitled)")` substitutes only for an ABSENT
|
|
157
|
+
key, and `{"title": null}` is a present one -- the lesson
|
|
158
|
+
`tools/base.py` states for tool arguments, arriving from the provider
|
|
159
|
+
end. Measured, a single row with `title: null` raised `ValidationError`
|
|
160
|
+
out of here, which `run` caught as a failed attempt, RETRIED TWICE,
|
|
161
|
+
and then reported as "search failed after 3 attempts" -- ten good
|
|
162
|
+
results lost to one malformed one, plus two pointless re-queries.
|
|
163
|
+
`arxiv._parse_atom_feed` had the identical defect and fixed it in batch
|
|
164
|
+
117 ("SKIPPED, NOT RAISED ... Raising here aborted the whole search
|
|
165
|
+
over one malformed entry"); the cure was one file over the whole time.
|
|
166
|
+
"""
|
|
167
|
+
third = ThirdPartyText("web_search")
|
|
168
|
+
out: list[SearchResult] = []
|
|
169
|
+
for r in raw:
|
|
170
|
+
if not isinstance(r, dict):
|
|
171
|
+
logger.warning(
|
|
172
|
+
"web_search skipped a result: the provider returned a %s "
|
|
173
|
+
"where a row was expected.", type(r).__name__)
|
|
174
|
+
continue
|
|
175
|
+
|
|
176
|
+
url, withheld = third.url(r.get("href"))
|
|
177
|
+
if withheld:
|
|
178
|
+
# NOT SILENT (#49). The length is the fact; the URL is not
|
|
179
|
+
# echoed, because a URL is one of the things the redactor is
|
|
180
|
+
# looking for userinfo inside.
|
|
181
|
+
logger.warning(
|
|
182
|
+
"web_search dropped a result: its URL is longer than the "
|
|
183
|
+
"%d characters fetch_url accepts, so it is a route this "
|
|
184
|
+
"harness would refuse to take.", MAX_URL_CHARS)
|
|
185
|
+
if url is None:
|
|
186
|
+
# A result is a (title, url, snippet) triple whose url is how
|
|
187
|
+
# the model reaches it. With the url gone there is nothing to
|
|
188
|
+
# cite and nothing to fetch, so this is the same call the
|
|
189
|
+
# policy filter below already makes -- and it is the call
|
|
190
|
+
# `_parse_atom_feed` makes for an entry with no id.
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
# `resolve=False` on purpose. This filters results; it does not
|
|
194
|
+
# fetch them, and a DNS lookup per hit is real latency for a check
|
|
195
|
+
# whose only failure mode is showing the model a URL that fetch_url
|
|
196
|
+
# will refuse anyway. An IP LITERAL is still checked, in every
|
|
197
|
+
# spelling the platform's resolver reads as a number -- a result
|
|
198
|
+
# pointing straight at 169.254.169.254 is dropped here rather than
|
|
199
|
+
# offered and then denied (#54).
|
|
200
|
+
#
|
|
201
|
+
# "EVERY SPELLING" SINCE BATCH 213 (#207, register item 53). Batch
|
|
202
|
+
# 156 narrowed this comment to dotted and bracketed literals,
|
|
203
|
+
# because `http://2130706433/`, `http://0x7f.0.0.1/` and
|
|
204
|
+
# `http://0x7F000001/` -- all 127.0.0.1 -- passed: `ipaddress`
|
|
205
|
+
# parses none of them. is_url_permitted now asks the resolver with
|
|
206
|
+
# AI_NUMERICHOST, which reads numbers and looks nothing up, so they
|
|
207
|
+
# are dropped here as fetch_url would refuse them.
|
|
208
|
+
#
|
|
209
|
+
# ABOVE THE POLICY CHECK AND BELOW THE BOUND, deliberately: the
|
|
210
|
+
# string checked here is the string the model is offered, so a
|
|
211
|
+
# redaction that rewrote the host cannot slip past the check that
|
|
212
|
+
# was run on the host before it.
|
|
213
|
+
if is_url_permitted(url, resolve=False):
|
|
214
|
+
continue
|
|
215
|
+
|
|
216
|
+
title, title_truncated = third.text(r.get("title"), MAX_TITLE_CHARS)
|
|
217
|
+
if title is None:
|
|
218
|
+
title = "(untitled)"
|
|
219
|
+
snippet, snippet_truncated = third.text(
|
|
220
|
+
r.get("body"), MAX_SNIPPET_CHARS)
|
|
221
|
+
if snippet is None:
|
|
222
|
+
snippet = ""
|
|
223
|
+
|
|
224
|
+
out.append(
|
|
225
|
+
SearchResult(
|
|
226
|
+
title=title,
|
|
227
|
+
url=url,
|
|
228
|
+
snippet=snippet,
|
|
229
|
+
published=None,
|
|
230
|
+
snippet_truncated=snippet_truncated,
|
|
231
|
+
title_truncated=title_truncated,
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
third.report()
|
|
235
|
+
return out
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _kept(results: list, asked: int) -> list:
|
|
239
|
+
"""`results`, cut to the number the caller asked for (A62).
|
|
240
|
+
|
|
241
|
+
THE SCHEMA ALREADY PROMISED THIS and nothing enforced it.
|
|
242
|
+
`num_results` is advertised `le=10` and passed to the provider as
|
|
243
|
+
`max_results`, and a provider is free to ignore it -- measured (batch
|
|
244
|
+
156), a call asking for 5 returned every one of 5,000 rows it was
|
|
245
|
+
handed. So this is a documented bound being kept rather than a new
|
|
246
|
+
policy, and it is the LIST axis of the same question batch 156 asked
|
|
247
|
+
of each value: per-value caps multiplied by an unbounded row count are
|
|
248
|
+
not a bound.
|
|
249
|
+
|
|
250
|
+
`_call_ddgs` is where the finding wanted a byte ceiling, and it cannot
|
|
251
|
+
have one: the library buffers the response and owns the socket, so
|
|
252
|
+
nothing above it sees bytes. What this layer CAN bound is what it
|
|
253
|
+
keeps, which is the quantity that reaches the model.
|
|
254
|
+
|
|
255
|
+
NO FLAG, deliberately, where every other cut in this batch gets one.
|
|
256
|
+
Nothing is being withheld from the caller: it asked for `asked`
|
|
257
|
+
results and it is getting `asked` results. The over-delivery is a
|
|
258
|
+
provider fact rather than a truncation, so it goes to the log, where
|
|
259
|
+
the person debugging a provider can see it.
|
|
260
|
+
"""
|
|
261
|
+
if len(results) > asked:
|
|
262
|
+
logger.warning(
|
|
263
|
+
"web_search kept %d of %d results: the provider returned more "
|
|
264
|
+
"than the %d asked for.", asked, len(results), asked)
|
|
265
|
+
return results[:asked]
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def run(params: dict) -> dict:
|
|
269
|
+
parsed = WebSearchParams(**params)
|
|
270
|
+
# §51 (PG9). `page` is part of the key for the reason `start` is on the
|
|
271
|
+
# arXiv side: it changes the response, so a key without it serves page 1
|
|
272
|
+
# to every request for page 2 and does it silently.
|
|
273
|
+
cache_key = f"{parsed.query}|{parsed.num_results}|{parsed.page}"
|
|
274
|
+
|
|
275
|
+
results = _cache.get(cache_key)
|
|
276
|
+
if results is not None:
|
|
277
|
+
logger.info("web_search cache hit for %r", parsed.query)
|
|
278
|
+
else:
|
|
279
|
+
last_exc: Optional[Exception] = None
|
|
280
|
+
results = []
|
|
281
|
+
for attempt in range(MAX_RETRIES + 1):
|
|
282
|
+
try:
|
|
283
|
+
raw = _call_ddgs(parsed.query, parsed.num_results, parsed.page)
|
|
284
|
+
results = _kept(_normalize(raw), parsed.num_results)
|
|
285
|
+
_cache.set(cache_key, results)
|
|
286
|
+
break
|
|
287
|
+
except Exception as e: # Simpler to try again then elaborate on every error type
|
|
288
|
+
last_exc = e
|
|
289
|
+
logger.warning(
|
|
290
|
+
"web_search attempt %s/%s failed for %r: %s",
|
|
291
|
+
attempt + 1, MAX_RETRIES + 1, parsed.query, e)
|
|
292
|
+
else:
|
|
293
|
+
# RETURNED, not raised -- same change and same reason as
|
|
294
|
+
# arxiv_search. A search provider being unreachable is a
|
|
295
|
+
# result the model can work around; raising made one
|
|
296
|
+
# transient failure in one pass fail a whole ten-pass
|
|
297
|
+
# research run. fetch_url was already the odd one out for
|
|
298
|
+
# doing this correctly.
|
|
299
|
+
return {
|
|
300
|
+
"error": f"search failed after {MAX_RETRIES + 1} "
|
|
301
|
+
f"attempts: {last_exc}"
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
if not results:
|
|
305
|
+
if parsed.page > 1:
|
|
306
|
+
return {
|
|
307
|
+
"results": [], "result_count": 0, "page": parsed.page,
|
|
308
|
+
"message": (
|
|
309
|
+
f"No results on page {parsed.page}. The provider "
|
|
310
|
+
f"returned nothing further for this query."
|
|
311
|
+
),
|
|
312
|
+
}
|
|
313
|
+
return {"results": [], "result_count": 0, "message": "No results found."}
|
|
314
|
+
|
|
315
|
+
out = {
|
|
316
|
+
"results": [r.model_dump() for r in results],
|
|
317
|
+
"result_count": len(results),
|
|
318
|
+
"page": parsed.page,
|
|
319
|
+
}
|
|
320
|
+
# PG4. DuckDuckGo reports no total, so the honest message names the next
|
|
321
|
+
# page without claiming how many there are -- and says the pages overlap,
|
|
322
|
+
# because measured they do. A full page is the only signal that more may
|
|
323
|
+
# exist; a short one is where the provider stopped.
|
|
324
|
+
if len(results) >= parsed.num_results:
|
|
325
|
+
out["message"] = (
|
|
326
|
+
f"Page {parsed.page}. Use page={parsed.page + 1} for more "
|
|
327
|
+
f"results; pages may overlap by a result or two."
|
|
328
|
+
)
|
|
329
|
+
return out
|