@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/tools/builtin/arxiv.py
CHANGED
|
@@ -1,223 +1,681 @@
|
|
|
1
|
-
"""
|
|
2
|
-
tools/builtin/arxiv.py
|
|
3
|
-
|
|
4
|
-
Searches arXiv for academic papers by keyword and optional subject
|
|
5
|
-
category, using arXiv's public query API. No API key needed. Useful
|
|
6
|
-
alongside web_search/fetch_url for grounding claims in published research
|
|
7
|
-
during the source_grounding pass.
|
|
8
|
-
|
|
9
|
-
The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
|
|
10
|
-
largely translating that into the same normalized result shape the other
|
|
11
|
-
tools use.
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
from __future__ import annotations
|
|
15
|
-
|
|
16
|
-
import logging
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
#
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
#
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
#
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
#
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
"
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/arxiv.py
|
|
3
|
+
|
|
4
|
+
Searches arXiv for academic papers by keyword and optional subject
|
|
5
|
+
category, using arXiv's public query API. No API key needed. Useful
|
|
6
|
+
alongside web_search/fetch_url for grounding claims in published research
|
|
7
|
+
during the source_grounding pass.
|
|
8
|
+
|
|
9
|
+
The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
|
|
10
|
+
largely translating that into the same normalized result shape the other
|
|
11
|
+
tools use.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
from typing import Optional
|
|
18
|
+
|
|
19
|
+
# TECHNICAL_DEBT 18, batch 116. This feed is attacker-influenced -- arXiv
|
|
20
|
+
# echoes the query back and the entries are third-party text -- and stdlib
|
|
21
|
+
# ElementTree expands internal entities, which is Bandit B405/B314 and was
|
|
22
|
+
# an accepted risk for the CI rollout rather than a decision that it was
|
|
23
|
+
# safe. `defusedxml.ElementTree` is the drop-in: `fromstring` refuses
|
|
24
|
+
# declared entities and external references, and it re-exports the SAME
|
|
25
|
+
# `ParseError` object as the stdlib, so the empty-redirect-body path below
|
|
26
|
+
# and the retry loop's except clause are unaffected. Measured, not assumed:
|
|
27
|
+
# stdlib expands `&lol2;` to "lollollol" where this raises
|
|
28
|
+
# EntitiesForbidden.
|
|
29
|
+
import defusedxml.ElementTree as ET
|
|
30
|
+
import httpx
|
|
31
|
+
from defusedxml import DefusedXmlException
|
|
32
|
+
from pydantic import BaseModel, Field, field_validator
|
|
33
|
+
|
|
34
|
+
from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
# HTTPS, and it is load-bearing. arXiv 301-redirects the http:// form, and
|
|
39
|
+
# three things then combine into a hard failure that looks nothing like a
|
|
40
|
+
# redirect: httpx does NOT follow redirects by default (unlike requests),
|
|
41
|
+
# raise_for_status() only raises on >= 400 so a 301 sails through, and the
|
|
42
|
+
# redirect body is empty -- so ET.fromstring("") raises ParseError, the
|
|
43
|
+
# retry loop runs three identical attempts, and the tool reports "arXiv
|
|
44
|
+
# search failed after 3 attempts". Verified against the live API.
|
|
45
|
+
ARXIV_API_URL = "https://export.arxiv.org/api/query"
|
|
46
|
+
ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
|
|
47
|
+
|
|
48
|
+
# ROADMAP_v3 §51 (PG3). arXiv answers in OpenSearch's paging elements as
|
|
49
|
+
# well as Atom's entries, and they are what let a result say how many papers
|
|
50
|
+
# the query has rather than only that it has more. Measured (batch 105):
|
|
51
|
+
# `all:transformer attention` reports totalResults=261387, with startIndex
|
|
52
|
+
# and itemsPerPage echoing the request.
|
|
53
|
+
OPENSEARCH_NS = {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"}
|
|
54
|
+
|
|
55
|
+
_VALID_SORT_BY = {"relevance", "lastUpdatedDate", "submittedDate"}
|
|
56
|
+
|
|
57
|
+
MAX_SUMMARY_CHARS = 600
|
|
58
|
+
REQUEST_TIMEOUT_S = 10.0
|
|
59
|
+
MAX_RETRIES = 2
|
|
60
|
+
CACHE_TTL_S = 300
|
|
61
|
+
|
|
62
|
+
# ROADMAP_v2 §31 (H7), #55, applied to the second tool that had the defect.
|
|
63
|
+
# `httpx.get(...).text` buffers the ENTIRE body before anything looks at it,
|
|
64
|
+
# and fetch_url.py's own comment rules out the shortcut: a Content-Length
|
|
65
|
+
# check is not the fix, because a hostile server can omit it or lie about
|
|
66
|
+
# it, so the only thing that bounds this is refusing to read past a cap.
|
|
67
|
+
# This feed is attacker-influenced -- the module header says so -- and this
|
|
68
|
+
# was the last `httpx.get` in the project.
|
|
69
|
+
#
|
|
70
|
+
# DERIVED FROM A MEASUREMENT, and the first derivation was wrong. It guessed
|
|
71
|
+
# "roughly 100 KB of Atom per hyperauthorship entry" and therefore 2 MiB for
|
|
72
|
+
# twenty of them; measured live, `all:LHCb observation branching fraction`
|
|
73
|
+
# answers max_results=20 with 838,560 bytes carrying 13,859 <author>
|
|
74
|
+
# elements, about 42 KB an entry, and four consecutive pages of it stayed
|
|
75
|
+
# under 839 KB. So 2 MiB clears the heaviest LEGITIMATE feed this tool can
|
|
76
|
+
# ask for by 2.5x -- which is the number that matters, because a cap under
|
|
77
|
+
# it is a FALSE REFUSAL: the tool reporting it read nothing for a query
|
|
78
|
+
# arXiv answered correctly.
|
|
79
|
+
#
|
|
80
|
+
# Loose next to fetch_url's 65,536 on purpose. That tool's URL is chosen by
|
|
81
|
+
# the model and can point anywhere; this endpoint is fixed and its response
|
|
82
|
+
# shape is known, so the bound's job here is the refusal rather than the
|
|
83
|
+
# tightness.
|
|
84
|
+
MAX_FEED_BYTES = 2_097_152
|
|
85
|
+
|
|
86
|
+
# ROADMAP_v3 §51 (PG8) for the shape, #131 for the reason: these bound
|
|
87
|
+
# model-facing text from a source nobody here wrote. The abstract was the
|
|
88
|
+
# only field anyone capped and it is not the field that grew -- measured
|
|
89
|
+
# (TECHNICAL_DEBT 41, re-measured in batch 126) a `str(result)` of 223,191
|
|
90
|
+
# characters, of which 204,871 (91.8%) were the author list, with one entry
|
|
91
|
+
# carrying 2,832 authors and eleven of twenty over the cap below. Everything
|
|
92
|
+
# the tool already bounds ceilings at about 16,900 characters even at
|
|
93
|
+
# max_results=20, so the capped fields could never have accounted for it.
|
|
94
|
+
#
|
|
95
|
+
# 25 is `core/reasoning/scholar.py`'s `_MAX_AUTHORS`, answering the same
|
|
96
|
+
# question the same way -- enough to find the strongest name on a paper
|
|
97
|
+
# without carrying a hyperauthorship record. NOT imported: tools/ does not
|
|
98
|
+
# import core/reasoning/, and one constant reaching across two layers would
|
|
99
|
+
# be worse than two constants with one reason each.
|
|
100
|
+
_MAX_AUTHORS = 25
|
|
101
|
+
|
|
102
|
+
# Measured over sixty real entries from three queries: the longest title was
|
|
103
|
+
# 188 characters ("Measurements of the Higgs boson production and decay
|
|
104
|
+
# rates..."). So this is inert for every real paper and bounds the malformed
|
|
105
|
+
# one -- which is why both of these get a FLAG and no recovery route. See
|
|
106
|
+
# `_parse_atom_feed`.
|
|
107
|
+
MAX_TITLE_CHARS = 300
|
|
108
|
+
MAX_CATEGORIES = 20
|
|
109
|
+
|
|
110
|
+
# Batch 156 (ROADMAP_v3 §73). THE CAPS ABOVE BOUND THE LISTS AND NOT THE
|
|
111
|
+
# VALUES IN THEM, which is the "which axis" question of batch 126 asked
|
|
112
|
+
# again one level down: `authors` kept 25 of them and `categories` 20, and
|
|
113
|
+
# a single `<name>` or a single `term` could be any length at all.
|
|
114
|
+
# Measured (batch 156) through the real parser, a 5,000-character author
|
|
115
|
+
# name, a 5,000-character category term and a 5,021-character pdf href all
|
|
116
|
+
# came back byte-for-byte while the 5,000-character title in the same entry
|
|
117
|
+
# was cut to 300 with its flag -- the control that proves the probe was
|
|
118
|
+
# reaching the caps.
|
|
119
|
+
#
|
|
120
|
+
# DERIVED, each from the longest legitimate shape of the value it bounds,
|
|
121
|
+
# because a cap that trims an ordinary entry turns its own flag into noise:
|
|
122
|
+
#
|
|
123
|
+
# * an author name is a person or a collaboration byline. The longest
|
|
124
|
+
# plausible is the latter ("The ATLAS and CMS Collaborations" is 32
|
|
125
|
+
# characters); 200 clears every real one several times over.
|
|
126
|
+
# * a category `term` comes from arXiv's published taxonomy, whose
|
|
127
|
+
# longest member is `cond-mat.stat-mech` at 18 characters. The longest
|
|
128
|
+
# appearing anywhere in this repository is `physics.gen-ph` at 14.
|
|
129
|
+
# * an id is `2401.00001v1` (12) in the new scheme or
|
|
130
|
+
# `math.GT/0309136v2` (17) in the old -- 17 is the longest of either
|
|
131
|
+
# form present in this repository, and 64 is not reachable by a
|
|
132
|
+
# well-formed one.
|
|
133
|
+
_MAX_AUTHOR_CHARS = 200
|
|
134
|
+
_MAX_CATEGORY_CHARS = 40
|
|
135
|
+
_MAX_ID_CHARS = 64
|
|
136
|
+
|
|
137
|
+
# One instance per tool, never shared: an equal key string from an
|
|
138
|
+
# arXiv query and a web query would otherwise collide.
|
|
139
|
+
_cache = TTLCache(CACHE_TTL_S)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ---- Schema exposed to the LLM -------------------------------------------
|
|
143
|
+
|
|
144
|
+
class ArxivSearchParams(BaseModel):
|
|
145
|
+
keywords: str = Field(..., description="Search keywords", min_length=1, max_length=400)
|
|
146
|
+
category: Optional[str] = Field(
|
|
147
|
+
None,
|
|
148
|
+
description=(
|
|
149
|
+
"Optional arXiv category code to restrict the search, e.g. "
|
|
150
|
+
"'cs.AI', 'cs.LG', 'physics.gen-ph', 'q-bio.NC'. Omit to search "
|
|
151
|
+
"all categories."
|
|
152
|
+
),
|
|
153
|
+
)
|
|
154
|
+
max_results: int = Field(5, ge=1, le=20, description="Number of results to return")
|
|
155
|
+
start: int = Field(
|
|
156
|
+
0,
|
|
157
|
+
ge=0,
|
|
158
|
+
description=(
|
|
159
|
+
"Number of results to skip before returning any (0-based). Use "
|
|
160
|
+
"the start named in a result's message to page further into a "
|
|
161
|
+
"search that has more papers than were returned."
|
|
162
|
+
),
|
|
163
|
+
)
|
|
164
|
+
sort_by: str = Field(
|
|
165
|
+
"relevance",
|
|
166
|
+
description="How to sort results: 'relevance', 'lastUpdatedDate', or 'submittedDate'",
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
@field_validator("keywords")
|
|
170
|
+
@classmethod
|
|
171
|
+
def strip_keywords(cls, value: str) -> str:
|
|
172
|
+
value = value.strip()
|
|
173
|
+
if not value:
|
|
174
|
+
raise ValueError("keywords cannot be empty")
|
|
175
|
+
return value
|
|
176
|
+
|
|
177
|
+
@field_validator("sort_by")
|
|
178
|
+
@classmethod
|
|
179
|
+
def validate_sort_by(cls, value: str) -> str:
|
|
180
|
+
if value not in _VALID_SORT_BY:
|
|
181
|
+
raise ValueError(f"sort_by must be one of {_VALID_SORT_BY}")
|
|
182
|
+
return value
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
TOOL_SCHEMA = {
|
|
186
|
+
"name": "arxiv_search",
|
|
187
|
+
"description": (
|
|
188
|
+
"Search arXiv for academic papers by keyword and optional subject "
|
|
189
|
+
"category. Returns titles, authors, abstracts, publish dates, and "
|
|
190
|
+
"PDF links -- use this to ground claims in published research."
|
|
191
|
+
),
|
|
192
|
+
"input_schema": ArxivSearchParams.model_json_schema(),
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class ArxivSearchError(Exception):
|
|
197
|
+
"""No longer raised by this module -- exhausted retries return an
|
|
198
|
+
error dict instead, so a transient network failure cannot fail a
|
|
199
|
+
ten-pass research run. Kept because it is part of this module's
|
|
200
|
+
public surface and something outside the repo may catch it."""
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class _FeedTooLarge(Exception):
|
|
204
|
+
"""The feed passed MAX_FEED_BYTES and was dropped UNREAD.
|
|
205
|
+
|
|
206
|
+
PRIVATE, and it never escapes `run()`. Deliberately not
|
|
207
|
+
`ArxivSearchError`: that one is documented above as part of this
|
|
208
|
+
module's public surface and as no longer raised, and quietly starting
|
|
209
|
+
to raise it again would make that docstring false.
|
|
210
|
+
|
|
211
|
+
A separate exception rather than a sentinel return because the two
|
|
212
|
+
outcomes are not the same shape -- every other path returns a feed to
|
|
213
|
+
parse, and this one has nothing to parse. `fetch_url` can `break` and
|
|
214
|
+
keep a truncated body because a cut page of prose is still prose; a cut
|
|
215
|
+
Atom document does not parse at all, so there is nothing to hand back.
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# ---- Query construction -----------------------------------------------------
|
|
220
|
+
|
|
221
|
+
def _build_search_query(keywords: str, category: Optional[str]) -> str:
|
|
222
|
+
"""
|
|
223
|
+
arXiv's query syntax uses field prefixes (ti:, abs:, au:, cat:, all:)
|
|
224
|
+
combined with AND/OR/ANDNOT. `all:` searches title, abstract, and
|
|
225
|
+
author. httpx handles URL-encoding the spaces/quotes once this string
|
|
226
|
+
is passed as a params value, so no manual encoding is needed here.
|
|
227
|
+
"""
|
|
228
|
+
if category:
|
|
229
|
+
return f"cat:{category} AND all:{keywords}"
|
|
230
|
+
return f"all:{keywords}"
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---- Provider call ----------------------------------------------------------
|
|
234
|
+
|
|
235
|
+
def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
|
|
236
|
+
start: int = 0) -> str:
|
|
237
|
+
"""The raw Atom XML, or `_FeedTooLarge` if it would not fit the bound.
|
|
238
|
+
|
|
239
|
+
STREAMED since batch 126 (§31 H7), so the connection is dropped inside
|
|
240
|
+
the `with` the moment the cap is passed rather than after the whole body
|
|
241
|
+
is already in memory. Returns `str` as it always did, so every test that
|
|
242
|
+
patches this function is unaffected by the change underneath it.
|
|
243
|
+
"""
|
|
244
|
+
with httpx.stream(
|
|
245
|
+
"GET",
|
|
246
|
+
ARXIV_API_URL,
|
|
247
|
+
params={
|
|
248
|
+
"search_query": search_query,
|
|
249
|
+
# §51 (PG3). Was hardcoded to 0, which made every page of every
|
|
250
|
+
# search the first one: the provider had implemented paging all
|
|
251
|
+
# along and this tool spent the parameter on a constant.
|
|
252
|
+
"start": start,
|
|
253
|
+
"max_results": max_results,
|
|
254
|
+
"sortBy": sort_by,
|
|
255
|
+
"sortOrder": "descending",
|
|
256
|
+
},
|
|
257
|
+
headers={"User-Agent": "agent-harness-arxiv-tool/1.0 (research pipeline)"},
|
|
258
|
+
timeout=REQUEST_TIMEOUT_S,
|
|
259
|
+
# Belt and braces beside the https:// above. If arXiv moves the
|
|
260
|
+
# endpoint again, following the redirect keeps this working
|
|
261
|
+
# instead of failing as an unparseable empty body.
|
|
262
|
+
follow_redirects=True,
|
|
263
|
+
) as response:
|
|
264
|
+
# Streamed, so status and headers are available before the body --
|
|
265
|
+
# the same ordering fetch_url relies on.
|
|
266
|
+
response.raise_for_status()
|
|
267
|
+
chunks = []
|
|
268
|
+
total = 0
|
|
269
|
+
for chunk in response.iter_bytes():
|
|
270
|
+
total += len(chunk)
|
|
271
|
+
if total > MAX_FEED_BYTES:
|
|
272
|
+
raise _FeedTooLarge(total)
|
|
273
|
+
chunks.append(chunk)
|
|
274
|
+
encoding = getattr(response, "encoding", None) or "utf-8"
|
|
275
|
+
|
|
276
|
+
# errors="replace", and it is not laziness. A bare decode raises
|
|
277
|
+
# UnicodeDecodeError on a feed that is not what its header claims, and
|
|
278
|
+
# that is a ValueError -- which NEITHER except clause in `run()` covers,
|
|
279
|
+
# so it would escape as a crash on precisely the hostile path the rest
|
|
280
|
+
# of this module is careful about. Replacing instead makes a nonsense
|
|
281
|
+
# body fail as a ParseError, which IS handled and IS retried.
|
|
282
|
+
#
|
|
283
|
+
# §51's PG5 concern about a cut multi-byte character does not apply:
|
|
284
|
+
# these bytes are always a COMPLETE document, because an incomplete one
|
|
285
|
+
# was refused above rather than truncated.
|
|
286
|
+
return b"".join(chunks).decode(encoding, errors="replace")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
# ---- Atom XML parsing --------------------------------------------------
|
|
290
|
+
|
|
291
|
+
def _total_results(root) -> Optional[int]:
|
|
292
|
+
"""How many papers the query has, from OpenSearch's own element (PG3).
|
|
293
|
+
|
|
294
|
+
Optional rather than assumed: the element is arXiv's to emit, and a
|
|
295
|
+
result that reported a total it had invented would be worse than one
|
|
296
|
+
that reports none.
|
|
297
|
+
"""
|
|
298
|
+
element = root.find("opensearch:totalResults", OPENSEARCH_NS)
|
|
299
|
+
if element is None or not (element.text or "").strip():
|
|
300
|
+
return None
|
|
301
|
+
try:
|
|
302
|
+
return int(element.text.strip())
|
|
303
|
+
except ValueError:
|
|
304
|
+
return None
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _text_of(parent, path) -> Optional[str]:
|
|
308
|
+
"""One child element's stripped text, or None.
|
|
309
|
+
|
|
310
|
+
EVERY lookup in `_parse_atom_feed` goes through this, because the six
|
|
311
|
+
ways that function used to raise were six slightly different spellings
|
|
312
|
+
of the same omission (batch 117, TECHNICAL_DEBT 33):
|
|
313
|
+
|
|
314
|
+
* a MISSING element -- `find()` returns None and `.text` raises
|
|
315
|
+
AttributeError. Three of the six.
|
|
316
|
+
* a PRESENT but EMPTY element -- `<title></title>` has `.text` of
|
|
317
|
+
None, so `.strip()` raises AttributeError and `[:10]` raises
|
|
318
|
+
TypeError. The remaining three, and they are the ones a reader
|
|
319
|
+
skips, because `summary` looked guarded by `or ""` and `published`
|
|
320
|
+
looked guarded by `is not None`. Neither covers both halves.
|
|
321
|
+
|
|
322
|
+
The feed is attacker-influenced -- this module's header says so, and
|
|
323
|
+
batch 116 added a parser that refuses hostile ENTITIES one function
|
|
324
|
+
away -- so its STRUCTURE cannot be trusted either.
|
|
325
|
+
"""
|
|
326
|
+
element = parent.find(path, ATOM_NS)
|
|
327
|
+
if element is None or element.text is None:
|
|
328
|
+
return None
|
|
329
|
+
stripped = element.text.strip()
|
|
330
|
+
return stripped or None
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _parse_atom_feed(xml_text: str) -> tuple:
|
|
334
|
+
"""(entries, how many the query has in total).
|
|
335
|
+
|
|
336
|
+
The total is OpenSearch's, not a count of what came back -- the point
|
|
337
|
+
of it is to say what was NOT returned, which a count of what was cannot.
|
|
338
|
+
"""
|
|
339
|
+
root = ET.fromstring(xml_text)
|
|
340
|
+
entries = root.findall("atom:entry", ATOM_NS)
|
|
341
|
+
total = _total_results(root)
|
|
342
|
+
|
|
343
|
+
third = ThirdPartyText("arxiv_search")
|
|
344
|
+
results = []
|
|
345
|
+
for index, entry in enumerate(entries):
|
|
346
|
+
raw_id = _text_of(entry, "atom:id")
|
|
347
|
+
title_text = _text_of(entry, "atom:title")
|
|
348
|
+
if raw_id is None or title_text is None:
|
|
349
|
+
# SKIPPED, NOT RAISED (batch 117). An id and a title are what
|
|
350
|
+
# make an entry citable; without either there is nothing for
|
|
351
|
+
# the model to ground a claim on, so the entry is dropped and
|
|
352
|
+
# the rest of the page is still returned. Raising here aborted
|
|
353
|
+
# the whole search over one malformed entry, and the raise did
|
|
354
|
+
# not even reach `run`'s handlers -- AttributeError is neither
|
|
355
|
+
# a ParseError nor a DefusedXmlException.
|
|
356
|
+
#
|
|
357
|
+
# The index, not the id: the id is what might be missing.
|
|
358
|
+
logger.warning(
|
|
359
|
+
"arxiv_search skipped entry %d of %d: no %s",
|
|
360
|
+
index + 1, len(entries),
|
|
361
|
+
"id" if raw_id is None else "title")
|
|
362
|
+
continue
|
|
363
|
+
raw_arxiv_id = (raw_id.split("/abs/")[-1]
|
|
364
|
+
if "/abs/" in raw_id else raw_id)
|
|
365
|
+
# THE ID IS NOT JUST DISPLAYED, it is the input to the two recovery
|
|
366
|
+
# URLs below, so an unbounded one is three unbounded values wearing
|
|
367
|
+
# one name -- measured (batch 156), a 5,000-character `<id>` made
|
|
368
|
+
# `full_summary_url` 5,022 characters long. Cut and flagged like any
|
|
369
|
+
# other text (A58); what the cut costs is handled where the routes
|
|
370
|
+
# are built, because a route assembled from half an id is a route to
|
|
371
|
+
# nowhere and that is the one thing a route must not be.
|
|
372
|
+
arxiv_id, id_truncated = third.text(raw_arxiv_id, _MAX_ID_CHARS)
|
|
373
|
+
|
|
374
|
+
title, title_truncated = third.text(
|
|
375
|
+
title_text.replace("\n", " "), MAX_TITLE_CHARS)
|
|
376
|
+
summary, summary_truncated = third.text(
|
|
377
|
+
(_text_of(entry, "atom:summary") or "").replace("\n", " "),
|
|
378
|
+
MAX_SUMMARY_CHARS)
|
|
379
|
+
# §51 (PG8/PG4). The cap stays -- it bounds model-facing text from a
|
|
380
|
+
# source nobody here wrote, which is #131's lesson -- but a value cut
|
|
381
|
+
# at it now SAYS it was cut. Until §51 a 600-character abstract and
|
|
382
|
+
# one that happened to be 600 characters long were the same thing to
|
|
383
|
+
# a reader, so the model could not tell an abstract it had all of
|
|
384
|
+
# from one it had the first two thirds of.
|
|
385
|
+
#
|
|
386
|
+
# REDACTED ABOVE THE CUT SINCE BATCH 156 (RB1, A59), which is the
|
|
387
|
+
# half §51 got the wrong way round. Both of these were sliced and
|
|
388
|
+
# `registry.dispatch` redacted the slice; measured, a `ghp_` token
|
|
389
|
+
# straddling either cut survived post-hoc redaction at 36 of its 39
|
|
390
|
+
# possible offsets. `fetch_url` had already fixed the same ordering
|
|
391
|
+
# for its own body, and `param_digest` had written down the reason
|
|
392
|
+
# before either of them. The flags now count REDACTED characters,
|
|
393
|
+
# which is RB3's consequence: redaction can shrink a value back
|
|
394
|
+
# under the cap, and when it does the cut is withholding nothing.
|
|
395
|
+
|
|
396
|
+
# `is not None` on the ELEMENT was not enough: `<published></published>`
|
|
397
|
+
# is present with `.text` of None, and `None[:10]` is a TypeError
|
|
398
|
+
# rather than an AttributeError -- the same omission wearing a
|
|
399
|
+
# different exception type, which is why both halves go through
|
|
400
|
+
# one helper now.
|
|
401
|
+
published_text = _text_of(entry, "atom:published")
|
|
402
|
+
published = published_text[:10] if published_text else None
|
|
403
|
+
|
|
404
|
+
# A NAMELESS ELEMENT IS DROPPED RATHER THAN INSERTED AS None, and
|
|
405
|
+
# this is one rule for BOTH lists below (TECHNICAL_DEBT 47). Both
|
|
406
|
+
# come from the same attacker-influenced feed and both reach the
|
|
407
|
+
# model the same way, so they must not disagree about it: a `None`
|
|
408
|
+
# in a list rendered into a citation reads as an author called None
|
|
409
|
+
# and as a category called None.
|
|
410
|
+
#
|
|
411
|
+
# `categories` kept its None until batch 127. The rule was stated
|
|
412
|
+
# here, two lines above the list that did not follow it, which is
|
|
413
|
+
# why the comment now names both instead of being copied.
|
|
414
|
+
#
|
|
415
|
+
# BOUNDED SINCE BATCH 126, and `authors` is the field
|
|
416
|
+
# TECHNICAL_DEBT 41 was about without naming it. The count is kept
|
|
417
|
+
# before the cut because `author_count` reports the real total --
|
|
418
|
+
# the point of the key is that the model can tell a 2,832-author
|
|
419
|
+
# paper from a 25-author one, which the truncated list alone cannot
|
|
420
|
+
# say.
|
|
421
|
+
#
|
|
422
|
+
# DROPPED BEFORE THE CUT for both, so the cap counts values the
|
|
423
|
+
# model can actually use: filtering after the slice would let a
|
|
424
|
+
# nameless element consume one of the 25 or 20 places.
|
|
425
|
+
#
|
|
426
|
+
# AND THE CAP ON EACH VALUE, not only on the count (batch 156). The
|
|
427
|
+
# two bounds answer different questions and the lists had only the
|
|
428
|
+
# first: 25 authors of 5,000 characters each is 125,000 characters
|
|
429
|
+
# of model-facing text that every count-based cap reports as
|
|
430
|
+
# bounded. Each value is cut through the same helper as the title
|
|
431
|
+
# and the abstract, so it is redacted first for the same reason.
|
|
432
|
+
all_authors = [
|
|
433
|
+
name for name in (
|
|
434
|
+
third.text(_text_of(author, "atom:name"),
|
|
435
|
+
_MAX_AUTHOR_CHARS)[0]
|
|
436
|
+
for author in entry.findall("atom:author", ATOM_NS)
|
|
437
|
+
) if name
|
|
438
|
+
]
|
|
439
|
+
author_count = len(all_authors)
|
|
440
|
+
authors = all_authors[:_MAX_AUTHORS]
|
|
441
|
+
all_categories = [
|
|
442
|
+
term for term in (
|
|
443
|
+
third.text(c.get("term"), _MAX_CATEGORY_CHARS)[0]
|
|
444
|
+
for c in entry.findall("atom:category", ATOM_NS)
|
|
445
|
+
) if term
|
|
446
|
+
]
|
|
447
|
+
categories = all_categories[:MAX_CATEGORIES]
|
|
448
|
+
|
|
449
|
+
# DROPPED, NOT CUT (A58), and this is the field that makes the
|
|
450
|
+
# distinction concrete. `pdf_url` exists to be fetched; cut to a cap
|
|
451
|
+
# it becomes a fetchable-looking string that is not the paper, and
|
|
452
|
+
# the model spends a tool call finding that out. Measured, a
|
|
453
|
+
# 5,021-character href passed through byte-for-byte before this.
|
|
454
|
+
# `MAX_URL_CHARS` is fetch_url's own limit, so anything past it is a
|
|
455
|
+
# route this harness would refuse anyway.
|
|
456
|
+
pdf_url, pdf_withheld = None, False
|
|
457
|
+
for link in entry.findall("atom:link", ATOM_NS):
|
|
458
|
+
if link.get("title") == "pdf":
|
|
459
|
+
pdf_url, pdf_withheld = third.url(link.get("href"),
|
|
460
|
+
MAX_URL_CHARS)
|
|
461
|
+
|
|
462
|
+
entry_result = {
|
|
463
|
+
"arxiv_id": arxiv_id,
|
|
464
|
+
"title": title,
|
|
465
|
+
"authors": authors,
|
|
466
|
+
"summary": summary,
|
|
467
|
+
"published": published,
|
|
468
|
+
"categories": categories,
|
|
469
|
+
"pdf_url": pdf_url,
|
|
470
|
+
}
|
|
471
|
+
if summary_truncated:
|
|
472
|
+
# PG4's whole complaint: a truncated value that says nothing
|
|
473
|
+
# about how to recover the rest is what sends the model to the
|
|
474
|
+
# shell. The abs page is the route, and it is a URL `fetch_url`
|
|
475
|
+
# will take.
|
|
476
|
+
entry_result["summary_truncated"] = True
|
|
477
|
+
if not id_truncated:
|
|
478
|
+
entry_result["full_summary_url"] = (
|
|
479
|
+
f"https://arxiv.org/abs/{arxiv_id}")
|
|
480
|
+
if author_count > _MAX_AUTHORS:
|
|
481
|
+
# PG8's shape exactly, including the recovery route PG4 asks
|
|
482
|
+
# for: the abs page lists every author and `fetch_url` will
|
|
483
|
+
# take it. `full_authors_url` rather than reusing
|
|
484
|
+
# `full_summary_url` -- same URL, but a key named for the
|
|
485
|
+
# summary carrying the author route is a name that lies, and
|
|
486
|
+
# the summary's key is absent whenever the abstract fit.
|
|
487
|
+
entry_result["authors_truncated"] = True
|
|
488
|
+
entry_result["author_count"] = author_count
|
|
489
|
+
# GUARDED ON THE ID, like `full_summary_url` above and for the
|
|
490
|
+
# same reason (A58): the route is assembled from `arxiv_id`, so
|
|
491
|
+
# an id that was cut yields a URL that 404s. PG4 asks for a
|
|
492
|
+
# route to what the model WANTED; a broken one answers a
|
|
493
|
+
# question nobody asked, and `authors_truncated` still says the
|
|
494
|
+
# list was cut.
|
|
495
|
+
if not id_truncated:
|
|
496
|
+
entry_result["full_authors_url"] = (
|
|
497
|
+
f"https://arxiv.org/abs/{arxiv_id}")
|
|
498
|
+
# THESE TWO GET A FLAG AND NO ROUTE, and the asymmetry is deliberate
|
|
499
|
+
# rather than an omission to be tidied up later. #49's rule is that
|
|
500
|
+
# nothing is altered silently, so the flag is owed. But PG4 exists so
|
|
501
|
+
# the model can reach what it WANTED, and a title past 300 characters
|
|
502
|
+
# or a twenty-first category is a malformed feed rather than a long
|
|
503
|
+
# paper -- measured, the longest title in sixty real entries was 188.
|
|
504
|
+
# Pointing the model at the abs page of a hostile entry would be
|
|
505
|
+
# answering a question nobody asked.
|
|
506
|
+
if title_truncated:
|
|
507
|
+
entry_result["title_truncated"] = True
|
|
508
|
+
if len(all_categories) > MAX_CATEGORIES:
|
|
509
|
+
entry_result["categories_truncated"] = True
|
|
510
|
+
# THE THREE FLAGS BATCH 156 OWES, same rule (#49): nothing here is
|
|
511
|
+
# altered silently. `id_truncated` is the one that matters most,
|
|
512
|
+
# because a cut id is a citation that does not resolve and the model
|
|
513
|
+
# has no other way to know.
|
|
514
|
+
if id_truncated:
|
|
515
|
+
entry_result["id_truncated"] = True
|
|
516
|
+
if pdf_withheld:
|
|
517
|
+
entry_result["pdf_url_truncated"] = True
|
|
518
|
+
results.append(entry_result)
|
|
519
|
+
|
|
520
|
+
third.report()
|
|
521
|
+
return results, total
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
# ---- Entry point called by the tool dispatcher -----------------------------
|
|
525
|
+
|
|
526
|
+
def run(params: dict) -> dict:
|
|
527
|
+
parsed = ArxivSearchParams(**params)
|
|
528
|
+
# §51 (PG9). `start` is part of the key because it changes the response.
|
|
529
|
+
# Without it page 2 of a search would be served page 1 out of the cache
|
|
530
|
+
# -- the same class of collision this cache's own docstring warns about
|
|
531
|
+
# between the two tools, arriving instead between two pages of one
|
|
532
|
+
# query, and silent in exactly the same way.
|
|
533
|
+
cache_key = (f"{parsed.keywords}|{parsed.category}|{parsed.max_results}"
|
|
534
|
+
f"|{parsed.sort_by}|{parsed.start}")
|
|
535
|
+
|
|
536
|
+
cached = _cache.get(cache_key)
|
|
537
|
+
total: Optional[int] = None
|
|
538
|
+
if cached is not None:
|
|
539
|
+
logger.info("arxiv_search cache hit for %r", parsed.keywords)
|
|
540
|
+
results, total = cached
|
|
541
|
+
else:
|
|
542
|
+
search_query = _build_search_query(parsed.keywords, parsed.category)
|
|
543
|
+
last_exc: Optional[Exception] = None
|
|
544
|
+
results = []
|
|
545
|
+
for attempt in range(MAX_RETRIES + 1):
|
|
546
|
+
try:
|
|
547
|
+
xml_text = _call_arxiv_api(search_query, parsed.max_results,
|
|
548
|
+
parsed.sort_by, parsed.start)
|
|
549
|
+
results, total = _parse_atom_feed(xml_text)
|
|
550
|
+
# A62. `max_results` is advertised `le=20` and reaches the
|
|
551
|
+
# provider, which is free to ignore it -- measured (batch
|
|
552
|
+
# 156), a feed of 5,000 entries against a request for 5
|
|
553
|
+
# returned all 5,000. The per-entry caps above are
|
|
554
|
+
# multiplied by this number, so bounding them without
|
|
555
|
+
# bounding it is not a bound. `total` is unaffected and
|
|
556
|
+
# still reports what the QUERY has, which is the figure
|
|
557
|
+
# that says what was not returned.
|
|
558
|
+
if len(results) > parsed.max_results:
|
|
559
|
+
logger.warning(
|
|
560
|
+
"arxiv_search kept %d of %d entries: the feed "
|
|
561
|
+
"carried more than the %d asked for.",
|
|
562
|
+
parsed.max_results, len(results), parsed.max_results)
|
|
563
|
+
results = results[:parsed.max_results]
|
|
564
|
+
_cache.set(cache_key, (results, total))
|
|
565
|
+
break
|
|
566
|
+
except _FeedTooLarge:
|
|
567
|
+
# NOT RETRIED, for the reason spelled out one clause down
|
|
568
|
+
# for a declared entity: a feed this large is this large
|
|
569
|
+
# again, so two more round trips reach the same place and
|
|
570
|
+
# then report "failed after 3 attempts", which describes a
|
|
571
|
+
# network problem that is not there.
|
|
572
|
+
#
|
|
573
|
+
# Returned rather than raised, like every other dead end in
|
|
574
|
+
# this function: a search tool that cannot answer is a
|
|
575
|
+
# result the model can work around.
|
|
576
|
+
logger.warning(
|
|
577
|
+
"arxiv_search dropped a feed over %d bytes for %r "
|
|
578
|
+
"(max_results=%d, start=%d)",
|
|
579
|
+
MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
|
|
580
|
+
parsed.start)
|
|
581
|
+
# "more than", not a size. `total` stopped at the first
|
|
582
|
+
# chunk past the cap, so printing it would be an instrument
|
|
583
|
+
# reporting a magnitude it did not measure -- the
|
|
584
|
+
# instrument-that-lies shape batches 118 and 119 each paid
|
|
585
|
+
# for once. The lever IS named, because the model controls
|
|
586
|
+
# it and a smaller page really does fit.
|
|
587
|
+
return {
|
|
588
|
+
"error": "arXiv returned more than %d bytes for this "
|
|
589
|
+
"query, so nothing was read from it. Retry "
|
|
590
|
+
"with a smaller max_results."
|
|
591
|
+
% MAX_FEED_BYTES
|
|
592
|
+
}
|
|
593
|
+
except DefusedXmlException as e:
|
|
594
|
+
# NOT folded into the tuple below, and that is the point.
|
|
595
|
+
# A malformed body is transient and worth three attempts;
|
|
596
|
+
# a feed that DECLARES an entity is not -- it refuses the
|
|
597
|
+
# same way every time, so retrying spends two more round
|
|
598
|
+
# trips to reach the same place and then reports "failed
|
|
599
|
+
# after 3 attempts", which describes a network problem
|
|
600
|
+
# that is not there.
|
|
601
|
+
#
|
|
602
|
+
# Returned rather than raised, for the reason the retry
|
|
603
|
+
# exhaustion below is returned: a search tool that cannot
|
|
604
|
+
# answer is a result the model can work around. These do
|
|
605
|
+
# NOT subclass ParseError -- they are ValueError -- so
|
|
606
|
+
# without this clause they would propagate out of run().
|
|
607
|
+
logger.warning(
|
|
608
|
+
"arxiv_search refused a hostile feed for %r: %s",
|
|
609
|
+
parsed.keywords, e)
|
|
610
|
+
# THE FEED'S OWN TEXT STOPS HERE (batch 116's review).
|
|
611
|
+
# `e`'s repr carries the entity's `system_id` verbatim from
|
|
612
|
+
# the feed, so interpolating `{e}` would put a string the
|
|
613
|
+
# ATTACKER chose into the model's context -- the one place
|
|
614
|
+
# §32's untrusted-content paragraph exists to bound. The
|
|
615
|
+
# log above keeps the whole exception, because it is
|
|
616
|
+
# operator-facing and the formatter redaction sink guards
|
|
617
|
+
# app.log; the model gets the category and, where the
|
|
618
|
+
# exception carries one, the entity NAME, which XML's Name
|
|
619
|
+
# production cannot make into a sentence.
|
|
620
|
+
#
|
|
621
|
+
# getattr, not `e.name`: ExternalReferenceForbidden has no
|
|
622
|
+
# `.name`, and an AttributeError raised while building an
|
|
623
|
+
# error message would replace a handled refusal with a
|
|
624
|
+
# crash on the hostile path.
|
|
625
|
+
declared = getattr(e, "name", None)
|
|
626
|
+
named = (" -- it declared the entity %r"
|
|
627
|
+
% str(declared)[:40]) if declared else ""
|
|
628
|
+
return {
|
|
629
|
+
"error": "arXiv returned a feed this parser refuses to "
|
|
630
|
+
"expand, so nothing was read from it: %s%s"
|
|
631
|
+
% (type(e).__name__, named)
|
|
632
|
+
}
|
|
633
|
+
except (httpx.HTTPError, ET.ParseError) as e:
|
|
634
|
+
last_exc = e
|
|
635
|
+
logger.warning(
|
|
636
|
+
"arxiv_search attempt %s/%s failed for %r: %s",
|
|
637
|
+
attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
|
|
638
|
+
else:
|
|
639
|
+
# RETURNED, not raised. A search tool that cannot reach its
|
|
640
|
+
# provider is a result the model can work around -- try
|
|
641
|
+
# web_search, or say the claim could not be grounded. Raising
|
|
642
|
+
# made one transient network failure, in one pass, fail an
|
|
643
|
+
# entire ten-pass research run; fetch_url has always returned
|
|
644
|
+
# an error dict here and arxiv_search was the outlier.
|
|
645
|
+
#
|
|
646
|
+
# dispatch() now contains a raise from any tool as well, so
|
|
647
|
+
# this is the message rather than the mechanism: the model
|
|
648
|
+
# gets something specific instead of a generic wrapper.
|
|
649
|
+
return {
|
|
650
|
+
"error": f"arXiv search failed after {MAX_RETRIES + 1} "
|
|
651
|
+
f"attempts: {last_exc}"
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
if not results:
|
|
655
|
+
if parsed.start:
|
|
656
|
+
# Distinguishable from "this query has nothing", which is what
|
|
657
|
+
# the bare message would have said about a page past the end.
|
|
658
|
+
return {
|
|
659
|
+
"results": [], "result_count": 0, "start": parsed.start,
|
|
660
|
+
"message": (
|
|
661
|
+
f"No papers at start={parsed.start}. The search has "
|
|
662
|
+
f"{total} in total." if total is not None else
|
|
663
|
+
f"No papers at start={parsed.start}, which is past the "
|
|
664
|
+
f"end of this search."
|
|
665
|
+
),
|
|
666
|
+
}
|
|
667
|
+
return {"results": [], "result_count": 0, "message": "No papers found."}
|
|
668
|
+
|
|
669
|
+
result = {"results": results, "result_count": len(results),
|
|
670
|
+
"start": parsed.start}
|
|
671
|
+
|
|
672
|
+
# PG4: say how to get the rest, not merely that there is a rest.
|
|
673
|
+
next_start = parsed.start + len(results)
|
|
674
|
+
if total is not None:
|
|
675
|
+
result["total_results"] = total
|
|
676
|
+
if next_start < total:
|
|
677
|
+
result["message"] = (
|
|
678
|
+
f"Returned papers {parsed.start + 1}-{next_start} of {total}. "
|
|
679
|
+
f"Use start={next_start} to see more."
|
|
680
|
+
)
|
|
681
|
+
return result
|