@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,681 +1,681 @@
1
- """
2
- tools/builtin/arxiv.py
3
-
4
- Searches arXiv for academic papers by keyword and optional subject
5
- category, using arXiv's public query API. No API key needed. Useful
6
- alongside web_search/fetch_url for grounding claims in published research
7
- during the source_grounding pass.
8
-
9
- The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
10
- largely translating that into the same normalized result shape the other
11
- tools use.
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- import logging
17
- from typing import Optional
18
-
19
- # TECHNICAL_DEBT 18, batch 116. This feed is attacker-influenced -- arXiv
20
- # echoes the query back and the entries are third-party text -- and stdlib
21
- # ElementTree expands internal entities, which is Bandit B405/B314 and was
22
- # an accepted risk for the CI rollout rather than a decision that it was
23
- # safe. `defusedxml.ElementTree` is the drop-in: `fromstring` refuses
24
- # declared entities and external references, and it re-exports the SAME
25
- # `ParseError` object as the stdlib, so the empty-redirect-body path below
26
- # and the retry loop's except clause are unaffected. Measured, not assumed:
27
- # stdlib expands `&lol2;` to "lollollol" where this raises
28
- # EntitiesForbidden.
29
- import defusedxml.ElementTree as ET
30
- import httpx
31
- from defusedxml import DefusedXmlException
32
- from pydantic import BaseModel, Field, field_validator
33
-
34
- from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
35
-
36
- logger = logging.getLogger(__name__)
37
-
38
- # HTTPS, and it is load-bearing. arXiv 301-redirects the http:// form, and
39
- # three things then combine into a hard failure that looks nothing like a
40
- # redirect: httpx does NOT follow redirects by default (unlike requests),
41
- # raise_for_status() only raises on >= 400 so a 301 sails through, and the
42
- # redirect body is empty -- so ET.fromstring("") raises ParseError, the
43
- # retry loop runs three identical attempts, and the tool reports "arXiv
44
- # search failed after 3 attempts". Verified against the live API.
45
- ARXIV_API_URL = "https://export.arxiv.org/api/query"
46
- ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
47
-
48
- # ROADMAP_v3 §51 (PG3). arXiv answers in OpenSearch's paging elements as
49
- # well as Atom's entries, and they are what let a result say how many papers
50
- # the query has rather than only that it has more. Measured (batch 105):
51
- # `all:transformer attention` reports totalResults=261387, with startIndex
52
- # and itemsPerPage echoing the request.
53
- OPENSEARCH_NS = {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"}
54
-
55
- _VALID_SORT_BY = {"relevance", "lastUpdatedDate", "submittedDate"}
56
-
57
- MAX_SUMMARY_CHARS = 600
58
- REQUEST_TIMEOUT_S = 10.0
59
- MAX_RETRIES = 2
60
- CACHE_TTL_S = 300
61
-
62
- # ROADMAP_v2 §31 (H7), #55, applied to the second tool that had the defect.
63
- # `httpx.get(...).text` buffers the ENTIRE body before anything looks at it,
64
- # and fetch_url.py's own comment rules out the shortcut: a Content-Length
65
- # check is not the fix, because a hostile server can omit it or lie about
66
- # it, so the only thing that bounds this is refusing to read past a cap.
67
- # This feed is attacker-influenced -- the module header says so -- and this
68
- # was the last `httpx.get` in the project.
69
- #
70
- # DERIVED FROM A MEASUREMENT, and the first derivation was wrong. It guessed
71
- # "roughly 100 KB of Atom per hyperauthorship entry" and therefore 2 MiB for
72
- # twenty of them; measured live, `all:LHCb observation branching fraction`
73
- # answers max_results=20 with 838,560 bytes carrying 13,859 <author>
74
- # elements, about 42 KB an entry, and four consecutive pages of it stayed
75
- # under 839 KB. So 2 MiB clears the heaviest LEGITIMATE feed this tool can
76
- # ask for by 2.5x -- which is the number that matters, because a cap under
77
- # it is a FALSE REFUSAL: the tool reporting it read nothing for a query
78
- # arXiv answered correctly.
79
- #
80
- # Loose next to fetch_url's 65,536 on purpose. That tool's URL is chosen by
81
- # the model and can point anywhere; this endpoint is fixed and its response
82
- # shape is known, so the bound's job here is the refusal rather than the
83
- # tightness.
84
- MAX_FEED_BYTES = 2_097_152
85
-
86
- # ROADMAP_v3 §51 (PG8) for the shape, #131 for the reason: these bound
87
- # model-facing text from a source nobody here wrote. The abstract was the
88
- # only field anyone capped and it is not the field that grew -- measured
89
- # (TECHNICAL_DEBT 41, re-measured in batch 126) a `str(result)` of 223,191
90
- # characters, of which 204,871 (91.8%) were the author list, with one entry
91
- # carrying 2,832 authors and eleven of twenty over the cap below. Everything
92
- # the tool already bounds ceilings at about 16,900 characters even at
93
- # max_results=20, so the capped fields could never have accounted for it.
94
- #
95
- # 25 is `core/reasoning/scholar.py`'s `_MAX_AUTHORS`, answering the same
96
- # question the same way -- enough to find the strongest name on a paper
97
- # without carrying a hyperauthorship record. NOT imported: tools/ does not
98
- # import core/reasoning/, and one constant reaching across two layers would
99
- # be worse than two constants with one reason each.
100
- _MAX_AUTHORS = 25
101
-
102
- # Measured over sixty real entries from three queries: the longest title was
103
- # 188 characters ("Measurements of the Higgs boson production and decay
104
- # rates..."). So this is inert for every real paper and bounds the malformed
105
- # one -- which is why both of these get a FLAG and no recovery route. See
106
- # `_parse_atom_feed`.
107
- MAX_TITLE_CHARS = 300
108
- MAX_CATEGORIES = 20
109
-
110
- # Batch 156 (ROADMAP_v3 §73). THE CAPS ABOVE BOUND THE LISTS AND NOT THE
111
- # VALUES IN THEM, which is the "which axis" question of batch 126 asked
112
- # again one level down: `authors` kept 25 of them and `categories` 20, and
113
- # a single `<name>` or a single `term` could be any length at all.
114
- # Measured (batch 156) through the real parser, a 5,000-character author
115
- # name, a 5,000-character category term and a 5,021-character pdf href all
116
- # came back byte-for-byte while the 5,000-character title in the same entry
117
- # was cut to 300 with its flag -- the control that proves the probe was
118
- # reaching the caps.
119
- #
120
- # DERIVED, each from the longest legitimate shape of the value it bounds,
121
- # because a cap that trims an ordinary entry turns its own flag into noise:
122
- #
123
- # * an author name is a person or a collaboration byline. The longest
124
- # plausible is the latter ("The ATLAS and CMS Collaborations" is 32
125
- # characters); 200 clears every real one several times over.
126
- # * a category `term` comes from arXiv's published taxonomy, whose
127
- # longest member is `cond-mat.stat-mech` at 18 characters. The longest
128
- # appearing anywhere in this repository is `physics.gen-ph` at 14.
129
- # * an id is `2401.00001v1` (12) in the new scheme or
130
- # `math.GT/0309136v2` (17) in the old -- 17 is the longest of either
131
- # form present in this repository, and 64 is not reachable by a
132
- # well-formed one.
133
- _MAX_AUTHOR_CHARS = 200
134
- _MAX_CATEGORY_CHARS = 40
135
- _MAX_ID_CHARS = 64
136
-
137
- # One instance per tool, never shared: an equal key string from an
138
- # arXiv query and a web query would otherwise collide.
139
- _cache = TTLCache(CACHE_TTL_S)
140
-
141
-
142
- # ---- Schema exposed to the LLM -------------------------------------------
143
-
144
- class ArxivSearchParams(BaseModel):
145
- keywords: str = Field(..., description="Search keywords", min_length=1, max_length=400)
146
- category: Optional[str] = Field(
147
- None,
148
- description=(
149
- "Optional arXiv category code to restrict the search, e.g. "
150
- "'cs.AI', 'cs.LG', 'physics.gen-ph', 'q-bio.NC'. Omit to search "
151
- "all categories."
152
- ),
153
- )
154
- max_results: int = Field(5, ge=1, le=20, description="Number of results to return")
155
- start: int = Field(
156
- 0,
157
- ge=0,
158
- description=(
159
- "Number of results to skip before returning any (0-based). Use "
160
- "the start named in a result's message to page further into a "
161
- "search that has more papers than were returned."
162
- ),
163
- )
164
- sort_by: str = Field(
165
- "relevance",
166
- description="How to sort results: 'relevance', 'lastUpdatedDate', or 'submittedDate'",
167
- )
168
-
169
- @field_validator("keywords")
170
- @classmethod
171
- def strip_keywords(cls, value: str) -> str:
172
- value = value.strip()
173
- if not value:
174
- raise ValueError("keywords cannot be empty")
175
- return value
176
-
177
- @field_validator("sort_by")
178
- @classmethod
179
- def validate_sort_by(cls, value: str) -> str:
180
- if value not in _VALID_SORT_BY:
181
- raise ValueError(f"sort_by must be one of {_VALID_SORT_BY}")
182
- return value
183
-
184
-
185
- TOOL_SCHEMA = {
186
- "name": "arxiv_search",
187
- "description": (
188
- "Search arXiv for academic papers by keyword and optional subject "
189
- "category. Returns titles, authors, abstracts, publish dates, and "
190
- "PDF links -- use this to ground claims in published research."
191
- ),
192
- "input_schema": ArxivSearchParams.model_json_schema(),
193
- }
194
-
195
-
196
- class ArxivSearchError(Exception):
197
- """No longer raised by this module -- exhausted retries return an
198
- error dict instead, so a transient network failure cannot fail a
199
- ten-pass research run. Kept because it is part of this module's
200
- public surface and something outside the repo may catch it."""
201
-
202
-
203
- class _FeedTooLarge(Exception):
204
- """The feed passed MAX_FEED_BYTES and was dropped UNREAD.
205
-
206
- PRIVATE, and it never escapes `run()`. Deliberately not
207
- `ArxivSearchError`: that one is documented above as part of this
208
- module's public surface and as no longer raised, and quietly starting
209
- to raise it again would make that docstring false.
210
-
211
- A separate exception rather than a sentinel return because the two
212
- outcomes are not the same shape -- every other path returns a feed to
213
- parse, and this one has nothing to parse. `fetch_url` can `break` and
214
- keep a truncated body because a cut page of prose is still prose; a cut
215
- Atom document does not parse at all, so there is nothing to hand back.
216
- """
217
-
218
-
219
- # ---- Query construction -----------------------------------------------------
220
-
221
- def _build_search_query(keywords: str, category: Optional[str]) -> str:
222
- """
223
- arXiv's query syntax uses field prefixes (ti:, abs:, au:, cat:, all:)
224
- combined with AND/OR/ANDNOT. `all:` searches title, abstract, and
225
- author. httpx handles URL-encoding the spaces/quotes once this string
226
- is passed as a params value, so no manual encoding is needed here.
227
- """
228
- if category:
229
- return f"cat:{category} AND all:{keywords}"
230
- return f"all:{keywords}"
231
-
232
-
233
- # ---- Provider call ----------------------------------------------------------
234
-
235
- def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
236
- start: int = 0) -> str:
237
- """The raw Atom XML, or `_FeedTooLarge` if it would not fit the bound.
238
-
239
- STREAMED since batch 126 (§31 H7), so the connection is dropped inside
240
- the `with` the moment the cap is passed rather than after the whole body
241
- is already in memory. Returns `str` as it always did, so every test that
242
- patches this function is unaffected by the change underneath it.
243
- """
244
- with httpx.stream(
245
- "GET",
246
- ARXIV_API_URL,
247
- params={
248
- "search_query": search_query,
249
- # §51 (PG3). Was hardcoded to 0, which made every page of every
250
- # search the first one: the provider had implemented paging all
251
- # along and this tool spent the parameter on a constant.
252
- "start": start,
253
- "max_results": max_results,
254
- "sortBy": sort_by,
255
- "sortOrder": "descending",
256
- },
257
- headers={"User-Agent": "agent-harness-arxiv-tool/1.0 (research pipeline)"},
258
- timeout=REQUEST_TIMEOUT_S,
259
- # Belt and braces beside the https:// above. If arXiv moves the
260
- # endpoint again, following the redirect keeps this working
261
- # instead of failing as an unparseable empty body.
262
- follow_redirects=True,
263
- ) as response:
264
- # Streamed, so status and headers are available before the body --
265
- # the same ordering fetch_url relies on.
266
- response.raise_for_status()
267
- chunks = []
268
- total = 0
269
- for chunk in response.iter_bytes():
270
- total += len(chunk)
271
- if total > MAX_FEED_BYTES:
272
- raise _FeedTooLarge(total)
273
- chunks.append(chunk)
274
- encoding = getattr(response, "encoding", None) or "utf-8"
275
-
276
- # errors="replace", and it is not laziness. A bare decode raises
277
- # UnicodeDecodeError on a feed that is not what its header claims, and
278
- # that is a ValueError -- which NEITHER except clause in `run()` covers,
279
- # so it would escape as a crash on precisely the hostile path the rest
280
- # of this module is careful about. Replacing instead makes a nonsense
281
- # body fail as a ParseError, which IS handled and IS retried.
282
- #
283
- # §51's PG5 concern about a cut multi-byte character does not apply:
284
- # these bytes are always a COMPLETE document, because an incomplete one
285
- # was refused above rather than truncated.
286
- return b"".join(chunks).decode(encoding, errors="replace")
287
-
288
-
289
- # ---- Atom XML parsing --------------------------------------------------
290
-
291
- def _total_results(root) -> Optional[int]:
292
- """How many papers the query has, from OpenSearch's own element (PG3).
293
-
294
- Optional rather than assumed: the element is arXiv's to emit, and a
295
- result that reported a total it had invented would be worse than one
296
- that reports none.
297
- """
298
- element = root.find("opensearch:totalResults", OPENSEARCH_NS)
299
- if element is None or not (element.text or "").strip():
300
- return None
301
- try:
302
- return int(element.text.strip())
303
- except ValueError:
304
- return None
305
-
306
-
307
- def _text_of(parent, path) -> Optional[str]:
308
- """One child element's stripped text, or None.
309
-
310
- EVERY lookup in `_parse_atom_feed` goes through this, because the six
311
- ways that function used to raise were six slightly different spellings
312
- of the same omission (batch 117, TECHNICAL_DEBT 33):
313
-
314
- * a MISSING element -- `find()` returns None and `.text` raises
315
- AttributeError. Three of the six.
316
- * a PRESENT but EMPTY element -- `<title></title>` has `.text` of
317
- None, so `.strip()` raises AttributeError and `[:10]` raises
318
- TypeError. The remaining three, and they are the ones a reader
319
- skips, because `summary` looked guarded by `or ""` and `published`
320
- looked guarded by `is not None`. Neither covers both halves.
321
-
322
- The feed is attacker-influenced -- this module's header says so, and
323
- batch 116 added a parser that refuses hostile ENTITIES one function
324
- away -- so its STRUCTURE cannot be trusted either.
325
- """
326
- element = parent.find(path, ATOM_NS)
327
- if element is None or element.text is None:
328
- return None
329
- stripped = element.text.strip()
330
- return stripped or None
331
-
332
-
333
- def _parse_atom_feed(xml_text: str) -> tuple:
334
- """(entries, how many the query has in total).
335
-
336
- The total is OpenSearch's, not a count of what came back -- the point
337
- of it is to say what was NOT returned, which a count of what was cannot.
338
- """
339
- root = ET.fromstring(xml_text)
340
- entries = root.findall("atom:entry", ATOM_NS)
341
- total = _total_results(root)
342
-
343
- third = ThirdPartyText("arxiv_search")
344
- results = []
345
- for index, entry in enumerate(entries):
346
- raw_id = _text_of(entry, "atom:id")
347
- title_text = _text_of(entry, "atom:title")
348
- if raw_id is None or title_text is None:
349
- # SKIPPED, NOT RAISED (batch 117). An id and a title are what
350
- # make an entry citable; without either there is nothing for
351
- # the model to ground a claim on, so the entry is dropped and
352
- # the rest of the page is still returned. Raising here aborted
353
- # the whole search over one malformed entry, and the raise did
354
- # not even reach `run`'s handlers -- AttributeError is neither
355
- # a ParseError nor a DefusedXmlException.
356
- #
357
- # The index, not the id: the id is what might be missing.
358
- logger.warning(
359
- "arxiv_search skipped entry %d of %d: no %s",
360
- index + 1, len(entries),
361
- "id" if raw_id is None else "title")
362
- continue
363
- raw_arxiv_id = (raw_id.split("/abs/")[-1]
364
- if "/abs/" in raw_id else raw_id)
365
- # THE ID IS NOT JUST DISPLAYED, it is the input to the two recovery
366
- # URLs below, so an unbounded one is three unbounded values wearing
367
- # one name -- measured (batch 156), a 5,000-character `<id>` made
368
- # `full_summary_url` 5,022 characters long. Cut and flagged like any
369
- # other text (A58); what the cut costs is handled where the routes
370
- # are built, because a route assembled from half an id is a route to
371
- # nowhere and that is the one thing a route must not be.
372
- arxiv_id, id_truncated = third.text(raw_arxiv_id, _MAX_ID_CHARS)
373
-
374
- title, title_truncated = third.text(
375
- title_text.replace("\n", " "), MAX_TITLE_CHARS)
376
- summary, summary_truncated = third.text(
377
- (_text_of(entry, "atom:summary") or "").replace("\n", " "),
378
- MAX_SUMMARY_CHARS)
379
- # §51 (PG8/PG4). The cap stays -- it bounds model-facing text from a
380
- # source nobody here wrote, which is #131's lesson -- but a value cut
381
- # at it now SAYS it was cut. Until §51 a 600-character abstract and
382
- # one that happened to be 600 characters long were the same thing to
383
- # a reader, so the model could not tell an abstract it had all of
384
- # from one it had the first two thirds of.
385
- #
386
- # REDACTED ABOVE THE CUT SINCE BATCH 156 (RB1, A59), which is the
387
- # half §51 got the wrong way round. Both of these were sliced and
388
- # `registry.dispatch` redacted the slice; measured, a `ghp_` token
389
- # straddling either cut survived post-hoc redaction at 36 of its 39
390
- # possible offsets. `fetch_url` had already fixed the same ordering
391
- # for its own body, and `param_digest` had written down the reason
392
- # before either of them. The flags now count REDACTED characters,
393
- # which is RB3's consequence: redaction can shrink a value back
394
- # under the cap, and when it does the cut is withholding nothing.
395
-
396
- # `is not None` on the ELEMENT was not enough: `<published></published>`
397
- # is present with `.text` of None, and `None[:10]` is a TypeError
398
- # rather than an AttributeError -- the same omission wearing a
399
- # different exception type, which is why both halves go through
400
- # one helper now.
401
- published_text = _text_of(entry, "atom:published")
402
- published = published_text[:10] if published_text else None
403
-
404
- # A NAMELESS ELEMENT IS DROPPED RATHER THAN INSERTED AS None, and
405
- # this is one rule for BOTH lists below (TECHNICAL_DEBT 47). Both
406
- # come from the same attacker-influenced feed and both reach the
407
- # model the same way, so they must not disagree about it: a `None`
408
- # in a list rendered into a citation reads as an author called None
409
- # and as a category called None.
410
- #
411
- # `categories` kept its None until batch 127. The rule was stated
412
- # here, two lines above the list that did not follow it, which is
413
- # why the comment now names both instead of being copied.
414
- #
415
- # BOUNDED SINCE BATCH 126, and `authors` is the field
416
- # TECHNICAL_DEBT 41 was about without naming it. The count is kept
417
- # before the cut because `author_count` reports the real total --
418
- # the point of the key is that the model can tell a 2,832-author
419
- # paper from a 25-author one, which the truncated list alone cannot
420
- # say.
421
- #
422
- # DROPPED BEFORE THE CUT for both, so the cap counts values the
423
- # model can actually use: filtering after the slice would let a
424
- # nameless element consume one of the 25 or 20 places.
425
- #
426
- # AND THE CAP ON EACH VALUE, not only on the count (batch 156). The
427
- # two bounds answer different questions and the lists had only the
428
- # first: 25 authors of 5,000 characters each is 125,000 characters
429
- # of model-facing text that every count-based cap reports as
430
- # bounded. Each value is cut through the same helper as the title
431
- # and the abstract, so it is redacted first for the same reason.
432
- all_authors = [
433
- name for name in (
434
- third.text(_text_of(author, "atom:name"),
435
- _MAX_AUTHOR_CHARS)[0]
436
- for author in entry.findall("atom:author", ATOM_NS)
437
- ) if name
438
- ]
439
- author_count = len(all_authors)
440
- authors = all_authors[:_MAX_AUTHORS]
441
- all_categories = [
442
- term for term in (
443
- third.text(c.get("term"), _MAX_CATEGORY_CHARS)[0]
444
- for c in entry.findall("atom:category", ATOM_NS)
445
- ) if term
446
- ]
447
- categories = all_categories[:MAX_CATEGORIES]
448
-
449
- # DROPPED, NOT CUT (A58), and this is the field that makes the
450
- # distinction concrete. `pdf_url` exists to be fetched; cut to a cap
451
- # it becomes a fetchable-looking string that is not the paper, and
452
- # the model spends a tool call finding that out. Measured, a
453
- # 5,021-character href passed through byte-for-byte before this.
454
- # `MAX_URL_CHARS` is fetch_url's own limit, so anything past it is a
455
- # route this harness would refuse anyway.
456
- pdf_url, pdf_withheld = None, False
457
- for link in entry.findall("atom:link", ATOM_NS):
458
- if link.get("title") == "pdf":
459
- pdf_url, pdf_withheld = third.url(link.get("href"),
460
- MAX_URL_CHARS)
461
-
462
- entry_result = {
463
- "arxiv_id": arxiv_id,
464
- "title": title,
465
- "authors": authors,
466
- "summary": summary,
467
- "published": published,
468
- "categories": categories,
469
- "pdf_url": pdf_url,
470
- }
471
- if summary_truncated:
472
- # PG4's whole complaint: a truncated value that says nothing
473
- # about how to recover the rest is what sends the model to the
474
- # shell. The abs page is the route, and it is a URL `fetch_url`
475
- # will take.
476
- entry_result["summary_truncated"] = True
477
- if not id_truncated:
478
- entry_result["full_summary_url"] = (
479
- f"https://arxiv.org/abs/{arxiv_id}")
480
- if author_count > _MAX_AUTHORS:
481
- # PG8's shape exactly, including the recovery route PG4 asks
482
- # for: the abs page lists every author and `fetch_url` will
483
- # take it. `full_authors_url` rather than reusing
484
- # `full_summary_url` -- same URL, but a key named for the
485
- # summary carrying the author route is a name that lies, and
486
- # the summary's key is absent whenever the abstract fit.
487
- entry_result["authors_truncated"] = True
488
- entry_result["author_count"] = author_count
489
- # GUARDED ON THE ID, like `full_summary_url` above and for the
490
- # same reason (A58): the route is assembled from `arxiv_id`, so
491
- # an id that was cut yields a URL that 404s. PG4 asks for a
492
- # route to what the model WANTED; a broken one answers a
493
- # question nobody asked, and `authors_truncated` still says the
494
- # list was cut.
495
- if not id_truncated:
496
- entry_result["full_authors_url"] = (
497
- f"https://arxiv.org/abs/{arxiv_id}")
498
- # THESE TWO GET A FLAG AND NO ROUTE, and the asymmetry is deliberate
499
- # rather than an omission to be tidied up later. #49's rule is that
500
- # nothing is altered silently, so the flag is owed. But PG4 exists so
501
- # the model can reach what it WANTED, and a title past 300 characters
502
- # or a twenty-first category is a malformed feed rather than a long
503
- # paper -- measured, the longest title in sixty real entries was 188.
504
- # Pointing the model at the abs page of a hostile entry would be
505
- # answering a question nobody asked.
506
- if title_truncated:
507
- entry_result["title_truncated"] = True
508
- if len(all_categories) > MAX_CATEGORIES:
509
- entry_result["categories_truncated"] = True
510
- # THE THREE FLAGS BATCH 156 OWES, same rule (#49): nothing here is
511
- # altered silently. `id_truncated` is the one that matters most,
512
- # because a cut id is a citation that does not resolve and the model
513
- # has no other way to know.
514
- if id_truncated:
515
- entry_result["id_truncated"] = True
516
- if pdf_withheld:
517
- entry_result["pdf_url_truncated"] = True
518
- results.append(entry_result)
519
-
520
- third.report()
521
- return results, total
522
-
523
-
524
- # ---- Entry point called by the tool dispatcher -----------------------------
525
-
526
- def run(params: dict) -> dict:
527
- parsed = ArxivSearchParams(**params)
528
- # §51 (PG9). `start` is part of the key because it changes the response.
529
- # Without it page 2 of a search would be served page 1 out of the cache
530
- # -- the same class of collision this cache's own docstring warns about
531
- # between the two tools, arriving instead between two pages of one
532
- # query, and silent in exactly the same way.
533
- cache_key = (f"{parsed.keywords}|{parsed.category}|{parsed.max_results}"
534
- f"|{parsed.sort_by}|{parsed.start}")
535
-
536
- cached = _cache.get(cache_key)
537
- total: Optional[int] = None
538
- if cached is not None:
539
- logger.info("arxiv_search cache hit for %r", parsed.keywords)
540
- results, total = cached
541
- else:
542
- search_query = _build_search_query(parsed.keywords, parsed.category)
543
- last_exc: Optional[Exception] = None
544
- results = []
545
- for attempt in range(MAX_RETRIES + 1):
546
- try:
547
- xml_text = _call_arxiv_api(search_query, parsed.max_results,
548
- parsed.sort_by, parsed.start)
549
- results, total = _parse_atom_feed(xml_text)
550
- # A62. `max_results` is advertised `le=20` and reaches the
551
- # provider, which is free to ignore it -- measured (batch
552
- # 156), a feed of 5,000 entries against a request for 5
553
- # returned all 5,000. The per-entry caps above are
554
- # multiplied by this number, so bounding them without
555
- # bounding it is not a bound. `total` is unaffected and
556
- # still reports what the QUERY has, which is the figure
557
- # that says what was not returned.
558
- if len(results) > parsed.max_results:
559
- logger.warning(
560
- "arxiv_search kept %d of %d entries: the feed "
561
- "carried more than the %d asked for.",
562
- parsed.max_results, len(results), parsed.max_results)
563
- results = results[:parsed.max_results]
564
- _cache.set(cache_key, (results, total))
565
- break
566
- except _FeedTooLarge:
567
- # NOT RETRIED, for the reason spelled out one clause down
568
- # for a declared entity: a feed this large is this large
569
- # again, so two more round trips reach the same place and
570
- # then report "failed after 3 attempts", which describes a
571
- # network problem that is not there.
572
- #
573
- # Returned rather than raised, like every other dead end in
574
- # this function: a search tool that cannot answer is a
575
- # result the model can work around.
576
- logger.warning(
577
- "arxiv_search dropped a feed over %d bytes for %r "
578
- "(max_results=%d, start=%d)",
579
- MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
580
- parsed.start)
581
- # "more than", not a size. `total` stopped at the first
582
- # chunk past the cap, so printing it would be an instrument
583
- # reporting a magnitude it did not measure -- the
584
- # instrument-that-lies shape batches 118 and 119 each paid
585
- # for once. The lever IS named, because the model controls
586
- # it and a smaller page really does fit.
587
- return {
588
- "error": "arXiv returned more than %d bytes for this "
589
- "query, so nothing was read from it. Retry "
590
- "with a smaller max_results."
591
- % MAX_FEED_BYTES
592
- }
593
- except DefusedXmlException as e:
594
- # NOT folded into the tuple below, and that is the point.
595
- # A malformed body is transient and worth three attempts;
596
- # a feed that DECLARES an entity is not -- it refuses the
597
- # same way every time, so retrying spends two more round
598
- # trips to reach the same place and then reports "failed
599
- # after 3 attempts", which describes a network problem
600
- # that is not there.
601
- #
602
- # Returned rather than raised, for the reason the retry
603
- # exhaustion below is returned: a search tool that cannot
604
- # answer is a result the model can work around. These do
605
- # NOT subclass ParseError -- they are ValueError -- so
606
- # without this clause they would propagate out of run().
607
- logger.warning(
608
- "arxiv_search refused a hostile feed for %r: %s",
609
- parsed.keywords, e)
610
- # THE FEED'S OWN TEXT STOPS HERE (batch 116's review).
611
- # `e`'s repr carries the entity's `system_id` verbatim from
612
- # the feed, so interpolating `{e}` would put a string the
613
- # ATTACKER chose into the model's context -- the one place
614
- # §32's untrusted-content paragraph exists to bound. The
615
- # log above keeps the whole exception, because it is
616
- # operator-facing and the formatter redaction sink guards
617
- # app.log; the model gets the category and, where the
618
- # exception carries one, the entity NAME, which XML's Name
619
- # production cannot make into a sentence.
620
- #
621
- # getattr, not `e.name`: ExternalReferenceForbidden has no
622
- # `.name`, and an AttributeError raised while building an
623
- # error message would replace a handled refusal with a
624
- # crash on the hostile path.
625
- declared = getattr(e, "name", None)
626
- named = (" -- it declared the entity %r"
627
- % str(declared)[:40]) if declared else ""
628
- return {
629
- "error": "arXiv returned a feed this parser refuses to "
630
- "expand, so nothing was read from it: %s%s"
631
- % (type(e).__name__, named)
632
- }
633
- except (httpx.HTTPError, ET.ParseError) as e:
634
- last_exc = e
635
- logger.warning(
636
- "arxiv_search attempt %s/%s failed for %r: %s",
637
- attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
638
- else:
639
- # RETURNED, not raised. A search tool that cannot reach its
640
- # provider is a result the model can work around -- try
641
- # web_search, or say the claim could not be grounded. Raising
642
- # made one transient network failure, in one pass, fail an
643
- # entire ten-pass research run; fetch_url has always returned
644
- # an error dict here and arxiv_search was the outlier.
645
- #
646
- # dispatch() now contains a raise from any tool as well, so
647
- # this is the message rather than the mechanism: the model
648
- # gets something specific instead of a generic wrapper.
649
- return {
650
- "error": f"arXiv search failed after {MAX_RETRIES + 1} "
651
- f"attempts: {last_exc}"
652
- }
653
-
654
- if not results:
655
- if parsed.start:
656
- # Distinguishable from "this query has nothing", which is what
657
- # the bare message would have said about a page past the end.
658
- return {
659
- "results": [], "result_count": 0, "start": parsed.start,
660
- "message": (
661
- f"No papers at start={parsed.start}. The search has "
662
- f"{total} in total." if total is not None else
663
- f"No papers at start={parsed.start}, which is past the "
664
- f"end of this search."
665
- ),
666
- }
667
- return {"results": [], "result_count": 0, "message": "No papers found."}
668
-
669
- result = {"results": results, "result_count": len(results),
670
- "start": parsed.start}
671
-
672
- # PG4: say how to get the rest, not merely that there is a rest.
673
- next_start = parsed.start + len(results)
674
- if total is not None:
675
- result["total_results"] = total
676
- if next_start < total:
677
- result["message"] = (
678
- f"Returned papers {parsed.start + 1}-{next_start} of {total}. "
679
- f"Use start={next_start} to see more."
680
- )
681
- return result
1
+ """
2
+ tools/builtin/arxiv.py
3
+
4
+ Searches arXiv for academic papers by keyword and optional subject
5
+ category, using arXiv's public query API. No API key needed. Useful
6
+ alongside web_search/fetch_url for grounding claims in published research
7
+ during the source_grounding pass.
8
+
9
+ The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
10
+ largely translating that into the same normalized result shape the other
11
+ tools use.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import logging
17
+ from typing import Optional
18
+
19
+ # TECHNICAL_DEBT 18, batch 116. This feed is attacker-influenced -- arXiv
20
+ # echoes the query back and the entries are third-party text -- and stdlib
21
+ # ElementTree expands internal entities, which is Bandit B405/B314 and was
22
+ # an accepted risk for the CI rollout rather than a decision that it was
23
+ # safe. `defusedxml.ElementTree` is the drop-in: `fromstring` refuses
24
+ # declared entities and external references, and it re-exports the SAME
25
+ # `ParseError` object as the stdlib, so the empty-redirect-body path below
26
+ # and the retry loop's except clause are unaffected. Measured, not assumed:
27
+ # stdlib expands `&lol2;` to "lollollol" where this raises
28
+ # EntitiesForbidden.
29
+ import defusedxml.ElementTree as ET
30
+ import httpx
31
+ from defusedxml import DefusedXmlException
32
+ from pydantic import BaseModel, Field, field_validator
33
+
34
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
35
+
36
+ logger = logging.getLogger(__name__)
37
+
38
+ # HTTPS, and it is load-bearing. arXiv 301-redirects the http:// form, and
39
+ # three things then combine into a hard failure that looks nothing like a
40
+ # redirect: httpx does NOT follow redirects by default (unlike requests),
41
+ # raise_for_status() only raises on >= 400 so a 301 sails through, and the
42
+ # redirect body is empty -- so ET.fromstring("") raises ParseError, the
43
+ # retry loop runs three identical attempts, and the tool reports "arXiv
44
+ # search failed after 3 attempts". Verified against the live API.
45
+ ARXIV_API_URL = "https://export.arxiv.org/api/query"
46
+ ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
47
+
48
+ # ROADMAP_v3 §51 (PG3). arXiv answers in OpenSearch's paging elements as
49
+ # well as Atom's entries, and they are what let a result say how many papers
50
+ # the query has rather than only that it has more. Measured (batch 105):
51
+ # `all:transformer attention` reports totalResults=261387, with startIndex
52
+ # and itemsPerPage echoing the request.
53
+ OPENSEARCH_NS = {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"}
54
+
55
+ _VALID_SORT_BY = {"relevance", "lastUpdatedDate", "submittedDate"}
56
+
57
+ MAX_SUMMARY_CHARS = 600
58
+ REQUEST_TIMEOUT_S = 10.0
59
+ MAX_RETRIES = 2
60
+ CACHE_TTL_S = 300
61
+
62
+ # ROADMAP_v2 §31 (H7), #55, applied to the second tool that had the defect.
63
+ # `httpx.get(...).text` buffers the ENTIRE body before anything looks at it,
64
+ # and fetch_url.py's own comment rules out the shortcut: a Content-Length
65
+ # check is not the fix, because a hostile server can omit it or lie about
66
+ # it, so the only thing that bounds this is refusing to read past a cap.
67
+ # This feed is attacker-influenced -- the module header says so -- and this
68
+ # was the last `httpx.get` in the project.
69
+ #
70
+ # DERIVED FROM A MEASUREMENT, and the first derivation was wrong. It guessed
71
+ # "roughly 100 KB of Atom per hyperauthorship entry" and therefore 2 MiB for
72
+ # twenty of them; measured live, `all:LHCb observation branching fraction`
73
+ # answers max_results=20 with 838,560 bytes carrying 13,859 <author>
74
+ # elements, about 42 KB an entry, and four consecutive pages of it stayed
75
+ # under 839 KB. So 2 MiB clears the heaviest LEGITIMATE feed this tool can
76
+ # ask for by 2.5x -- which is the number that matters, because a cap under
77
+ # it is a FALSE REFUSAL: the tool reporting it read nothing for a query
78
+ # arXiv answered correctly.
79
+ #
80
+ # Loose next to fetch_url's 65,536 on purpose. That tool's URL is chosen by
81
+ # the model and can point anywhere; this endpoint is fixed and its response
82
+ # shape is known, so the bound's job here is the refusal rather than the
83
+ # tightness.
84
+ MAX_FEED_BYTES = 2_097_152
85
+
86
+ # ROADMAP_v3 §51 (PG8) for the shape, #131 for the reason: these bound
87
+ # model-facing text from a source nobody here wrote. The abstract was the
88
+ # only field anyone capped and it is not the field that grew -- measured
89
+ # (TECHNICAL_DEBT 41, re-measured in batch 126) a `str(result)` of 223,191
90
+ # characters, of which 204,871 (91.8%) were the author list, with one entry
91
+ # carrying 2,832 authors and eleven of twenty over the cap below. Everything
92
+ # the tool already bounds ceilings at about 16,900 characters even at
93
+ # max_results=20, so the capped fields could never have accounted for it.
94
+ #
95
+ # 25 is `core/reasoning/scholar.py`'s `_MAX_AUTHORS`, answering the same
96
+ # question the same way -- enough to find the strongest name on a paper
97
+ # without carrying a hyperauthorship record. NOT imported: tools/ does not
98
+ # import core/reasoning/, and one constant reaching across two layers would
99
+ # be worse than two constants with one reason each.
100
+ _MAX_AUTHORS = 25
101
+
102
+ # Measured over sixty real entries from three queries: the longest title was
103
+ # 188 characters ("Measurements of the Higgs boson production and decay
104
+ # rates..."). So this is inert for every real paper and bounds the malformed
105
+ # one -- which is why both of these get a FLAG and no recovery route. See
106
+ # `_parse_atom_feed`.
107
+ MAX_TITLE_CHARS = 300
108
+ MAX_CATEGORIES = 20
109
+
110
+ # Batch 156 (ROADMAP_v3 §73). THE CAPS ABOVE BOUND THE LISTS AND NOT THE
111
+ # VALUES IN THEM, which is the "which axis" question of batch 126 asked
112
+ # again one level down: `authors` kept 25 of them and `categories` 20, and
113
+ # a single `<name>` or a single `term` could be any length at all.
114
+ # Measured (batch 156) through the real parser, a 5,000-character author
115
+ # name, a 5,000-character category term and a 5,021-character pdf href all
116
+ # came back byte-for-byte while the 5,000-character title in the same entry
117
+ # was cut to 300 with its flag -- the control that proves the probe was
118
+ # reaching the caps.
119
+ #
120
+ # DERIVED, each from the longest legitimate shape of the value it bounds,
121
+ # because a cap that trims an ordinary entry turns its own flag into noise:
122
+ #
123
+ # * an author name is a person or a collaboration byline. The longest
124
+ # plausible is the latter ("The ATLAS and CMS Collaborations" is 32
125
+ # characters); 200 clears every real one several times over.
126
+ # * a category `term` comes from arXiv's published taxonomy, whose
127
+ # longest member is `cond-mat.stat-mech` at 18 characters. The longest
128
+ # appearing anywhere in this repository is `physics.gen-ph` at 14.
129
+ # * an id is `2401.00001v1` (12) in the new scheme or
130
+ # `math.GT/0309136v2` (17) in the old -- 17 is the longest of either
131
+ # form present in this repository, and 64 is not reachable by a
132
+ # well-formed one.
133
+ _MAX_AUTHOR_CHARS = 200
134
+ _MAX_CATEGORY_CHARS = 40
135
+ _MAX_ID_CHARS = 64
136
+
137
+ # One instance per tool, never shared: an equal key string from an
138
+ # arXiv query and a web query would otherwise collide.
139
+ _cache = TTLCache(CACHE_TTL_S)
140
+
141
+
142
+ # ---- Schema exposed to the LLM -------------------------------------------
143
+
144
+ class ArxivSearchParams(BaseModel):
145
+ keywords: str = Field(..., description="Search keywords", min_length=1, max_length=400)
146
+ category: Optional[str] = Field(
147
+ None,
148
+ description=(
149
+ "Optional arXiv category code to restrict the search, e.g. "
150
+ "'cs.AI', 'cs.LG', 'physics.gen-ph', 'q-bio.NC'. Omit to search "
151
+ "all categories."
152
+ ),
153
+ )
154
+ max_results: int = Field(5, ge=1, le=20, description="Number of results to return")
155
+ start: int = Field(
156
+ 0,
157
+ ge=0,
158
+ description=(
159
+ "Number of results to skip before returning any (0-based). Use "
160
+ "the start named in a result's message to page further into a "
161
+ "search that has more papers than were returned."
162
+ ),
163
+ )
164
+ sort_by: str = Field(
165
+ "relevance",
166
+ description="How to sort results: 'relevance', 'lastUpdatedDate', or 'submittedDate'",
167
+ )
168
+
169
+ @field_validator("keywords")
170
+ @classmethod
171
+ def strip_keywords(cls, value: str) -> str:
172
+ value = value.strip()
173
+ if not value:
174
+ raise ValueError("keywords cannot be empty")
175
+ return value
176
+
177
+ @field_validator("sort_by")
178
+ @classmethod
179
+ def validate_sort_by(cls, value: str) -> str:
180
+ if value not in _VALID_SORT_BY:
181
+ raise ValueError(f"sort_by must be one of {_VALID_SORT_BY}")
182
+ return value
183
+
184
+
185
+ TOOL_SCHEMA = {
186
+ "name": "arxiv_search",
187
+ "description": (
188
+ "Search arXiv for academic papers by keyword and optional subject "
189
+ "category. Returns titles, authors, abstracts, publish dates, and "
190
+ "PDF links -- use this to ground claims in published research."
191
+ ),
192
+ "input_schema": ArxivSearchParams.model_json_schema(),
193
+ }
194
+
195
+
196
+ class ArxivSearchError(Exception):
197
+ """No longer raised by this module -- exhausted retries return an
198
+ error dict instead, so a transient network failure cannot fail a
199
+ ten-pass research run. Kept because it is part of this module's
200
+ public surface and something outside the repo may catch it."""
201
+
202
+
203
+ class _FeedTooLarge(Exception):
204
+ """The feed passed MAX_FEED_BYTES and was dropped UNREAD.
205
+
206
+ PRIVATE, and it never escapes `run()`. Deliberately not
207
+ `ArxivSearchError`: that one is documented above as part of this
208
+ module's public surface and as no longer raised, and quietly starting
209
+ to raise it again would make that docstring false.
210
+
211
+ A separate exception rather than a sentinel return because the two
212
+ outcomes are not the same shape -- every other path returns a feed to
213
+ parse, and this one has nothing to parse. `fetch_url` can `break` and
214
+ keep a truncated body because a cut page of prose is still prose; a cut
215
+ Atom document does not parse at all, so there is nothing to hand back.
216
+ """
217
+
218
+
219
+ # ---- Query construction -----------------------------------------------------
220
+
221
+ def _build_search_query(keywords: str, category: Optional[str]) -> str:
222
+ """
223
+ arXiv's query syntax uses field prefixes (ti:, abs:, au:, cat:, all:)
224
+ combined with AND/OR/ANDNOT. `all:` searches title, abstract, and
225
+ author. httpx handles URL-encoding the spaces/quotes once this string
226
+ is passed as a params value, so no manual encoding is needed here.
227
+ """
228
+ if category:
229
+ return f"cat:{category} AND all:{keywords}"
230
+ return f"all:{keywords}"
231
+
232
+
233
+ # ---- Provider call ----------------------------------------------------------
234
+
235
+ def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
236
+ start: int = 0) -> str:
237
+ """The raw Atom XML, or `_FeedTooLarge` if it would not fit the bound.
238
+
239
+ STREAMED since batch 126 (§31 H7), so the connection is dropped inside
240
+ the `with` the moment the cap is passed rather than after the whole body
241
+ is already in memory. Returns `str` as it always did, so every test that
242
+ patches this function is unaffected by the change underneath it.
243
+ """
244
+ with httpx.stream(
245
+ "GET",
246
+ ARXIV_API_URL,
247
+ params={
248
+ "search_query": search_query,
249
+ # §51 (PG3). Was hardcoded to 0, which made every page of every
250
+ # search the first one: the provider had implemented paging all
251
+ # along and this tool spent the parameter on a constant.
252
+ "start": start,
253
+ "max_results": max_results,
254
+ "sortBy": sort_by,
255
+ "sortOrder": "descending",
256
+ },
257
+ headers={"User-Agent": "agent-harness-arxiv-tool/1.0 (research pipeline)"},
258
+ timeout=REQUEST_TIMEOUT_S,
259
+ # Belt and braces beside the https:// above. If arXiv moves the
260
+ # endpoint again, following the redirect keeps this working
261
+ # instead of failing as an unparseable empty body.
262
+ follow_redirects=True,
263
+ ) as response:
264
+ # Streamed, so status and headers are available before the body --
265
+ # the same ordering fetch_url relies on.
266
+ response.raise_for_status()
267
+ chunks = []
268
+ total = 0
269
+ for chunk in response.iter_bytes():
270
+ total += len(chunk)
271
+ if total > MAX_FEED_BYTES:
272
+ raise _FeedTooLarge(total)
273
+ chunks.append(chunk)
274
+ encoding = getattr(response, "encoding", None) or "utf-8"
275
+
276
+ # errors="replace", and it is not laziness. A bare decode raises
277
+ # UnicodeDecodeError on a feed that is not what its header claims, and
278
+ # that is a ValueError -- which NEITHER except clause in `run()` covers,
279
+ # so it would escape as a crash on precisely the hostile path the rest
280
+ # of this module is careful about. Replacing instead makes a nonsense
281
+ # body fail as a ParseError, which IS handled and IS retried.
282
+ #
283
+ # §51's PG5 concern about a cut multi-byte character does not apply:
284
+ # these bytes are always a COMPLETE document, because an incomplete one
285
+ # was refused above rather than truncated.
286
+ return b"".join(chunks).decode(encoding, errors="replace")
287
+
288
+
289
+ # ---- Atom XML parsing --------------------------------------------------
290
+
291
+ def _total_results(root) -> Optional[int]:
292
+ """How many papers the query has, from OpenSearch's own element (PG3).
293
+
294
+ Optional rather than assumed: the element is arXiv's to emit, and a
295
+ result that reported a total it had invented would be worse than one
296
+ that reports none.
297
+ """
298
+ element = root.find("opensearch:totalResults", OPENSEARCH_NS)
299
+ if element is None or not (element.text or "").strip():
300
+ return None
301
+ try:
302
+ return int(element.text.strip())
303
+ except ValueError:
304
+ return None
305
+
306
+
307
+ def _text_of(parent, path) -> Optional[str]:
308
+ """One child element's stripped text, or None.
309
+
310
+ EVERY lookup in `_parse_atom_feed` goes through this, because the six
311
+ ways that function used to raise were six slightly different spellings
312
+ of the same omission (batch 117, TECHNICAL_DEBT 33):
313
+
314
+ * a MISSING element -- `find()` returns None and `.text` raises
315
+ AttributeError. Three of the six.
316
+ * a PRESENT but EMPTY element -- `<title></title>` has `.text` of
317
+ None, so `.strip()` raises AttributeError and `[:10]` raises
318
+ TypeError. The remaining three, and they are the ones a reader
319
+ skips, because `summary` looked guarded by `or ""` and `published`
320
+ looked guarded by `is not None`. Neither covers both halves.
321
+
322
+ The feed is attacker-influenced -- this module's header says so, and
323
+ batch 116 added a parser that refuses hostile ENTITIES one function
324
+ away -- so its STRUCTURE cannot be trusted either.
325
+ """
326
+ element = parent.find(path, ATOM_NS)
327
+ if element is None or element.text is None:
328
+ return None
329
+ stripped = element.text.strip()
330
+ return stripped or None
331
+
332
+
333
+ def _parse_atom_feed(xml_text: str) -> tuple:
334
+ """(entries, how many the query has in total).
335
+
336
+ The total is OpenSearch's, not a count of what came back -- the point
337
+ of it is to say what was NOT returned, which a count of what was cannot.
338
+ """
339
+ root = ET.fromstring(xml_text)
340
+ entries = root.findall("atom:entry", ATOM_NS)
341
+ total = _total_results(root)
342
+
343
+ third = ThirdPartyText("arxiv_search")
344
+ results = []
345
+ for index, entry in enumerate(entries):
346
+ raw_id = _text_of(entry, "atom:id")
347
+ title_text = _text_of(entry, "atom:title")
348
+ if raw_id is None or title_text is None:
349
+ # SKIPPED, NOT RAISED (batch 117). An id and a title are what
350
+ # make an entry citable; without either there is nothing for
351
+ # the model to ground a claim on, so the entry is dropped and
352
+ # the rest of the page is still returned. Raising here aborted
353
+ # the whole search over one malformed entry, and the raise did
354
+ # not even reach `run`'s handlers -- AttributeError is neither
355
+ # a ParseError nor a DefusedXmlException.
356
+ #
357
+ # The index, not the id: the id is what might be missing.
358
+ logger.warning(
359
+ "arxiv_search skipped entry %d of %d: no %s",
360
+ index + 1, len(entries),
361
+ "id" if raw_id is None else "title")
362
+ continue
363
+ raw_arxiv_id = (raw_id.split("/abs/")[-1]
364
+ if "/abs/" in raw_id else raw_id)
365
+ # THE ID IS NOT JUST DISPLAYED, it is the input to the two recovery
366
+ # URLs below, so an unbounded one is three unbounded values wearing
367
+ # one name -- measured (batch 156), a 5,000-character `<id>` made
368
+ # `full_summary_url` 5,022 characters long. Cut and flagged like any
369
+ # other text (A58); what the cut costs is handled where the routes
370
+ # are built, because a route assembled from half an id is a route to
371
+ # nowhere and that is the one thing a route must not be.
372
+ arxiv_id, id_truncated = third.text(raw_arxiv_id, _MAX_ID_CHARS)
373
+
374
+ title, title_truncated = third.text(
375
+ title_text.replace("\n", " "), MAX_TITLE_CHARS)
376
+ summary, summary_truncated = third.text(
377
+ (_text_of(entry, "atom:summary") or "").replace("\n", " "),
378
+ MAX_SUMMARY_CHARS)
379
+ # §51 (PG8/PG4). The cap stays -- it bounds model-facing text from a
380
+ # source nobody here wrote, which is #131's lesson -- but a value cut
381
+ # at it now SAYS it was cut. Until §51 a 600-character abstract and
382
+ # one that happened to be 600 characters long were the same thing to
383
+ # a reader, so the model could not tell an abstract it had all of
384
+ # from one it had the first two thirds of.
385
+ #
386
+ # REDACTED ABOVE THE CUT SINCE BATCH 156 (RB1, A59), which is the
387
+ # half §51 got the wrong way round. Both of these were sliced and
388
+ # `registry.dispatch` redacted the slice; measured, a `ghp_` token
389
+ # straddling either cut survived post-hoc redaction at 36 of its 39
390
+ # possible offsets. `fetch_url` had already fixed the same ordering
391
+ # for its own body, and `param_digest` had written down the reason
392
+ # before either of them. The flags now count REDACTED characters,
393
+ # which is RB3's consequence: redaction can shrink a value back
394
+ # under the cap, and when it does the cut is withholding nothing.
395
+
396
+ # `is not None` on the ELEMENT was not enough: `<published></published>`
397
+ # is present with `.text` of None, and `None[:10]` is a TypeError
398
+ # rather than an AttributeError -- the same omission wearing a
399
+ # different exception type, which is why both halves go through
400
+ # one helper now.
401
+ published_text = _text_of(entry, "atom:published")
402
+ published = published_text[:10] if published_text else None
403
+
404
+ # A NAMELESS ELEMENT IS DROPPED RATHER THAN INSERTED AS None, and
405
+ # this is one rule for BOTH lists below (TECHNICAL_DEBT 47). Both
406
+ # come from the same attacker-influenced feed and both reach the
407
+ # model the same way, so they must not disagree about it: a `None`
408
+ # in a list rendered into a citation reads as an author called None
409
+ # and as a category called None.
410
+ #
411
+ # `categories` kept its None until batch 127. The rule was stated
412
+ # here, two lines above the list that did not follow it, which is
413
+ # why the comment now names both instead of being copied.
414
+ #
415
+ # BOUNDED SINCE BATCH 126, and `authors` is the field
416
+ # TECHNICAL_DEBT 41 was about without naming it. The count is kept
417
+ # before the cut because `author_count` reports the real total --
418
+ # the point of the key is that the model can tell a 2,832-author
419
+ # paper from a 25-author one, which the truncated list alone cannot
420
+ # say.
421
+ #
422
+ # DROPPED BEFORE THE CUT for both, so the cap counts values the
423
+ # model can actually use: filtering after the slice would let a
424
+ # nameless element consume one of the 25 or 20 places.
425
+ #
426
+ # AND THE CAP ON EACH VALUE, not only on the count (batch 156). The
427
+ # two bounds answer different questions and the lists had only the
428
+ # first: 25 authors of 5,000 characters each is 125,000 characters
429
+ # of model-facing text that every count-based cap reports as
430
+ # bounded. Each value is cut through the same helper as the title
431
+ # and the abstract, so it is redacted first for the same reason.
432
+ all_authors = [
433
+ name for name in (
434
+ third.text(_text_of(author, "atom:name"),
435
+ _MAX_AUTHOR_CHARS)[0]
436
+ for author in entry.findall("atom:author", ATOM_NS)
437
+ ) if name
438
+ ]
439
+ author_count = len(all_authors)
440
+ authors = all_authors[:_MAX_AUTHORS]
441
+ all_categories = [
442
+ term for term in (
443
+ third.text(c.get("term"), _MAX_CATEGORY_CHARS)[0]
444
+ for c in entry.findall("atom:category", ATOM_NS)
445
+ ) if term
446
+ ]
447
+ categories = all_categories[:MAX_CATEGORIES]
448
+
449
+ # DROPPED, NOT CUT (A58), and this is the field that makes the
450
+ # distinction concrete. `pdf_url` exists to be fetched; cut to a cap
451
+ # it becomes a fetchable-looking string that is not the paper, and
452
+ # the model spends a tool call finding that out. Measured, a
453
+ # 5,021-character href passed through byte-for-byte before this.
454
+ # `MAX_URL_CHARS` is fetch_url's own limit, so anything past it is a
455
+ # route this harness would refuse anyway.
456
+ pdf_url, pdf_withheld = None, False
457
+ for link in entry.findall("atom:link", ATOM_NS):
458
+ if link.get("title") == "pdf":
459
+ pdf_url, pdf_withheld = third.url(link.get("href"),
460
+ MAX_URL_CHARS)
461
+
462
+ entry_result = {
463
+ "arxiv_id": arxiv_id,
464
+ "title": title,
465
+ "authors": authors,
466
+ "summary": summary,
467
+ "published": published,
468
+ "categories": categories,
469
+ "pdf_url": pdf_url,
470
+ }
471
+ if summary_truncated:
472
+ # PG4's whole complaint: a truncated value that says nothing
473
+ # about how to recover the rest is what sends the model to the
474
+ # shell. The abs page is the route, and it is a URL `fetch_url`
475
+ # will take.
476
+ entry_result["summary_truncated"] = True
477
+ if not id_truncated:
478
+ entry_result["full_summary_url"] = (
479
+ f"https://arxiv.org/abs/{arxiv_id}")
480
+ if author_count > _MAX_AUTHORS:
481
+ # PG8's shape exactly, including the recovery route PG4 asks
482
+ # for: the abs page lists every author and `fetch_url` will
483
+ # take it. `full_authors_url` rather than reusing
484
+ # `full_summary_url` -- same URL, but a key named for the
485
+ # summary carrying the author route is a name that lies, and
486
+ # the summary's key is absent whenever the abstract fit.
487
+ entry_result["authors_truncated"] = True
488
+ entry_result["author_count"] = author_count
489
+ # GUARDED ON THE ID, like `full_summary_url` above and for the
490
+ # same reason (A58): the route is assembled from `arxiv_id`, so
491
+ # an id that was cut yields a URL that 404s. PG4 asks for a
492
+ # route to what the model WANTED; a broken one answers a
493
+ # question nobody asked, and `authors_truncated` still says the
494
+ # list was cut.
495
+ if not id_truncated:
496
+ entry_result["full_authors_url"] = (
497
+ f"https://arxiv.org/abs/{arxiv_id}")
498
+ # THESE TWO GET A FLAG AND NO ROUTE, and the asymmetry is deliberate
499
+ # rather than an omission to be tidied up later. #49's rule is that
500
+ # nothing is altered silently, so the flag is owed. But PG4 exists so
501
+ # the model can reach what it WANTED, and a title past 300 characters
502
+ # or a twenty-first category is a malformed feed rather than a long
503
+ # paper -- measured, the longest title in sixty real entries was 188.
504
+ # Pointing the model at the abs page of a hostile entry would be
505
+ # answering a question nobody asked.
506
+ if title_truncated:
507
+ entry_result["title_truncated"] = True
508
+ if len(all_categories) > MAX_CATEGORIES:
509
+ entry_result["categories_truncated"] = True
510
+ # THE THREE FLAGS BATCH 156 OWES, same rule (#49): nothing here is
511
+ # altered silently. `id_truncated` is the one that matters most,
512
+ # because a cut id is a citation that does not resolve and the model
513
+ # has no other way to know.
514
+ if id_truncated:
515
+ entry_result["id_truncated"] = True
516
+ if pdf_withheld:
517
+ entry_result["pdf_url_truncated"] = True
518
+ results.append(entry_result)
519
+
520
+ third.report()
521
+ return results, total
522
+
523
+
524
+ # ---- Entry point called by the tool dispatcher -----------------------------
525
+
526
+ def run(params: dict) -> dict:
527
+ parsed = ArxivSearchParams(**params)
528
+ # §51 (PG9). `start` is part of the key because it changes the response.
529
+ # Without it page 2 of a search would be served page 1 out of the cache
530
+ # -- the same class of collision this cache's own docstring warns about
531
+ # between the two tools, arriving instead between two pages of one
532
+ # query, and silent in exactly the same way.
533
+ cache_key = (f"{parsed.keywords}|{parsed.category}|{parsed.max_results}"
534
+ f"|{parsed.sort_by}|{parsed.start}")
535
+
536
+ cached = _cache.get(cache_key)
537
+ total: Optional[int] = None
538
+ if cached is not None:
539
+ logger.info("arxiv_search cache hit for %r", parsed.keywords)
540
+ results, total = cached
541
+ else:
542
+ search_query = _build_search_query(parsed.keywords, parsed.category)
543
+ last_exc: Optional[Exception] = None
544
+ results = []
545
+ for attempt in range(MAX_RETRIES + 1):
546
+ try:
547
+ xml_text = _call_arxiv_api(search_query, parsed.max_results,
548
+ parsed.sort_by, parsed.start)
549
+ results, total = _parse_atom_feed(xml_text)
550
+ # A62. `max_results` is advertised `le=20` and reaches the
551
+ # provider, which is free to ignore it -- measured (batch
552
+ # 156), a feed of 5,000 entries against a request for 5
553
+ # returned all 5,000. The per-entry caps above are
554
+ # multiplied by this number, so bounding them without
555
+ # bounding it is not a bound. `total` is unaffected and
556
+ # still reports what the QUERY has, which is the figure
557
+ # that says what was not returned.
558
+ if len(results) > parsed.max_results:
559
+ logger.warning(
560
+ "arxiv_search kept %d of %d entries: the feed "
561
+ "carried more than the %d asked for.",
562
+ parsed.max_results, len(results), parsed.max_results)
563
+ results = results[:parsed.max_results]
564
+ _cache.set(cache_key, (results, total))
565
+ break
566
+ except _FeedTooLarge:
567
+ # NOT RETRIED, for the reason spelled out one clause down
568
+ # for a declared entity: a feed this large is this large
569
+ # again, so two more round trips reach the same place and
570
+ # then report "failed after 3 attempts", which describes a
571
+ # network problem that is not there.
572
+ #
573
+ # Returned rather than raised, like every other dead end in
574
+ # this function: a search tool that cannot answer is a
575
+ # result the model can work around.
576
+ logger.warning(
577
+ "arxiv_search dropped a feed over %d bytes for %r "
578
+ "(max_results=%d, start=%d)",
579
+ MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
580
+ parsed.start)
581
+ # "more than", not a size. `total` stopped at the first
582
+ # chunk past the cap, so printing it would be an instrument
583
+ # reporting a magnitude it did not measure -- the
584
+ # instrument-that-lies shape batches 118 and 119 each paid
585
+ # for once. The lever IS named, because the model controls
586
+ # it and a smaller page really does fit.
587
+ return {
588
+ "error": "arXiv returned more than %d bytes for this "
589
+ "query, so nothing was read from it. Retry "
590
+ "with a smaller max_results."
591
+ % MAX_FEED_BYTES
592
+ }
593
+ except DefusedXmlException as e:
594
+ # NOT folded into the tuple below, and that is the point.
595
+ # A malformed body is transient and worth three attempts;
596
+ # a feed that DECLARES an entity is not -- it refuses the
597
+ # same way every time, so retrying spends two more round
598
+ # trips to reach the same place and then reports "failed
599
+ # after 3 attempts", which describes a network problem
600
+ # that is not there.
601
+ #
602
+ # Returned rather than raised, for the reason the retry
603
+ # exhaustion below is returned: a search tool that cannot
604
+ # answer is a result the model can work around. These do
605
+ # NOT subclass ParseError -- they are ValueError -- so
606
+ # without this clause they would propagate out of run().
607
+ logger.warning(
608
+ "arxiv_search refused a hostile feed for %r: %s",
609
+ parsed.keywords, e)
610
+ # THE FEED'S OWN TEXT STOPS HERE (batch 116's review).
611
+ # `e`'s repr carries the entity's `system_id` verbatim from
612
+ # the feed, so interpolating `{e}` would put a string the
613
+ # ATTACKER chose into the model's context -- the one place
614
+ # §32's untrusted-content paragraph exists to bound. The
615
+ # log above keeps the whole exception, because it is
616
+ # operator-facing and the formatter redaction sink guards
617
+ # app.log; the model gets the category and, where the
618
+ # exception carries one, the entity NAME, which XML's Name
619
+ # production cannot make into a sentence.
620
+ #
621
+ # getattr, not `e.name`: ExternalReferenceForbidden has no
622
+ # `.name`, and an AttributeError raised while building an
623
+ # error message would replace a handled refusal with a
624
+ # crash on the hostile path.
625
+ declared = getattr(e, "name", None)
626
+ named = (" -- it declared the entity %r"
627
+ % str(declared)[:40]) if declared else ""
628
+ return {
629
+ "error": "arXiv returned a feed this parser refuses to "
630
+ "expand, so nothing was read from it: %s%s"
631
+ % (type(e).__name__, named)
632
+ }
633
+ except (httpx.HTTPError, ET.ParseError) as e:
634
+ last_exc = e
635
+ logger.warning(
636
+ "arxiv_search attempt %s/%s failed for %r: %s",
637
+ attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
638
+ else:
639
+ # RETURNED, not raised. A search tool that cannot reach its
640
+ # provider is a result the model can work around -- try
641
+ # web_search, or say the claim could not be grounded. Raising
642
+ # made one transient network failure, in one pass, fail an
643
+ # entire ten-pass research run; fetch_url has always returned
644
+ # an error dict here and arxiv_search was the outlier.
645
+ #
646
+ # dispatch() now contains a raise from any tool as well, so
647
+ # this is the message rather than the mechanism: the model
648
+ # gets something specific instead of a generic wrapper.
649
+ return {
650
+ "error": f"arXiv search failed after {MAX_RETRIES + 1} "
651
+ f"attempts: {last_exc}"
652
+ }
653
+
654
+ if not results:
655
+ if parsed.start:
656
+ # Distinguishable from "this query has nothing", which is what
657
+ # the bare message would have said about a page past the end.
658
+ return {
659
+ "results": [], "result_count": 0, "start": parsed.start,
660
+ "message": (
661
+ f"No papers at start={parsed.start}. The search has "
662
+ f"{total} in total." if total is not None else
663
+ f"No papers at start={parsed.start}, which is past the "
664
+ f"end of this search."
665
+ ),
666
+ }
667
+ return {"results": [], "result_count": 0, "message": "No papers found."}
668
+
669
+ result = {"results": results, "result_count": len(results),
670
+ "start": parsed.start}
671
+
672
+ # PG4: say how to get the rest, not merely that there is a rest.
673
+ next_start = parsed.start + len(results)
674
+ if total is not None:
675
+ result["total_results"] = total
676
+ if next_start < total:
677
+ result["message"] = (
678
+ f"Returned papers {parsed.start + 1}-{next_start} of {total}. "
679
+ f"Use start={next_start} to see more."
680
+ )
681
+ return result