@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,681 +1,685 @@
1
- """
2
- tools/builtin/arxiv.py
3
-
4
- Searches arXiv for academic papers by keyword and optional subject
5
- category, using arXiv's public query API. No API key needed. Useful
6
- alongside web_search/fetch_url for grounding claims in published research
7
- during the source_grounding pass.
8
-
9
- The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
10
- largely translating that into the same normalized result shape the other
11
- tools use.
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- import logging
17
- from typing import Optional
18
-
19
- # TECHNICAL_DEBT 18, batch 116. This feed is attacker-influenced -- arXiv
20
- # echoes the query back and the entries are third-party text -- and stdlib
21
- # ElementTree expands internal entities, which is Bandit B405/B314 and was
22
- # an accepted risk for the CI rollout rather than a decision that it was
23
- # safe. `defusedxml.ElementTree` is the drop-in: `fromstring` refuses
24
- # declared entities and external references, and it re-exports the SAME
25
- # `ParseError` object as the stdlib, so the empty-redirect-body path below
26
- # and the retry loop's except clause are unaffected. Measured, not assumed:
27
- # stdlib expands `&lol2;` to "lollollol" where this raises
28
- # EntitiesForbidden.
29
- import defusedxml.ElementTree as ET
30
- import httpx
31
- from defusedxml import DefusedXmlException
32
- from pydantic import BaseModel, Field, field_validator
33
-
34
- from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
35
-
36
- logger = logging.getLogger(__name__)
37
-
38
- # HTTPS, and it is load-bearing. arXiv 301-redirects the http:// form, and
39
- # three things then combine into a hard failure that looks nothing like a
40
- # redirect: httpx does NOT follow redirects by default (unlike requests),
41
- # raise_for_status() only raises on >= 400 so a 301 sails through, and the
42
- # redirect body is empty -- so ET.fromstring("") raises ParseError, the
43
- # retry loop runs three identical attempts, and the tool reports "arXiv
44
- # search failed after 3 attempts". Verified against the live API.
45
- ARXIV_API_URL = "https://export.arxiv.org/api/query"
46
- ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
47
-
48
- # ROADMAP_v3 §51 (PG3). arXiv answers in OpenSearch's paging elements as
49
- # well as Atom's entries, and they are what let a result say how many papers
50
- # the query has rather than only that it has more. Measured (batch 105):
51
- # `all:transformer attention` reports totalResults=261387, with startIndex
52
- # and itemsPerPage echoing the request.
53
- OPENSEARCH_NS = {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"}
54
-
55
- _VALID_SORT_BY = {"relevance", "lastUpdatedDate", "submittedDate"}
56
-
57
- MAX_SUMMARY_CHARS = 600
58
- REQUEST_TIMEOUT_S = 10.0
59
- MAX_RETRIES = 2
60
- CACHE_TTL_S = 300
61
-
62
- # ROADMAP_v2 §31 (H7), #55, applied to the second tool that had the defect.
63
- # `httpx.get(...).text` buffers the ENTIRE body before anything looks at it,
64
- # and fetch_url.py's own comment rules out the shortcut: a Content-Length
65
- # check is not the fix, because a hostile server can omit it or lie about
66
- # it, so the only thing that bounds this is refusing to read past a cap.
67
- # This feed is attacker-influenced -- the module header says so -- and this
68
- # was the last `httpx.get` in the project.
69
- #
70
- # DERIVED FROM A MEASUREMENT, and the first derivation was wrong. It guessed
71
- # "roughly 100 KB of Atom per hyperauthorship entry" and therefore 2 MiB for
72
- # twenty of them; measured live, `all:LHCb observation branching fraction`
73
- # answers max_results=20 with 838,560 bytes carrying 13,859 <author>
74
- # elements, about 42 KB an entry, and four consecutive pages of it stayed
75
- # under 839 KB. So 2 MiB clears the heaviest LEGITIMATE feed this tool can
76
- # ask for by 2.5x -- which is the number that matters, because a cap under
77
- # it is a FALSE REFUSAL: the tool reporting it read nothing for a query
78
- # arXiv answered correctly.
79
- #
80
- # Loose next to fetch_url's 65,536 on purpose. That tool's URL is chosen by
81
- # the model and can point anywhere; this endpoint is fixed and its response
82
- # shape is known, so the bound's job here is the refusal rather than the
83
- # tightness.
84
- MAX_FEED_BYTES = 2_097_152
85
-
86
- # ROADMAP_v3 §51 (PG8) for the shape, #131 for the reason: these bound
87
- # model-facing text from a source nobody here wrote. The abstract was the
88
- # only field anyone capped and it is not the field that grew -- measured
89
- # (TECHNICAL_DEBT 41, re-measured in batch 126) a `str(result)` of 223,191
90
- # characters, of which 204,871 (91.8%) were the author list, with one entry
91
- # carrying 2,832 authors and eleven of twenty over the cap below. Everything
92
- # the tool already bounds ceilings at about 16,900 characters even at
93
- # max_results=20, so the capped fields could never have accounted for it.
94
- #
95
- # 25 is `core/reasoning/scholar.py`'s `_MAX_AUTHORS`, answering the same
96
- # question the same way -- enough to find the strongest name on a paper
97
- # without carrying a hyperauthorship record. NOT imported: tools/ does not
98
- # import core/reasoning/, and one constant reaching across two layers would
99
- # be worse than two constants with one reason each.
100
- _MAX_AUTHORS = 25
101
-
102
- # Measured over sixty real entries from three queries: the longest title was
103
- # 188 characters ("Measurements of the Higgs boson production and decay
104
- # rates..."). So this is inert for every real paper and bounds the malformed
105
- # one -- which is why both of these get a FLAG and no recovery route. See
106
- # `_parse_atom_feed`.
107
- MAX_TITLE_CHARS = 300
108
- MAX_CATEGORIES = 20
109
-
110
- # Batch 156 (ROADMAP_v3 §73). THE CAPS ABOVE BOUND THE LISTS AND NOT THE
111
- # VALUES IN THEM, which is the "which axis" question of batch 126 asked
112
- # again one level down: `authors` kept 25 of them and `categories` 20, and
113
- # a single `<name>` or a single `term` could be any length at all.
114
- # Measured (batch 156) through the real parser, a 5,000-character author
115
- # name, a 5,000-character category term and a 5,021-character pdf href all
116
- # came back byte-for-byte while the 5,000-character title in the same entry
117
- # was cut to 300 with its flag -- the control that proves the probe was
118
- # reaching the caps.
119
- #
120
- # DERIVED, each from the longest legitimate shape of the value it bounds,
121
- # because a cap that trims an ordinary entry turns its own flag into noise:
122
- #
123
- # * an author name is a person or a collaboration byline. The longest
124
- # plausible is the latter ("The ATLAS and CMS Collaborations" is 32
125
- # characters); 200 clears every real one several times over.
126
- # * a category `term` comes from arXiv's published taxonomy, whose
127
- # longest member is `cond-mat.stat-mech` at 18 characters. The longest
128
- # appearing anywhere in this repository is `physics.gen-ph` at 14.
129
- # * an id is `2401.00001v1` (12) in the new scheme or
130
- # `math.GT/0309136v2` (17) in the old -- 17 is the longest of either
131
- # form present in this repository, and 64 is not reachable by a
132
- # well-formed one.
133
- _MAX_AUTHOR_CHARS = 200
134
- _MAX_CATEGORY_CHARS = 40
135
- _MAX_ID_CHARS = 64
136
-
137
- # One instance per tool, never shared: an equal key string from an
138
- # arXiv query and a web query would otherwise collide.
139
- _cache = TTLCache(CACHE_TTL_S)
140
-
141
-
142
- # ---- Schema exposed to the LLM -------------------------------------------
143
-
144
- class ArxivSearchParams(BaseModel):
145
- keywords: str = Field(..., description="Search keywords", min_length=1, max_length=400)
146
- category: Optional[str] = Field(
147
- None,
148
- description=(
149
- "Optional arXiv category code to restrict the search, e.g. "
150
- "'cs.AI', 'cs.LG', 'physics.gen-ph', 'q-bio.NC'. Omit to search "
151
- "all categories."
152
- ),
153
- )
154
- max_results: int = Field(5, ge=1, le=20, description="Number of results to return")
155
- start: int = Field(
156
- 0,
157
- ge=0,
158
- description=(
159
- "Number of results to skip before returning any (0-based). Use "
160
- "the start named in a result's message to page further into a "
161
- "search that has more papers than were returned."
162
- ),
163
- )
164
- sort_by: str = Field(
165
- "relevance",
166
- description="How to sort results: 'relevance', 'lastUpdatedDate', or 'submittedDate'",
167
- )
168
-
169
- @field_validator("keywords")
170
- @classmethod
171
- def strip_keywords(cls, value: str) -> str:
172
- value = value.strip()
173
- if not value:
174
- raise ValueError("keywords cannot be empty")
175
- return value
176
-
177
- @field_validator("sort_by")
178
- @classmethod
179
- def validate_sort_by(cls, value: str) -> str:
180
- if value not in _VALID_SORT_BY:
181
- raise ValueError(f"sort_by must be one of {_VALID_SORT_BY}")
182
- return value
183
-
184
-
185
- TOOL_SCHEMA = {
186
- "name": "arxiv_search",
187
- "description": (
188
- "Search arXiv for academic papers by keyword and optional subject "
189
- "category. Returns titles, authors, abstracts, publish dates, and "
190
- "PDF links -- use this to ground claims in published research."
191
- ),
192
- "input_schema": ArxivSearchParams.model_json_schema(),
193
- }
194
-
195
-
196
- class ArxivSearchError(Exception):
197
- """No longer raised by this module -- exhausted retries return an
198
- error dict instead, so a transient network failure cannot fail a
199
- ten-pass research run. Kept because it is part of this module's
200
- public surface and something outside the repo may catch it."""
201
-
202
-
203
- class _FeedTooLarge(Exception):
204
- """The feed passed MAX_FEED_BYTES and was dropped UNREAD.
205
-
206
- PRIVATE, and it never escapes `run()`. Deliberately not
207
- `ArxivSearchError`: that one is documented above as part of this
208
- module's public surface and as no longer raised, and quietly starting
209
- to raise it again would make that docstring false.
210
-
211
- A separate exception rather than a sentinel return because the two
212
- outcomes are not the same shape -- every other path returns a feed to
213
- parse, and this one has nothing to parse. `fetch_url` can `break` and
214
- keep a truncated body because a cut page of prose is still prose; a cut
215
- Atom document does not parse at all, so there is nothing to hand back.
216
- """
217
-
218
-
219
- # ---- Query construction -----------------------------------------------------
220
-
221
- def _build_search_query(keywords: str, category: Optional[str]) -> str:
222
- """
223
- arXiv's query syntax uses field prefixes (ti:, abs:, au:, cat:, all:)
224
- combined with AND/OR/ANDNOT. `all:` searches title, abstract, and
225
- author. httpx handles URL-encoding the spaces/quotes once this string
226
- is passed as a params value, so no manual encoding is needed here.
227
- """
228
- if category:
229
- return f"cat:{category} AND all:{keywords}"
230
- return f"all:{keywords}"
231
-
232
-
233
- # ---- Provider call ----------------------------------------------------------
234
-
235
- def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
236
- start: int = 0) -> str:
237
- """The raw Atom XML, or `_FeedTooLarge` if it would not fit the bound.
238
-
239
- STREAMED since batch 126 (§31 H7), so the connection is dropped inside
240
- the `with` the moment the cap is passed rather than after the whole body
241
- is already in memory. Returns `str` as it always did, so every test that
242
- patches this function is unaffected by the change underneath it.
243
- """
244
- with httpx.stream(
245
- "GET",
246
- ARXIV_API_URL,
247
- params={
248
- "search_query": search_query,
249
- # §51 (PG3). Was hardcoded to 0, which made every page of every
250
- # search the first one: the provider had implemented paging all
251
- # along and this tool spent the parameter on a constant.
252
- "start": start,
253
- "max_results": max_results,
254
- "sortBy": sort_by,
255
- "sortOrder": "descending",
256
- },
257
- headers={"User-Agent": "agent-harness-arxiv-tool/1.0 (research pipeline)"},
258
- timeout=REQUEST_TIMEOUT_S,
259
- # Belt and braces beside the https:// above. If arXiv moves the
260
- # endpoint again, following the redirect keeps this working
261
- # instead of failing as an unparseable empty body.
262
- follow_redirects=True,
263
- ) as response:
264
- # Streamed, so status and headers are available before the body --
265
- # the same ordering fetch_url relies on.
266
- response.raise_for_status()
267
- chunks = []
268
- total = 0
269
- for chunk in response.iter_bytes():
270
- total += len(chunk)
271
- if total > MAX_FEED_BYTES:
272
- raise _FeedTooLarge(total)
273
- chunks.append(chunk)
274
- encoding = getattr(response, "encoding", None) or "utf-8"
275
-
276
- # errors="replace", and it is not laziness. A bare decode raises
277
- # UnicodeDecodeError on a feed that is not what its header claims, and
278
- # that is a ValueError -- which NEITHER except clause in `run()` covers,
279
- # so it would escape as a crash on precisely the hostile path the rest
280
- # of this module is careful about. Replacing instead makes a nonsense
281
- # body fail as a ParseError, which IS handled and IS retried.
282
- #
283
- # §51's PG5 concern about a cut multi-byte character does not apply:
284
- # these bytes are always a COMPLETE document, because an incomplete one
285
- # was refused above rather than truncated.
286
- return b"".join(chunks).decode(encoding, errors="replace")
287
-
288
-
289
- # ---- Atom XML parsing --------------------------------------------------
290
-
291
- def _total_results(root) -> Optional[int]:
292
- """How many papers the query has, from OpenSearch's own element (PG3).
293
-
294
- Optional rather than assumed: the element is arXiv's to emit, and a
295
- result that reported a total it had invented would be worse than one
296
- that reports none.
297
- """
298
- element = root.find("opensearch:totalResults", OPENSEARCH_NS)
299
- if element is None or not (element.text or "").strip():
300
- return None
301
- try:
302
- return int(element.text.strip())
303
- except ValueError:
304
- return None
305
-
306
-
307
- def _text_of(parent, path) -> Optional[str]:
308
- """One child element's stripped text, or None.
309
-
310
- EVERY lookup in `_parse_atom_feed` goes through this, because the six
311
- ways that function used to raise were six slightly different spellings
312
- of the same omission (batch 117, TECHNICAL_DEBT 33):
313
-
314
- * a MISSING element -- `find()` returns None and `.text` raises
315
- AttributeError. Three of the six.
316
- * a PRESENT but EMPTY element -- `<title></title>` has `.text` of
317
- None, so `.strip()` raises AttributeError and `[:10]` raises
318
- TypeError. The remaining three, and they are the ones a reader
319
- skips, because `summary` looked guarded by `or ""` and `published`
320
- looked guarded by `is not None`. Neither covers both halves.
321
-
322
- The feed is attacker-influenced -- this module's header says so, and
323
- batch 116 added a parser that refuses hostile ENTITIES one function
324
- away -- so its STRUCTURE cannot be trusted either.
325
- """
326
- element = parent.find(path, ATOM_NS)
327
- if element is None or element.text is None:
328
- return None
329
- stripped = element.text.strip()
330
- return stripped or None
331
-
332
-
333
- def _parse_atom_feed(xml_text: str) -> tuple:
334
- """(entries, how many the query has in total).
335
-
336
- The total is OpenSearch's, not a count of what came back -- the point
337
- of it is to say what was NOT returned, which a count of what was cannot.
338
- """
339
- root = ET.fromstring(xml_text)
340
- entries = root.findall("atom:entry", ATOM_NS)
341
- total = _total_results(root)
342
-
343
- third = ThirdPartyText("arxiv_search")
344
- results = []
345
- for index, entry in enumerate(entries):
346
- raw_id = _text_of(entry, "atom:id")
347
- title_text = _text_of(entry, "atom:title")
348
- if raw_id is None or title_text is None:
349
- # SKIPPED, NOT RAISED (batch 117). An id and a title are what
350
- # make an entry citable; without either there is nothing for
351
- # the model to ground a claim on, so the entry is dropped and
352
- # the rest of the page is still returned. Raising here aborted
353
- # the whole search over one malformed entry, and the raise did
354
- # not even reach `run`'s handlers -- AttributeError is neither
355
- # a ParseError nor a DefusedXmlException.
356
- #
357
- # The index, not the id: the id is what might be missing.
358
- logger.warning(
359
- "arxiv_search skipped entry %d of %d: no %s",
360
- index + 1, len(entries),
361
- "id" if raw_id is None else "title")
362
- continue
363
- raw_arxiv_id = (raw_id.split("/abs/")[-1]
364
- if "/abs/" in raw_id else raw_id)
365
- # THE ID IS NOT JUST DISPLAYED, it is the input to the two recovery
366
- # URLs below, so an unbounded one is three unbounded values wearing
367
- # one name -- measured (batch 156), a 5,000-character `<id>` made
368
- # `full_summary_url` 5,022 characters long. Cut and flagged like any
369
- # other text (A58); what the cut costs is handled where the routes
370
- # are built, because a route assembled from half an id is a route to
371
- # nowhere and that is the one thing a route must not be.
372
- arxiv_id, id_truncated = third.text(raw_arxiv_id, _MAX_ID_CHARS)
373
-
374
- title, title_truncated = third.text(
375
- title_text.replace("\n", " "), MAX_TITLE_CHARS)
376
- summary, summary_truncated = third.text(
377
- (_text_of(entry, "atom:summary") or "").replace("\n", " "),
378
- MAX_SUMMARY_CHARS)
379
- # §51 (PG8/PG4). The cap stays -- it bounds model-facing text from a
380
- # source nobody here wrote, which is #131's lesson -- but a value cut
381
- # at it now SAYS it was cut. Until §51 a 600-character abstract and
382
- # one that happened to be 600 characters long were the same thing to
383
- # a reader, so the model could not tell an abstract it had all of
384
- # from one it had the first two thirds of.
385
- #
386
- # REDACTED ABOVE THE CUT SINCE BATCH 156 (RB1, A59), which is the
387
- # half §51 got the wrong way round. Both of these were sliced and
388
- # `registry.dispatch` redacted the slice; measured, a `ghp_` token
389
- # straddling either cut survived post-hoc redaction at 36 of its 39
390
- # possible offsets. `fetch_url` had already fixed the same ordering
391
- # for its own body, and `param_digest` had written down the reason
392
- # before either of them. The flags now count REDACTED characters,
393
- # which is RB3's consequence: redaction can shrink a value back
394
- # under the cap, and when it does the cut is withholding nothing.
395
-
396
- # `is not None` on the ELEMENT was not enough: `<published></published>`
397
- # is present with `.text` of None, and `None[:10]` is a TypeError
398
- # rather than an AttributeError -- the same omission wearing a
399
- # different exception type, which is why both halves go through
400
- # one helper now.
401
- published_text = _text_of(entry, "atom:published")
402
- published = published_text[:10] if published_text else None
403
-
404
- # A NAMELESS ELEMENT IS DROPPED RATHER THAN INSERTED AS None, and
405
- # this is one rule for BOTH lists below (TECHNICAL_DEBT 47). Both
406
- # come from the same attacker-influenced feed and both reach the
407
- # model the same way, so they must not disagree about it: a `None`
408
- # in a list rendered into a citation reads as an author called None
409
- # and as a category called None.
410
- #
411
- # `categories` kept its None until batch 127. The rule was stated
412
- # here, two lines above the list that did not follow it, which is
413
- # why the comment now names both instead of being copied.
414
- #
415
- # BOUNDED SINCE BATCH 126, and `authors` is the field
416
- # TECHNICAL_DEBT 41 was about without naming it. The count is kept
417
- # before the cut because `author_count` reports the real total --
418
- # the point of the key is that the model can tell a 2,832-author
419
- # paper from a 25-author one, which the truncated list alone cannot
420
- # say.
421
- #
422
- # DROPPED BEFORE THE CUT for both, so the cap counts values the
423
- # model can actually use: filtering after the slice would let a
424
- # nameless element consume one of the 25 or 20 places.
425
- #
426
- # AND THE CAP ON EACH VALUE, not only on the count (batch 156). The
427
- # two bounds answer different questions and the lists had only the
428
- # first: 25 authors of 5,000 characters each is 125,000 characters
429
- # of model-facing text that every count-based cap reports as
430
- # bounded. Each value is cut through the same helper as the title
431
- # and the abstract, so it is redacted first for the same reason.
432
- all_authors = [
433
- name for name in (
434
- third.text(_text_of(author, "atom:name"),
435
- _MAX_AUTHOR_CHARS)[0]
436
- for author in entry.findall("atom:author", ATOM_NS)
437
- ) if name
438
- ]
439
- author_count = len(all_authors)
440
- authors = all_authors[:_MAX_AUTHORS]
441
- all_categories = [
442
- term for term in (
443
- third.text(c.get("term"), _MAX_CATEGORY_CHARS)[0]
444
- for c in entry.findall("atom:category", ATOM_NS)
445
- ) if term
446
- ]
447
- categories = all_categories[:MAX_CATEGORIES]
448
-
449
- # DROPPED, NOT CUT (A58), and this is the field that makes the
450
- # distinction concrete. `pdf_url` exists to be fetched; cut to a cap
451
- # it becomes a fetchable-looking string that is not the paper, and
452
- # the model spends a tool call finding that out. Measured, a
453
- # 5,021-character href passed through byte-for-byte before this.
454
- # `MAX_URL_CHARS` is fetch_url's own limit, so anything past it is a
455
- # route this harness would refuse anyway.
456
- pdf_url, pdf_withheld = None, False
457
- for link in entry.findall("atom:link", ATOM_NS):
458
- if link.get("title") == "pdf":
459
- pdf_url, pdf_withheld = third.url(link.get("href"),
460
- MAX_URL_CHARS)
461
-
462
- entry_result = {
463
- "arxiv_id": arxiv_id,
464
- "title": title,
465
- "authors": authors,
466
- "summary": summary,
467
- "published": published,
468
- "categories": categories,
469
- "pdf_url": pdf_url,
470
- }
471
- if summary_truncated:
472
- # PG4's whole complaint: a truncated value that says nothing
473
- # about how to recover the rest is what sends the model to the
474
- # shell. The abs page is the route, and it is a URL `fetch_url`
475
- # will take.
476
- entry_result["summary_truncated"] = True
477
- if not id_truncated:
478
- entry_result["full_summary_url"] = (
479
- f"https://arxiv.org/abs/{arxiv_id}")
480
- if author_count > _MAX_AUTHORS:
481
- # PG8's shape exactly, including the recovery route PG4 asks
482
- # for: the abs page lists every author and `fetch_url` will
483
- # take it. `full_authors_url` rather than reusing
484
- # `full_summary_url` -- same URL, but a key named for the
485
- # summary carrying the author route is a name that lies, and
486
- # the summary's key is absent whenever the abstract fit.
487
- entry_result["authors_truncated"] = True
488
- entry_result["author_count"] = author_count
489
- # GUARDED ON THE ID, like `full_summary_url` above and for the
490
- # same reason (A58): the route is assembled from `arxiv_id`, so
491
- # an id that was cut yields a URL that 404s. PG4 asks for a
492
- # route to what the model WANTED; a broken one answers a
493
- # question nobody asked, and `authors_truncated` still says the
494
- # list was cut.
495
- if not id_truncated:
496
- entry_result["full_authors_url"] = (
497
- f"https://arxiv.org/abs/{arxiv_id}")
498
- # THESE TWO GET A FLAG AND NO ROUTE, and the asymmetry is deliberate
499
- # rather than an omission to be tidied up later. #49's rule is that
500
- # nothing is altered silently, so the flag is owed. But PG4 exists so
501
- # the model can reach what it WANTED, and a title past 300 characters
502
- # or a twenty-first category is a malformed feed rather than a long
503
- # paper -- measured, the longest title in sixty real entries was 188.
504
- # Pointing the model at the abs page of a hostile entry would be
505
- # answering a question nobody asked.
506
- if title_truncated:
507
- entry_result["title_truncated"] = True
508
- if len(all_categories) > MAX_CATEGORIES:
509
- entry_result["categories_truncated"] = True
510
- # THE THREE FLAGS BATCH 156 OWES, same rule (#49): nothing here is
511
- # altered silently. `id_truncated` is the one that matters most,
512
- # because a cut id is a citation that does not resolve and the model
513
- # has no other way to know.
514
- if id_truncated:
515
- entry_result["id_truncated"] = True
516
- if pdf_withheld:
517
- entry_result["pdf_url_truncated"] = True
518
- results.append(entry_result)
519
-
520
- third.report()
521
- return results, total
522
-
523
-
524
- # ---- Entry point called by the tool dispatcher -----------------------------
525
-
526
- def run(params: dict) -> dict:
527
- parsed = ArxivSearchParams(**params)
528
- # §51 (PG9). `start` is part of the key because it changes the response.
529
- # Without it page 2 of a search would be served page 1 out of the cache
530
- # -- the same class of collision this cache's own docstring warns about
531
- # between the two tools, arriving instead between two pages of one
532
- # query, and silent in exactly the same way.
533
- cache_key = (f"{parsed.keywords}|{parsed.category}|{parsed.max_results}"
534
- f"|{parsed.sort_by}|{parsed.start}")
535
-
536
- cached = _cache.get(cache_key)
537
- total: Optional[int] = None
538
- if cached is not None:
539
- logger.info("arxiv_search cache hit for %r", parsed.keywords)
540
- results, total = cached
541
- else:
542
- search_query = _build_search_query(parsed.keywords, parsed.category)
543
- last_exc: Optional[Exception] = None
544
- results = []
545
- for attempt in range(MAX_RETRIES + 1):
546
- try:
547
- xml_text = _call_arxiv_api(search_query, parsed.max_results,
548
- parsed.sort_by, parsed.start)
549
- results, total = _parse_atom_feed(xml_text)
550
- # A62. `max_results` is advertised `le=20` and reaches the
551
- # provider, which is free to ignore it -- measured (batch
552
- # 156), a feed of 5,000 entries against a request for 5
553
- # returned all 5,000. The per-entry caps above are
554
- # multiplied by this number, so bounding them without
555
- # bounding it is not a bound. `total` is unaffected and
556
- # still reports what the QUERY has, which is the figure
557
- # that says what was not returned.
558
- if len(results) > parsed.max_results:
559
- logger.warning(
560
- "arxiv_search kept %d of %d entries: the feed "
561
- "carried more than the %d asked for.",
562
- parsed.max_results, len(results), parsed.max_results)
563
- results = results[:parsed.max_results]
564
- _cache.set(cache_key, (results, total))
565
- break
566
- except _FeedTooLarge:
567
- # NOT RETRIED, for the reason spelled out one clause down
568
- # for a declared entity: a feed this large is this large
569
- # again, so two more round trips reach the same place and
570
- # then report "failed after 3 attempts", which describes a
571
- # network problem that is not there.
572
- #
573
- # Returned rather than raised, like every other dead end in
574
- # this function: a search tool that cannot answer is a
575
- # result the model can work around.
576
- logger.warning(
577
- "arxiv_search dropped a feed over %d bytes for %r "
578
- "(max_results=%d, start=%d)",
579
- MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
580
- parsed.start)
581
- # "more than", not a size. `total` stopped at the first
582
- # chunk past the cap, so printing it would be an instrument
583
- # reporting a magnitude it did not measure -- the
584
- # instrument-that-lies shape batches 118 and 119 each paid
585
- # for once. The lever IS named, because the model controls
586
- # it and a smaller page really does fit.
587
- return {
588
- "error": "arXiv returned more than %d bytes for this "
589
- "query, so nothing was read from it. Retry "
590
- "with a smaller max_results."
591
- % MAX_FEED_BYTES
592
- }
593
- except DefusedXmlException as e:
594
- # NOT folded into the tuple below, and that is the point.
595
- # A malformed body is transient and worth three attempts;
596
- # a feed that DECLARES an entity is not -- it refuses the
597
- # same way every time, so retrying spends two more round
598
- # trips to reach the same place and then reports "failed
599
- # after 3 attempts", which describes a network problem
600
- # that is not there.
601
- #
602
- # Returned rather than raised, for the reason the retry
603
- # exhaustion below is returned: a search tool that cannot
604
- # answer is a result the model can work around. These do
605
- # NOT subclass ParseError -- they are ValueError -- so
606
- # without this clause they would propagate out of run().
607
- logger.warning(
608
- "arxiv_search refused a hostile feed for %r: %s",
609
- parsed.keywords, e)
610
- # THE FEED'S OWN TEXT STOPS HERE (batch 116's review).
611
- # `e`'s repr carries the entity's `system_id` verbatim from
612
- # the feed, so interpolating `{e}` would put a string the
613
- # ATTACKER chose into the model's context -- the one place
614
- # §32's untrusted-content paragraph exists to bound. The
615
- # log above keeps the whole exception, because it is
616
- # operator-facing and the formatter redaction sink guards
617
- # app.log; the model gets the category and, where the
618
- # exception carries one, the entity NAME, which XML's Name
619
- # production cannot make into a sentence.
620
- #
621
- # getattr, not `e.name`: ExternalReferenceForbidden has no
622
- # `.name`, and an AttributeError raised while building an
623
- # error message would replace a handled refusal with a
624
- # crash on the hostile path.
625
- declared = getattr(e, "name", None)
626
- named = (" -- it declared the entity %r"
627
- % str(declared)[:40]) if declared else ""
628
- return {
629
- "error": "arXiv returned a feed this parser refuses to "
630
- "expand, so nothing was read from it: %s%s"
631
- % (type(e).__name__, named)
632
- }
633
- except (httpx.HTTPError, ET.ParseError) as e:
634
- last_exc = e
635
- logger.warning(
636
- "arxiv_search attempt %s/%s failed for %r: %s",
637
- attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
638
- else:
639
- # RETURNED, not raised. A search tool that cannot reach its
640
- # provider is a result the model can work around -- try
641
- # web_search, or say the claim could not be grounded. Raising
642
- # made one transient network failure, in one pass, fail an
643
- # entire ten-pass research run; fetch_url has always returned
644
- # an error dict here and arxiv_search was the outlier.
645
- #
646
- # dispatch() now contains a raise from any tool as well, so
647
- # this is the message rather than the mechanism: the model
648
- # gets something specific instead of a generic wrapper.
649
- return {
650
- "error": f"arXiv search failed after {MAX_RETRIES + 1} "
651
- f"attempts: {last_exc}"
652
- }
653
-
654
- if not results:
655
- if parsed.start:
656
- # Distinguishable from "this query has nothing", which is what
657
- # the bare message would have said about a page past the end.
658
- return {
659
- "results": [], "result_count": 0, "start": parsed.start,
660
- "message": (
661
- f"No papers at start={parsed.start}. The search has "
662
- f"{total} in total." if total is not None else
663
- f"No papers at start={parsed.start}, which is past the "
664
- f"end of this search."
665
- ),
666
- }
667
- return {"results": [], "result_count": 0, "message": "No papers found."}
668
-
669
- result = {"results": results, "result_count": len(results),
670
- "start": parsed.start}
671
-
672
- # PG4: say how to get the rest, not merely that there is a rest.
673
- next_start = parsed.start + len(results)
674
- if total is not None:
675
- result["total_results"] = total
676
- if next_start < total:
677
- result["message"] = (
678
- f"Returned papers {parsed.start + 1}-{next_start} of {total}. "
679
- f"Use start={next_start} to see more."
680
- )
681
- return result
1
+ """
2
+ tools/builtin/arxiv.py
3
+
4
+ Searches arXiv for academic papers by keyword and optional subject
5
+ category, using arXiv's public query API. No API key needed. Useful
6
+ alongside web_search/fetch_url for grounding claims in published research
7
+ during the source_grounding pass.
8
+
9
+ The arXiv API returns Atom 1.0 XML (not JSON) -- this file's job is
10
+ largely translating that into the same normalized result shape the other
11
+ tools use.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import logging
17
+ from typing import Optional
18
+
19
+ # TECHNICAL_DEBT 18, batch 116. This feed is attacker-influenced -- arXiv
20
+ # echoes the query back and the entries are third-party text -- and stdlib
21
+ # ElementTree expands internal entities, which is Bandit B405/B314 and was
22
+ # an accepted risk for the CI rollout rather than a decision that it was
23
+ # safe. `defusedxml.ElementTree` is the drop-in: `fromstring` refuses
24
+ # declared entities and external references, and it re-exports the SAME
25
+ # `ParseError` object as the stdlib, so the empty-redirect-body path below
26
+ # and the retry loop's except clause are unaffected. Measured, not assumed:
27
+ # stdlib expands `&lol2;` to "lollollol" where this raises
28
+ # EntitiesForbidden.
29
+ import defusedxml.ElementTree as ET
30
+ import httpx
31
+ from defusedxml import DefusedXmlException
32
+ from pydantic import BaseModel, Field, field_validator
33
+
34
+ from tools.builtin import _net_common
35
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+ # HTTPS, and it is load-bearing. arXiv 301-redirects the http:// form, and
40
+ # three things then combine into a hard failure that looks nothing like a
41
+ # redirect: httpx does NOT follow redirects by default (unlike requests),
42
+ # raise_for_status() only raises on >= 400 so a 301 sails through, and the
43
+ # redirect body is empty -- so ET.fromstring("") raises ParseError, the
44
+ # retry loop runs three identical attempts, and the tool reports "arXiv
45
+ # search failed after 3 attempts". Verified against the live API.
46
+ ARXIV_API_URL = "https://export.arxiv.org/api/query"
47
+ ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
48
+
49
+ # ROADMAP_v3 §51 (PG3). arXiv answers in OpenSearch's paging elements as
50
+ # well as Atom's entries, and they are what let a result say how many papers
51
+ # the query has rather than only that it has more. Measured (batch 105):
52
+ # `all:transformer attention` reports totalResults=261387, with startIndex
53
+ # and itemsPerPage echoing the request.
54
+ OPENSEARCH_NS = {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"}
55
+
56
+ _VALID_SORT_BY = {"relevance", "lastUpdatedDate", "submittedDate"}
57
+
58
+ MAX_SUMMARY_CHARS = 600
59
+ REQUEST_TIMEOUT_S = 10.0
60
+ MAX_RETRIES = 2
61
+ CACHE_TTL_S = 300
62
+
63
+ # ROADMAP_v2 §31 (H7), #55, applied to the second tool that had the defect.
64
+ # `httpx.get(...).text` buffers the ENTIRE body before anything looks at it,
65
+ # and fetch_url.py's own comment rules out the shortcut: a Content-Length
66
+ # check is not the fix, because a hostile server can omit it or lie about
67
+ # it, so the only thing that bounds this is refusing to read past a cap.
68
+ # This feed is attacker-influenced -- the module header says so -- and this
69
+ # was the last `httpx.get` in the project.
70
+ #
71
+ # DERIVED FROM A MEASUREMENT, and the first derivation was wrong. It guessed
72
+ # "roughly 100 KB of Atom per hyperauthorship entry" and therefore 2 MiB for
73
+ # twenty of them; measured live, `all:LHCb observation branching fraction`
74
+ # answers max_results=20 with 838,560 bytes carrying 13,859 <author>
75
+ # elements, about 42 KB an entry, and four consecutive pages of it stayed
76
+ # under 839 KB. So 2 MiB clears the heaviest LEGITIMATE feed this tool can
77
+ # ask for by 2.5x -- which is the number that matters, because a cap under
78
+ # it is a FALSE REFUSAL: the tool reporting it read nothing for a query
79
+ # arXiv answered correctly.
80
+ #
81
+ # Loose next to fetch_url's 65,536 on purpose. That tool's URL is chosen by
82
+ # the model and can point anywhere; this endpoint is fixed and its response
83
+ # shape is known, so the bound's job here is the refusal rather than the
84
+ # tightness.
85
+ MAX_FEED_BYTES = 2_097_152
86
+
87
+ # ROADMAP_v3 §51 (PG8) for the shape, #131 for the reason: these bound
88
+ # model-facing text from a source nobody here wrote. The abstract was the
89
+ # only field anyone capped and it is not the field that grew -- measured
90
+ # (TECHNICAL_DEBT 41, re-measured in batch 126) a `str(result)` of 223,191
91
+ # characters, of which 204,871 (91.8%) were the author list, with one entry
92
+ # carrying 2,832 authors and eleven of twenty over the cap below. Everything
93
+ # the tool already bounds ceilings at about 16,900 characters even at
94
+ # max_results=20, so the capped fields could never have accounted for it.
95
+ #
96
+ # 25 is `core/reasoning/scholar.py`'s `_MAX_AUTHORS`, answering the same
97
+ # question the same way -- enough to find the strongest name on a paper
98
+ # without carrying a hyperauthorship record. NOT imported: tools/ does not
99
+ # import core/reasoning/, and one constant reaching across two layers would
100
+ # be worse than two constants with one reason each.
101
+ _MAX_AUTHORS = 25
102
+
103
+ # Measured over sixty real entries from three queries: the longest title was
104
+ # 188 characters ("Measurements of the Higgs boson production and decay
105
+ # rates..."). So this is inert for every real paper and bounds the malformed
106
+ # one -- which is why both of these get a FLAG and no recovery route. See
107
+ # `_parse_atom_feed`.
108
+ MAX_TITLE_CHARS = 300
109
+ MAX_CATEGORIES = 20
110
+
111
+ # Batch 156 (ROADMAP_v3 §73). THE CAPS ABOVE BOUND THE LISTS AND NOT THE
112
+ # VALUES IN THEM, which is the "which axis" question of batch 126 asked
113
+ # again one level down: `authors` kept 25 of them and `categories` 20, and
114
+ # a single `<name>` or a single `term` could be any length at all.
115
+ # Measured (batch 156) through the real parser, a 5,000-character author
116
+ # name, a 5,000-character category term and a 5,021-character pdf href all
117
+ # came back byte-for-byte while the 5,000-character title in the same entry
118
+ # was cut to 300 with its flag -- the control that proves the probe was
119
+ # reaching the caps.
120
+ #
121
+ # DERIVED, each from the longest legitimate shape of the value it bounds,
122
+ # because a cap that trims an ordinary entry turns its own flag into noise:
123
+ #
124
+ # * an author name is a person or a collaboration byline. The longest
125
+ # plausible is the latter ("The ATLAS and CMS Collaborations" is 32
126
+ # characters); 200 clears every real one several times over.
127
+ # * a category `term` comes from arXiv's published taxonomy, whose
128
+ # longest member is `cond-mat.stat-mech` at 18 characters. The longest
129
+ # appearing anywhere in this repository is `physics.gen-ph` at 14.
130
+ # * an id is `2401.00001v1` (12) in the new scheme or
131
+ # `math.GT/0309136v2` (17) in the old -- 17 is the longest of either
132
+ # form present in this repository, and 64 is not reachable by a
133
+ # well-formed one.
134
+ _MAX_AUTHOR_CHARS = 200
135
+ _MAX_CATEGORY_CHARS = 40
136
+ _MAX_ID_CHARS = 64
137
+
138
+ # One instance per tool, never shared: an equal key string from an
139
+ # arXiv query and a web query would otherwise collide.
140
+ _cache = TTLCache(CACHE_TTL_S)
141
+
142
+
143
+ # ---- Schema exposed to the LLM -------------------------------------------
144
+
145
+ class ArxivSearchParams(BaseModel):
146
+ keywords: str = Field(..., description="Search keywords", min_length=1, max_length=400)
147
+ category: Optional[str] = Field(
148
+ None,
149
+ description=(
150
+ "Optional arXiv category code to restrict the search, e.g. "
151
+ "'cs.AI', 'cs.LG', 'physics.gen-ph', 'q-bio.NC'. Omit to search "
152
+ "all categories."
153
+ ),
154
+ )
155
+ max_results: int = Field(5, ge=1, le=20, description="Number of results to return")
156
+ start: int = Field(
157
+ 0,
158
+ ge=0,
159
+ description=(
160
+ "Number of results to skip before returning any (0-based). Use "
161
+ "the start named in a result's message to page further into a "
162
+ "search that has more papers than were returned."
163
+ ),
164
+ )
165
+ sort_by: str = Field(
166
+ "relevance",
167
+ description="How to sort results: 'relevance', 'lastUpdatedDate', or 'submittedDate'",
168
+ )
169
+
170
+ @field_validator("keywords")
171
+ @classmethod
172
+ def strip_keywords(cls, value: str) -> str:
173
+ value = value.strip()
174
+ if not value:
175
+ raise ValueError("keywords cannot be empty")
176
+ return value
177
+
178
+ @field_validator("sort_by")
179
+ @classmethod
180
+ def validate_sort_by(cls, value: str) -> str:
181
+ if value not in _VALID_SORT_BY:
182
+ raise ValueError(f"sort_by must be one of {_VALID_SORT_BY}")
183
+ return value
184
+
185
+
186
+ TOOL_SCHEMA = {
187
+ "name": "arxiv_search",
188
+ "description": (
189
+ "Search arXiv for academic papers by keyword and optional subject "
190
+ "category. Returns titles, authors, abstracts, publish dates, and "
191
+ "PDF links -- use this to ground claims in published research."
192
+ ),
193
+ "input_schema": ArxivSearchParams.model_json_schema(),
194
+ }
195
+
196
+
197
+ class ArxivSearchError(Exception):
198
+ """No longer raised by this module -- exhausted retries return an
199
+ error dict instead, so a transient network failure cannot fail a
200
+ ten-pass research run. Kept because it is part of this module's
201
+ public surface and something outside the repo may catch it."""
202
+
203
+
204
+ class _FeedTooLarge(Exception):
205
+ """The feed passed MAX_FEED_BYTES and was dropped UNREAD.
206
+
207
+ PRIVATE, and it never escapes `run()`. Deliberately not
208
+ `ArxivSearchError`: that one is documented above as part of this
209
+ module's public surface and as no longer raised, and quietly starting
210
+ to raise it again would make that docstring false.
211
+
212
+ A separate exception rather than a sentinel return because the two
213
+ outcomes are not the same shape -- every other path returns a feed to
214
+ parse, and this one has nothing to parse. `fetch_url` can `break` and
215
+ keep a truncated body because a cut page of prose is still prose; a cut
216
+ Atom document does not parse at all, so there is nothing to hand back.
217
+ """
218
+
219
+
220
+ # ---- Query construction -----------------------------------------------------
221
+
222
+ def _build_search_query(keywords: str, category: Optional[str]) -> str:
223
+ """
224
+ arXiv's query syntax uses field prefixes (ti:, abs:, au:, cat:, all:)
225
+ combined with AND/OR/ANDNOT. `all:` searches title, abstract, and
226
+ author. httpx handles URL-encoding the spaces/quotes once this string
227
+ is passed as a params value, so no manual encoding is needed here.
228
+ """
229
+ if category:
230
+ return f"cat:{category} AND all:{keywords}"
231
+ return f"all:{keywords}"
232
+
233
+
234
+ # ---- Provider call ----------------------------------------------------------
235
+
236
+ def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
237
+ start: int = 0) -> str:
238
+ """The raw Atom XML, or `_FeedTooLarge` if it would not fit the bound.
239
+
240
+ STREAMED since batch 126 (§31 H7), so the connection is dropped inside
241
+ the `with` the moment the cap is passed rather than after the whole body
242
+ is already in memory. Returns `str` as it always did, so every test that
243
+ patches this function is unaffected by the change underneath it.
244
+ """
245
+ # ROADMAP_v3 §53: `httpx.stream`, admitted and counted against the tool
246
+ # meter. A refusal raises through `run()`'s except clauses, which name
247
+ # httpx's and the parser's errors only, to dispatch.
248
+ with _net_common.stream(
249
+ "GET",
250
+ ARXIV_API_URL,
251
+ params={
252
+ "search_query": search_query,
253
+ # §51 (PG3). Was hardcoded to 0, which made every page of every
254
+ # search the first one: the provider had implemented paging all
255
+ # along and this tool spent the parameter on a constant.
256
+ "start": start,
257
+ "max_results": max_results,
258
+ "sortBy": sort_by,
259
+ "sortOrder": "descending",
260
+ },
261
+ headers={"User-Agent": "agent-harness-arxiv-tool/1.0 (research pipeline)"},
262
+ timeout=REQUEST_TIMEOUT_S,
263
+ # Belt and braces beside the https:// above. If arXiv moves the
264
+ # endpoint again, following the redirect keeps this working
265
+ # instead of failing as an unparseable empty body.
266
+ follow_redirects=True,
267
+ ) as response:
268
+ # Streamed, so status and headers are available before the body --
269
+ # the same ordering fetch_url relies on.
270
+ response.raise_for_status()
271
+ chunks = []
272
+ total = 0
273
+ for chunk in response.iter_bytes():
274
+ total += len(chunk)
275
+ if total > MAX_FEED_BYTES:
276
+ raise _FeedTooLarge(total)
277
+ chunks.append(chunk)
278
+ encoding = getattr(response, "encoding", None) or "utf-8"
279
+
280
+ # errors="replace", and it is not laziness. A bare decode raises
281
+ # UnicodeDecodeError on a feed that is not what its header claims, and
282
+ # that is a ValueError -- which NEITHER except clause in `run()` covers,
283
+ # so it would escape as a crash on precisely the hostile path the rest
284
+ # of this module is careful about. Replacing instead makes a nonsense
285
+ # body fail as a ParseError, which IS handled and IS retried.
286
+ #
287
+ # §51's PG5 concern about a cut multi-byte character does not apply:
288
+ # these bytes are always a COMPLETE document, because an incomplete one
289
+ # was refused above rather than truncated.
290
+ return b"".join(chunks).decode(encoding, errors="replace")
291
+
292
+
293
+ # ---- Atom XML parsing --------------------------------------------------
294
+
295
+ def _total_results(root) -> Optional[int]:
296
+ """How many papers the query has, from OpenSearch's own element (PG3).
297
+
298
+ Optional rather than assumed: the element is arXiv's to emit, and a
299
+ result that reported a total it had invented would be worse than one
300
+ that reports none.
301
+ """
302
+ element = root.find("opensearch:totalResults", OPENSEARCH_NS)
303
+ if element is None or not (element.text or "").strip():
304
+ return None
305
+ try:
306
+ return int(element.text.strip())
307
+ except ValueError:
308
+ return None
309
+
310
+
311
+ def _text_of(parent, path) -> Optional[str]:
312
+ """One child element's stripped text, or None.
313
+
314
+ EVERY lookup in `_parse_atom_feed` goes through this, because the six
315
+ ways that function used to raise were six slightly different spellings
316
+ of the same omission (batch 117, TECHNICAL_DEBT 33):
317
+
318
+ * a MISSING element -- `find()` returns None and `.text` raises
319
+ AttributeError. Three of the six.
320
+ * a PRESENT but EMPTY element -- `<title></title>` has `.text` of
321
+ None, so `.strip()` raises AttributeError and `[:10]` raises
322
+ TypeError. The remaining three, and they are the ones a reader
323
+ skips, because `summary` looked guarded by `or ""` and `published`
324
+ looked guarded by `is not None`. Neither covers both halves.
325
+
326
+ The feed is attacker-influenced -- this module's header says so, and
327
+ batch 116 added a parser that refuses hostile ENTITIES one function
328
+ away -- so its STRUCTURE cannot be trusted either.
329
+ """
330
+ element = parent.find(path, ATOM_NS)
331
+ if element is None or element.text is None:
332
+ return None
333
+ stripped = element.text.strip()
334
+ return stripped or None
335
+
336
+
337
+ def _parse_atom_feed(xml_text: str) -> tuple:
338
+ """(entries, how many the query has in total).
339
+
340
+ The total is OpenSearch's, not a count of what came back -- the point
341
+ of it is to say what was NOT returned, which a count of what was cannot.
342
+ """
343
+ root = ET.fromstring(xml_text)
344
+ entries = root.findall("atom:entry", ATOM_NS)
345
+ total = _total_results(root)
346
+
347
+ third = ThirdPartyText("arxiv_search")
348
+ results = []
349
+ for index, entry in enumerate(entries):
350
+ raw_id = _text_of(entry, "atom:id")
351
+ title_text = _text_of(entry, "atom:title")
352
+ if raw_id is None or title_text is None:
353
+ # SKIPPED, NOT RAISED (batch 117). An id and a title are what
354
+ # make an entry citable; without either there is nothing for
355
+ # the model to ground a claim on, so the entry is dropped and
356
+ # the rest of the page is still returned. Raising here aborted
357
+ # the whole search over one malformed entry, and the raise did
358
+ # not even reach `run`'s handlers -- AttributeError is neither
359
+ # a ParseError nor a DefusedXmlException.
360
+ #
361
+ # The index, not the id: the id is what might be missing.
362
+ logger.warning(
363
+ "arxiv_search skipped entry %d of %d: no %s",
364
+ index + 1, len(entries),
365
+ "id" if raw_id is None else "title")
366
+ continue
367
+ raw_arxiv_id = (raw_id.split("/abs/")[-1]
368
+ if "/abs/" in raw_id else raw_id)
369
+ # THE ID IS NOT JUST DISPLAYED, it is the input to the two recovery
370
+ # URLs below, so an unbounded one is three unbounded values wearing
371
+ # one name -- measured (batch 156), a 5,000-character `<id>` made
372
+ # `full_summary_url` 5,022 characters long. Cut and flagged like any
373
+ # other text (A58); what the cut costs is handled where the routes
374
+ # are built, because a route assembled from half an id is a route to
375
+ # nowhere and that is the one thing a route must not be.
376
+ arxiv_id, id_truncated = third.text(raw_arxiv_id, _MAX_ID_CHARS)
377
+
378
+ title, title_truncated = third.text(
379
+ title_text.replace("\n", " "), MAX_TITLE_CHARS)
380
+ summary, summary_truncated = third.text(
381
+ (_text_of(entry, "atom:summary") or "").replace("\n", " "),
382
+ MAX_SUMMARY_CHARS)
383
+ # §51 (PG8/PG4). The cap stays -- it bounds model-facing text from a
384
+ # source nobody here wrote, which is #131's lesson -- but a value cut
385
+ # at it now SAYS it was cut. Until §51 a 600-character abstract and
386
+ # one that happened to be 600 characters long were the same thing to
387
+ # a reader, so the model could not tell an abstract it had all of
388
+ # from one it had the first two thirds of.
389
+ #
390
+ # REDACTED ABOVE THE CUT SINCE BATCH 156 (RB1, A59), which is the
391
+ # half §51 got the wrong way round. Both of these were sliced and
392
+ # `registry.dispatch` redacted the slice; measured, a `ghp_` token
393
+ # straddling either cut survived post-hoc redaction at 36 of its 39
394
+ # possible offsets. `fetch_url` had already fixed the same ordering
395
+ # for its own body, and `param_digest` had written down the reason
396
+ # before either of them. The flags now count REDACTED characters,
397
+ # which is RB3's consequence: redaction can shrink a value back
398
+ # under the cap, and when it does the cut is withholding nothing.
399
+
400
+ # `is not None` on the ELEMENT was not enough: `<published></published>`
401
+ # is present with `.text` of None, and `None[:10]` is a TypeError
402
+ # rather than an AttributeError -- the same omission wearing a
403
+ # different exception type, which is why both halves go through
404
+ # one helper now.
405
+ published_text = _text_of(entry, "atom:published")
406
+ published = published_text[:10] if published_text else None
407
+
408
+ # A NAMELESS ELEMENT IS DROPPED RATHER THAN INSERTED AS None, and
409
+ # this is one rule for BOTH lists below (TECHNICAL_DEBT 47). Both
410
+ # come from the same attacker-influenced feed and both reach the
411
+ # model the same way, so they must not disagree about it: a `None`
412
+ # in a list rendered into a citation reads as an author called None
413
+ # and as a category called None.
414
+ #
415
+ # `categories` kept its None until batch 127. The rule was stated
416
+ # here, two lines above the list that did not follow it, which is
417
+ # why the comment now names both instead of being copied.
418
+ #
419
+ # BOUNDED SINCE BATCH 126, and `authors` is the field
420
+ # TECHNICAL_DEBT 41 was about without naming it. The count is kept
421
+ # before the cut because `author_count` reports the real total --
422
+ # the point of the key is that the model can tell a 2,832-author
423
+ # paper from a 25-author one, which the truncated list alone cannot
424
+ # say.
425
+ #
426
+ # DROPPED BEFORE THE CUT for both, so the cap counts values the
427
+ # model can actually use: filtering after the slice would let a
428
+ # nameless element consume one of the 25 or 20 places.
429
+ #
430
+ # AND THE CAP ON EACH VALUE, not only on the count (batch 156). The
431
+ # two bounds answer different questions and the lists had only the
432
+ # first: 25 authors of 5,000 characters each is 125,000 characters
433
+ # of model-facing text that every count-based cap reports as
434
+ # bounded. Each value is cut through the same helper as the title
435
+ # and the abstract, so it is redacted first for the same reason.
436
+ all_authors = [
437
+ name for name in (
438
+ third.text(_text_of(author, "atom:name"),
439
+ _MAX_AUTHOR_CHARS)[0]
440
+ for author in entry.findall("atom:author", ATOM_NS)
441
+ ) if name
442
+ ]
443
+ author_count = len(all_authors)
444
+ authors = all_authors[:_MAX_AUTHORS]
445
+ all_categories = [
446
+ term for term in (
447
+ third.text(c.get("term"), _MAX_CATEGORY_CHARS)[0]
448
+ for c in entry.findall("atom:category", ATOM_NS)
449
+ ) if term
450
+ ]
451
+ categories = all_categories[:MAX_CATEGORIES]
452
+
453
+ # DROPPED, NOT CUT (A58), and this is the field that makes the
454
+ # distinction concrete. `pdf_url` exists to be fetched; cut to a cap
455
+ # it becomes a fetchable-looking string that is not the paper, and
456
+ # the model spends a tool call finding that out. Measured, a
457
+ # 5,021-character href passed through byte-for-byte before this.
458
+ # `MAX_URL_CHARS` is fetch_url's own limit, so anything past it is a
459
+ # route this harness would refuse anyway.
460
+ pdf_url, pdf_withheld = None, False
461
+ for link in entry.findall("atom:link", ATOM_NS):
462
+ if link.get("title") == "pdf":
463
+ pdf_url, pdf_withheld = third.url(link.get("href"),
464
+ MAX_URL_CHARS)
465
+
466
+ entry_result = {
467
+ "arxiv_id": arxiv_id,
468
+ "title": title,
469
+ "authors": authors,
470
+ "summary": summary,
471
+ "published": published,
472
+ "categories": categories,
473
+ "pdf_url": pdf_url,
474
+ }
475
+ if summary_truncated:
476
+ # PG4's whole complaint: a truncated value that says nothing
477
+ # about how to recover the rest is what sends the model to the
478
+ # shell. The abs page is the route, and it is a URL `fetch_url`
479
+ # will take.
480
+ entry_result["summary_truncated"] = True
481
+ if not id_truncated:
482
+ entry_result["full_summary_url"] = (
483
+ f"https://arxiv.org/abs/{arxiv_id}")
484
+ if author_count > _MAX_AUTHORS:
485
+ # PG8's shape exactly, including the recovery route PG4 asks
486
+ # for: the abs page lists every author and `fetch_url` will
487
+ # take it. `full_authors_url` rather than reusing
488
+ # `full_summary_url` -- same URL, but a key named for the
489
+ # summary carrying the author route is a name that lies, and
490
+ # the summary's key is absent whenever the abstract fit.
491
+ entry_result["authors_truncated"] = True
492
+ entry_result["author_count"] = author_count
493
+ # GUARDED ON THE ID, like `full_summary_url` above and for the
494
+ # same reason (A58): the route is assembled from `arxiv_id`, so
495
+ # an id that was cut yields a URL that 404s. PG4 asks for a
496
+ # route to what the model WANTED; a broken one answers a
497
+ # question nobody asked, and `authors_truncated` still says the
498
+ # list was cut.
499
+ if not id_truncated:
500
+ entry_result["full_authors_url"] = (
501
+ f"https://arxiv.org/abs/{arxiv_id}")
502
+ # THESE TWO GET A FLAG AND NO ROUTE, and the asymmetry is deliberate
503
+ # rather than an omission to be tidied up later. #49's rule is that
504
+ # nothing is altered silently, so the flag is owed. But PG4 exists so
505
+ # the model can reach what it WANTED, and a title past 300 characters
506
+ # or a twenty-first category is a malformed feed rather than a long
507
+ # paper -- measured, the longest title in sixty real entries was 188.
508
+ # Pointing the model at the abs page of a hostile entry would be
509
+ # answering a question nobody asked.
510
+ if title_truncated:
511
+ entry_result["title_truncated"] = True
512
+ if len(all_categories) > MAX_CATEGORIES:
513
+ entry_result["categories_truncated"] = True
514
+ # THE THREE FLAGS BATCH 156 OWES, same rule (#49): nothing here is
515
+ # altered silently. `id_truncated` is the one that matters most,
516
+ # because a cut id is a citation that does not resolve and the model
517
+ # has no other way to know.
518
+ if id_truncated:
519
+ entry_result["id_truncated"] = True
520
+ if pdf_withheld:
521
+ entry_result["pdf_url_truncated"] = True
522
+ results.append(entry_result)
523
+
524
+ third.report()
525
+ return results, total
526
+
527
+
528
+ # ---- Entry point called by the tool dispatcher -----------------------------
529
+
530
+ def run(params: dict) -> dict:
531
+ parsed = ArxivSearchParams(**params)
532
+ # §51 (PG9). `start` is part of the key because it changes the response.
533
+ # Without it page 2 of a search would be served page 1 out of the cache
534
+ # -- the same class of collision this cache's own docstring warns about
535
+ # between the two tools, arriving instead between two pages of one
536
+ # query, and silent in exactly the same way.
537
+ cache_key = (f"{parsed.keywords}|{parsed.category}|{parsed.max_results}"
538
+ f"|{parsed.sort_by}|{parsed.start}")
539
+
540
+ cached = _cache.get(cache_key)
541
+ total: Optional[int] = None
542
+ if cached is not None:
543
+ logger.info("arxiv_search cache hit for %r", parsed.keywords)
544
+ results, total = cached
545
+ else:
546
+ search_query = _build_search_query(parsed.keywords, parsed.category)
547
+ last_exc: Optional[Exception] = None
548
+ results = []
549
+ for attempt in range(MAX_RETRIES + 1):
550
+ try:
551
+ xml_text = _call_arxiv_api(search_query, parsed.max_results,
552
+ parsed.sort_by, parsed.start)
553
+ results, total = _parse_atom_feed(xml_text)
554
+ # A62. `max_results` is advertised `le=20` and reaches the
555
+ # provider, which is free to ignore it -- measured (batch
556
+ # 156), a feed of 5,000 entries against a request for 5
557
+ # returned all 5,000. The per-entry caps above are
558
+ # multiplied by this number, so bounding them without
559
+ # bounding it is not a bound. `total` is unaffected and
560
+ # still reports what the QUERY has, which is the figure
561
+ # that says what was not returned.
562
+ if len(results) > parsed.max_results:
563
+ logger.warning(
564
+ "arxiv_search kept %d of %d entries: the feed "
565
+ "carried more than the %d asked for.",
566
+ parsed.max_results, len(results), parsed.max_results)
567
+ results = results[:parsed.max_results]
568
+ _cache.set(cache_key, (results, total))
569
+ break
570
+ except _FeedTooLarge:
571
+ # NOT RETRIED, for the reason spelled out one clause down
572
+ # for a declared entity: a feed this large is this large
573
+ # again, so two more round trips reach the same place and
574
+ # then report "failed after 3 attempts", which describes a
575
+ # network problem that is not there.
576
+ #
577
+ # Returned rather than raised, like every other dead end in
578
+ # this function: a search tool that cannot answer is a
579
+ # result the model can work around.
580
+ logger.warning(
581
+ "arxiv_search dropped a feed over %d bytes for %r "
582
+ "(max_results=%d, start=%d)",
583
+ MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
584
+ parsed.start)
585
+ # "more than", not a size. `total` stopped at the first
586
+ # chunk past the cap, so printing it would be an instrument
587
+ # reporting a magnitude it did not measure -- the
588
+ # instrument-that-lies shape batches 118 and 119 each paid
589
+ # for once. The lever IS named, because the model controls
590
+ # it and a smaller page really does fit.
591
+ return {
592
+ "error": "arXiv returned more than %d bytes for this "
593
+ "query, so nothing was read from it. Retry "
594
+ "with a smaller max_results."
595
+ % MAX_FEED_BYTES
596
+ }
597
+ except DefusedXmlException as e:
598
+ # NOT folded into the tuple below, and that is the point.
599
+ # A malformed body is transient and worth three attempts;
600
+ # a feed that DECLARES an entity is not -- it refuses the
601
+ # same way every time, so retrying spends two more round
602
+ # trips to reach the same place and then reports "failed
603
+ # after 3 attempts", which describes a network problem
604
+ # that is not there.
605
+ #
606
+ # Returned rather than raised, for the reason the retry
607
+ # exhaustion below is returned: a search tool that cannot
608
+ # answer is a result the model can work around. These do
609
+ # NOT subclass ParseError -- they are ValueError -- so
610
+ # without this clause they would propagate out of run().
611
+ logger.warning(
612
+ "arxiv_search refused a hostile feed for %r: %s",
613
+ parsed.keywords, e)
614
+ # THE FEED'S OWN TEXT STOPS HERE (batch 116's review).
615
+ # `e`'s repr carries the entity's `system_id` verbatim from
616
+ # the feed, so interpolating `{e}` would put a string the
617
+ # ATTACKER chose into the model's context -- the one place
618
+ # §32's untrusted-content paragraph exists to bound. The
619
+ # log above keeps the whole exception, because it is
620
+ # operator-facing and the formatter redaction sink guards
621
+ # app.log; the model gets the category and, where the
622
+ # exception carries one, the entity NAME, which XML's Name
623
+ # production cannot make into a sentence.
624
+ #
625
+ # getattr, not `e.name`: ExternalReferenceForbidden has no
626
+ # `.name`, and an AttributeError raised while building an
627
+ # error message would replace a handled refusal with a
628
+ # crash on the hostile path.
629
+ declared = getattr(e, "name", None)
630
+ named = (" -- it declared the entity %r"
631
+ % str(declared)[:40]) if declared else ""
632
+ return {
633
+ "error": "arXiv returned a feed this parser refuses to "
634
+ "expand, so nothing was read from it: %s%s"
635
+ % (type(e).__name__, named)
636
+ }
637
+ except (httpx.HTTPError, ET.ParseError) as e:
638
+ last_exc = e
639
+ logger.warning(
640
+ "arxiv_search attempt %s/%s failed for %r: %s",
641
+ attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
642
+ else:
643
+ # RETURNED, not raised. A search tool that cannot reach its
644
+ # provider is a result the model can work around -- try
645
+ # web_search, or say the claim could not be grounded. Raising
646
+ # made one transient network failure, in one pass, fail an
647
+ # entire ten-pass research run; fetch_url has always returned
648
+ # an error dict here and arxiv_search was the outlier.
649
+ #
650
+ # dispatch() now contains a raise from any tool as well, so
651
+ # this is the message rather than the mechanism: the model
652
+ # gets something specific instead of a generic wrapper.
653
+ return {
654
+ "error": f"arXiv search failed after {MAX_RETRIES + 1} "
655
+ f"attempts: {last_exc}"
656
+ }
657
+
658
+ if not results:
659
+ if parsed.start:
660
+ # Distinguishable from "this query has nothing", which is what
661
+ # the bare message would have said about a page past the end.
662
+ return {
663
+ "results": [], "result_count": 0, "start": parsed.start,
664
+ "message": (
665
+ f"No papers at start={parsed.start}. The search has "
666
+ f"{total} in total." if total is not None else
667
+ f"No papers at start={parsed.start}, which is past the "
668
+ f"end of this search."
669
+ ),
670
+ }
671
+ return {"results": [], "result_count": 0, "message": "No papers found."}
672
+
673
+ result = {"results": results, "result_count": len(results),
674
+ "start": parsed.start}
675
+
676
+ # PG4: say how to get the rest, not merely that there is a rest.
677
+ next_start = parsed.start + len(results)
678
+ if total is not None:
679
+ result["total_results"] = total
680
+ if next_start < total:
681
+ result["message"] = (
682
+ f"Returned papers {parsed.start + 1}-{next_start} of {total}. "
683
+ f"Use start={next_start} to see more."
684
+ )
685
+ return result