@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,371 +1,639 @@
1
- from __future__ import annotations
2
-
3
- import logging
4
-
5
- import httpx
6
- from pydantic import BaseModel, Field
7
-
8
- from safety.policy_enforcement import is_url_permitted, redact_output_text
9
- from tools.builtin._net_common import MAX_URL_CHARS, TTLCache
10
-
11
- logger = logging.getLogger(__name__)
12
-
13
- _TOOL_DESCRIPTION = (
14
- "Retrieve the contents of a web page using a URL. Long pages are "
15
- "returned one page at a time -- when a result says it was truncated, "
16
- "its message names the offset to pass for the next page."
17
- )
18
- MAX_CONTENT_CHARS = 5000
19
-
20
- # ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
21
- # before the connection is dropped, as distinct from how many characters
22
- # are returned. Both are needed and they bound different things:
23
- # MAX_CONTENT_CHARS is what the model reads, this is what the process
24
- # holds.
25
- #
26
- # Generous next to 5000 characters -- a multi-byte encoding can spend
27
- # four bytes on one of them -- because the point is not to be tight. It
28
- # is that `response.text` had no ceiling AT ALL: httpx.get buffers the
29
- # entire body before the slice happens, and the body is chosen by
30
- # whatever the URL points at. A Content-Length check is not the fix
31
- # either; a hostile server can omit it or lie about it, so the only
32
- # thing that actually bounds this is refusing to read past a cap.
33
- MAX_CONTENT_BYTES = 65_536
34
- REQUEST_TIMEOUT_S = 8.0
35
-
36
- # Redirects are followed BY HAND (see run()), so this is the loop bound,
37
- # not httpx's. httpx's own default is 20; this is lower because a
38
- # research fetch that needs more than five hops is not a fetch worth
39
- # making, and every hop costs a policy check and a DNS resolution.
40
- MAX_REDIRECTS = 5
41
-
42
- # ROADMAP_v3 §51 (PG5). How long a fetched body stays available to be paged
43
- # through. The same 300 s `arxiv.py` and `web_search.py` each use, and for a
44
- # weaker reason than theirs: they cache to avoid repeating a query, this
45
- # caches so that page 2 IS page 2.
46
- #
47
- # MEASURED (batch 105), which is why this is a cache and not a re-fetch.
48
- # Five real URLs fetched twice 45 seconds apart -- the interval a model
49
- # actually pages over, not the milliseconds an eager probe measures:
50
- # four were byte-identical and `news.ycombinator.com` was not. Its first
51
- # difference landed at byte 2,019, INSIDE page 1, and the text at
52
- # offset 5000..10000 differed between the two fetches. So a model that read
53
- # page 1 and asked for page 2 would have been handed two halves of two
54
- # different documents, spliced at an arbitrary point, with nothing in the
55
- # result able to say so. A news front page is exactly what a research run
56
- # reads.
57
- #
58
- # The second measured reason is cost. Paging an 88 KB document by re-fetch
59
- # is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
60
- # 5.3 MB for `bbc.com/news`. The cache holds at most the byte cap per URL.
61
- CACHE_TTL_S = 300
62
-
63
- # Keyed by URL -- BOTH the requested spelling and the one that answered,
64
- # because a redirect means the model may page with either. Holds
65
- # (answering url, decoded and REDACTED body, whether the byte cap stopped
66
- # the read).
67
- #
68
- # Redacted, since §52 (RB1). Two things follow and both are wanted: no page
69
- # cut from this body can split a credential, and the process no longer holds
70
- # unredacted remote content for the life of the entry -- before §51 the body
71
- # was a local that died with the call, and holding it extended the lifetime
72
- # of text nobody here wrote from one function call to 300 seconds.
73
- #
74
- # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6).
75
- # Measured (batch 106): 100 fetches held 6.3 MB and nothing was ever dropped,
76
- # because `TTLCache` expired on read alone and nothing read those keys again.
77
- _body_cache = TTLCache(CACHE_TTL_S)
78
-
79
-
80
- class FetchURLParams(BaseModel):
81
- # `MAX_URL_CHARS`, not 400 spelled again: batch 156 derived the
82
- # bound the two SEARCH tools use from this one, because a URL
83
- # they offer past it is a route this gate refuses. Two files
84
- # holding one number is the state a pair is in just before it
85
- # stops being a pair, and the relation is what matters here.
86
- url: str = Field(..., description="URL to be fetched", min_length=1, max_length=MAX_URL_CHARS)
87
- offset: int = Field(
88
- 0,
89
- ge=0,
90
- description=(
91
- "Number of characters to skip before reading (0-based). Use the "
92
- "offset named in a truncated result's message to read the next "
93
- "page of a long document."
94
- ),
95
- )
96
-
97
-
98
- TOOL_SCHEMA = {
99
- "name": "fetch_url",
100
- "description": _TOOL_DESCRIPTION,
101
- "input_schema": FetchURLParams.model_json_schema(),
102
- }
103
-
104
-
105
- def _bounded_body(response) -> tuple:
106
- """(decoded text, whether the server still had more to send).
107
-
108
- Stops after MAX_CONTENT_BYTES rather than reading to the end. The
109
- extra chunk beyond the cap is what makes `more` honest: without it,
110
- a body that happens to end exactly at the cap is indistinguishable
111
- from one that was cut off.
112
-
113
- NOT THE PLACE TO REDACT, though it is the first place the body exists
114
- (§52). This reads BYTES and the redactor works on decoded strings: a
115
- credential spanning the boundary between two `iter_bytes` chunks would
116
- be as invisible here as one spanning two pages is downstream, which is
117
- the very failure §52 exists to close. Redaction runs once in `run()`,
118
- on the whole decoded body, above both the cache and the slice.
119
- """
120
- chunks, total, more = [], 0, False
121
- for chunk in response.iter_bytes():
122
- chunks.append(chunk)
123
- total += len(chunk)
124
- if total > MAX_CONTENT_BYTES:
125
- more = True
126
- break
127
- raw = b"".join(chunks)[:MAX_CONTENT_BYTES]
128
- encoding = getattr(response, "encoding", None) or "utf-8"
129
- try:
130
- return raw.decode(encoding, errors="replace"), more
131
- except LookupError:
132
- # A server may name an encoding this build of Python does not
133
- # have. That is not a reason to lose the page.
134
- return raw.decode("utf-8", errors="replace"), more
135
-
136
-
137
- def _page(url: str, body: str, more: bool, offset: int) -> dict:
138
- """One page of a held body, plus how to reach the next (§51, PG1/PG4).
139
-
140
- `read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
141
- a message naming the offset to use next. One paging idiom across the
142
- tools that have one, rather than a second one to learn here.
143
-
144
- THE THREE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS.
145
- `truncated` is still "there is more than you were given", and at
146
- offset 0 it computes to precisely what it did before §51 -- the byte
147
- cap bit, or the body is longer than one page. `source_corpus` and the
148
- grounding passes read `url` and `content`; the register's constraint is
149
- that added keys are safe and changed ones are not.
150
-
151
- EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because
152
- `body` arrives redacted. That is the only position space this tool
153
- speaks, and it is self-consistent: a caller only ever passes back an
154
- offset this function named. Two consequences, stated rather than left to
155
- be discovered -- a document containing credentials reports a smaller
156
- `chars_available` than its source has, and the same URL reports
157
- different totals with `REDACT_TOOL_OUTPUTS` on and off.
158
- """
159
- total = len(body)
160
- chunk = body[offset:offset + MAX_CONTENT_CHARS]
161
- next_offset = offset + len(chunk)
162
- beyond = next_offset < total
163
-
164
- result = {
165
- "url": url,
166
- "content": chunk,
167
- "truncated": beyond or more,
168
- "offset": offset,
169
- "chars_available": total,
170
- }
171
-
172
- if offset >= total and total:
173
- # Past the end is not an error, for `read`'s reason: a model that
174
- # walked one page too far should be told where the end was, not
175
- # handed a failure it has to reason about.
176
- result["message"] = (
177
- f"Offset {offset} is past the end of this document, which has "
178
- f"{total} characters. Nothing was returned."
179
- )
180
- elif beyond:
181
- result["message"] = (
182
- f"Returned characters {offset}-{next_offset} of {total}. "
183
- f"Use offset={next_offset} to read further."
184
- )
185
- elif more:
186
- # PG6, and the whole of it. The cap is NOT raised, so the honest
187
- # thing is to say the remainder was never retrieved rather than
188
- # name an offset that would return nothing. Measured (batch 105):
189
- # an ordinary Python docs page is 88 KB, so roughly a quarter of it
190
- # is past this bound, and `bbc.com/news` is 382 KB.
191
- result["message"] = (
192
- f"Returned characters {offset}-{next_offset}, the end of the "
193
- f"readable window. This document was cut at the "
194
- f"{MAX_CONTENT_BYTES}-byte read limit, so the rest of it was "
195
- f"never retrieved and no further page exists."
196
- )
197
- elif offset:
198
- result["message"] = (
199
- f"Returned characters {offset}-{next_offset} of {total}. "
200
- f"This is the last page."
201
- )
202
-
203
- return result
204
-
205
-
206
- def _redacted(url: str, body: str) -> str:
207
- """The body with credentials replaced, said once (§52, RB1 and RB2).
208
-
209
- ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
210
- `check_output_policy`, so this text was always redacted -- but it was
211
- redacted AFTER the page had been cut out of it, and a pattern only
212
- matches what it can see whole. Measured (batch 106), a `ghp_` token
213
- straddling character 5,000 survived both pages untouched, was rejoined by
214
- the corpus's PG7 append, and landed in `sources/<sha256>.txt` and in the
215
- window sent to the embedding provider. Every one of the ten patterns is
216
- defeated by some cut; the fixed-length tokens and both credential shapes
217
- are defeated by EVERY interior cut, because a prefix of a fixed-length
218
- token is not that token.
219
-
220
- This repo had already locked the rule twice. `param_digest` "redacts
221
- BEFORE truncating" because "truncating first cuts a credential below the
222
- 20 characters its pattern needs", and `tui/app.py`'s diff block runs on
223
- whole texts because a secret "would be just as readable split across two
224
- wrapped rows as on one". §51's paging is a third cut, and the first to
225
- get the order wrong.
226
-
227
- RB2, the warning. Redacting here means `check_output_policy` finds
228
- nothing left to alter, so #49's "nothing this function does is silent"
229
- would stop covering this tool without anybody noticing. One line at the
230
- fetch replaces up to fourteen at the pages, and it keeps that function's
231
- own rule: the match is never echoed, because saying what was found would
232
- put it in app.log and every sink a WARNING reaches. The URL is named --
233
- it is what makes the line actionable, and `logging_setup`'s formatter
234
- redaction is the sink-side guard for a URL carrying userinfo, exactly as
235
- it is for the `fetch_url failed for %s` line below.
236
- """
237
- clean = redact_output_text(body)
238
- if clean != body:
239
- logger.warning(
240
- "Output policy altered fetch_url result: credentials replaced "
241
- "in the body of %s before it was paged.", url)
242
- return clean
243
-
244
-
245
- def _cached(url: str):
246
- """The held body for `url`, or None. Policy is the CALLER's job."""
247
- return _body_cache.get(url)
248
-
249
-
250
- def _hold(requested: str, answering: str, body: str, more: bool) -> None:
251
- _body_cache.set(requested, (answering, body, more))
252
- if answering != requested:
253
- _body_cache.set(answering, (answering, body, more))
254
-
255
-
256
- def run(params: dict) -> dict:
257
- """Fetch one URL, checking policy BEFORE every request it issues.
258
-
259
- Redirects are followed manually (#53, #54). The version this replaced
260
- passed `follow_redirects=True` and checked the blocklist once, against
261
- the argument -- so one 302 walked past the control, and the blocked
262
- host had already served its body by the time anything could object.
263
- §25 R5's dispatch-wide check has the same blind spot for the same
264
- reason: it scans the argument, and the argument was clean.
265
-
266
- Checking each hop BEFORE issuing it, rather than checking the history
267
- afterwards, is the whole point. For a malware host the difference is
268
- whether it was contacted; for `169.254.169.254` the request IS the
269
- disclosure, so a post-hoc check does not fix #54 at all.
270
-
271
- The returned `url` is the one that ANSWERED, not the one that was
272
- asked for. That half is a correctness fix as much as a security one:
273
- the grounding passes attribute fetched text to the URL in this field,
274
- and output_writer builds each run's sources/ directory from it -- so
275
- a redirect chain used to silently rewrite what a claim was grounded
276
- in, with nothing in the result, the transcript or the MessageLog
277
- recording where the content actually came from.
278
- """
279
- parsed = FetchURLParams(**params)
280
-
281
- # §51 (PG5). A page after the first is served from the body the first
282
- # one held, so that every page of a document comes from ONE fetch and
283
- # page 2 continues page 1 rather than continuing whatever the server
284
- # is serving now.
285
- #
286
- # THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
287
- # consulted before the blocklist is a blocklist with a hole in it: the
288
- # body was permitted when it was fetched, and `is_url_permitted` is not
289
- # a constant -- it resolves names, and a run's own configuration can
290
- # change under it. Skipping it here would rebuild #54 one layer up,
291
- # where the control is bypassed by a second call rather than a
292
- # redirect.
293
- #
294
- # offset 0 never reads the cache: a fresh read of a page is a fresh
295
- # read, which is what every caller that passes no offset already
296
- # expects and what makes the held copy current for the pages after it.
297
- if parsed.offset:
298
- refusal = is_url_permitted(parsed.url)
299
- if refusal:
300
- return {"error": refusal}
301
- held = _cached(parsed.url)
302
- if held is not None:
303
- answering, body, more = held
304
- return _page(answering, body, more, parsed.offset)
305
-
306
- url = parsed.url
307
- for _ in range(MAX_REDIRECTS):
308
- refusal = is_url_permitted(url)
309
- if refusal:
310
- return {"error": refusal}
311
-
312
- try:
313
- # §31 (H7): streamed, so the headers are available before
314
- # any body is pulled. Two consequences, both wanted -- a
315
- # redirect now costs no body at all (the old path downloaded
316
- # every hop's), and the body that IS wanted stops at
317
- # MAX_CONTENT_BYTES instead of being buffered whole and then
318
- # sliced. read_run's guard is the model: bound before, not
319
- # after.
320
- with httpx.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
321
- follow_redirects=False) as response:
322
- if response.is_redirect:
323
- # .next_request is None on a redirect with no usable
324
- # Location; treat that as the end of the chain rather
325
- # than following nothing.
326
- if response.next_request is None:
327
- break
328
- url = str(response.next_request.url)
329
- continue
330
- response.raise_for_status()
331
- body, more = _bounded_body(response)
332
- except httpx.HTTPError as e:
333
- # Interpolated, NOT extra={}: the default formatter renders only
334
- # %(message)s, so every field passed via extra was silently
335
- # dropped -- fifteen consecutive "fetch_url failed" lines with
336
- # no URL and no reason, which is what made a run of ordinary
337
- # 404s indistinguishable from a broken tool.
338
- logger.warning("fetch_url failed for %s: %s", url, e)
339
- return {"error": f"Could not fetch URL: {e}"}
340
-
341
- # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position
342
- # is the fix: the redactor sees the whole document exactly once,
343
- # while it is still one string. Everything below this line -- the
344
- # held copy, page 1, and every page a later offset cuts out of it --
345
- # is a slice of already-clean text.
346
- body = _redacted(url, body)
347
-
348
- # Held before the page is cut, so a later offset reads the body
349
- # this fetch actually retrieved (PG5). Keyed by both spellings,
350
- # because `url` here is the one that ANSWERED and the model may
351
- # page with the one it asked for.
352
- _hold(parsed.url, url, body, more)
353
-
354
- # `truncated` is still true when EITHER bound bit: more characters
355
- # were decoded than are returned, or the byte cap stopped the read
356
- # with the server still sending. A silently short page is how a
357
- # grounding pass concludes a source does not say something.
358
- #
359
- # `or more` CANNOT FIRE with the shipped constants, and that is
360
- # deliberate rather than an oversight: 65,536 bytes is at least
361
- # 16,384 characters even at four bytes each, so the first term is
362
- # always already true when the byte cap bites. It is here for the
363
- # day someone raises MAX_CONTENT_CHARS or lowers
364
- # MAX_CONTENT_BYTES, which is exactly when a silent under-report
365
- # would appear. A mutation deleting it survived until a test moved
366
- # the constants to reach it. §51 moved the expression into
367
- # `_page` as `beyond or more`, where `beyond` is the same question
368
- # asked of an arbitrary offset instead of only the first page.
369
- return _page(url, body, more, parsed.offset)
370
-
371
- return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from dataclasses import dataclass, field
5
+ from typing import Optional
6
+
7
+ import httpx
8
+ from pydantic import BaseModel, Field
9
+
10
+ import config
11
+ from safety.policy_enforcement import is_url_permitted, redact_output_text
12
+ from tools.builtin import _net_common, _page_text
13
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ _TOOL_DESCRIPTION = (
18
+ "Retrieve the readable text of a web page or PDF by URL. Pages are "
19
+ "returned as text without the site's navigation; a link in it is marked "
20
+ "like [L3], and the result lists the URL for each marker on that page. "
21
+ "Long documents are returned one page at a time -- when a result says "
22
+ "it was truncated, its message names the offset to pass for the next "
23
+ "page."
24
+ )
25
+ MAX_CONTENT_CHARS = 5000
26
+
27
+ # ROADMAP_v2 §31 (H7), #55. How many BYTES of a TEXT body -- JSON, plain
28
+ # text, XML, source -- may be pulled off the wire before the connection is
29
+ # dropped, as distinct from how many characters are returned. Both are
30
+ # needed and they bound different things: MAX_CONTENT_CHARS is what the
31
+ # model reads, this is what the process holds.
32
+ #
33
+ # Generous next to 5000 characters -- a multi-byte encoding can spend
34
+ # four bytes on one of them -- because the point is not to be tight. It
35
+ # is that `response.text` had no ceiling AT ALL: httpx.get buffers the
36
+ # entire body before the slice happens, and the body is chosen by
37
+ # whatever the URL points at. A Content-Length check is not the fix
38
+ # either; a hostile server can omit it or lie about it, so the only
39
+ # thing that actually bounds this is refusing to read past a cap.
40
+ #
41
+ # HTML AND PDF HAVE THEIR OWN BOUNDS SINCE BATCH 240 (`fetch_max_html_mb`,
42
+ # `fetch_max_pdf_mb`), the owner's amendment of PG6. Measured, an HTML page
43
+ # is 75-99% markup, so this cap stopped Wikipedia before its article's
44
+ # first sentence; what is held now is the EXTRACTED text, bounded by
45
+ # HELD_CHARS, so memory no longer follows the markup.
46
+ MAX_CONTENT_BYTES = 65_536
47
+ REQUEST_TIMEOUT_S = 8.0
48
+
49
+ # Batch 240, the owner's choice: how much extracted text one fetch holds to
50
+ # be paged through. A Wikipedia article is ~90,000 characters and the Llama
51
+ # 2 report ~260,000. An offset past it re-fetches and serves the window it
52
+ # asks for, so a longer document is still readable to its end; it is not
53
+ # all held at once.
54
+ HELD_CHARS = 200_000
55
+
56
+ # How many link markers one page of text lists URLs for. Past it the page
57
+ # says how many more it cited, so the cut is visible.
58
+ LINKS_PER_PAGE = 30
59
+
60
+ # Redirects are followed BY HAND (see run()), so this is the loop bound,
61
+ # not httpx's. httpx's own default is 20; this is lower because a
62
+ # research fetch that needs more than five hops is not a fetch worth
63
+ # making, and every hop costs a policy check and a DNS resolution.
64
+ MAX_REDIRECTS = 5
65
+
66
+ # What fetch_url asks for. A server choosing between representations then
67
+ # prefers the ones this tool can read.
68
+ _ACCEPT = ("text/html,application/xhtml+xml,application/pdf;q=0.9,"
69
+ "text/plain;q=0.8,*/*;q=0.5")
70
+
71
+ # ROADMAP_v3 §51 (PG5). How long a fetched document stays available to be
72
+ # paged through. The same 300 s `arxiv.py` and `web_search.py` each use, and
73
+ # for a weaker reason than theirs: they cache to avoid repeating a query,
74
+ # this caches so that page 2 IS page 2.
75
+ #
76
+ # MEASURED (batch 105), which is why this is a cache and not a re-fetch.
77
+ # Five real URLs fetched twice 45 seconds apart -- the interval a model
78
+ # actually pages over, not the milliseconds an eager probe measures:
79
+ # four were byte-identical and `news.ycombinator.com` was not. Its first
80
+ # difference landed at byte 2,019, INSIDE page 1, and the text at
81
+ # offset 5000..10000 differed between the two fetches. So a model that read
82
+ # page 1 and asked for page 2 would have been handed two halves of two
83
+ # different documents, spliced at an arbitrary point, with nothing in the
84
+ # result able to say so. A news front page is exactly what a research run
85
+ # reads.
86
+ #
87
+ # Since batch 240 a page PAST the held window does re-fetch, because the
88
+ # owner asked for the whole document to be readable. The splice that
89
+ # measurement found is then named rather than hidden: the held entry
90
+ # carries a digest of the whole document's text, and a re-fetch whose
91
+ # digest differs says the document changed.
92
+ #
93
+ # The second measured reason is cost. Paging an 88 KB document by re-fetch
94
+ # is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
95
+ # 5.3 MB for `bbc.com/news`.
96
+ CACHE_TTL_S = 300
97
+
98
+ # Keyed by URL -- BOTH the requested spelling and the one that answered,
99
+ # because a redirect means the model may page with either. Holds a `_Held`:
100
+ # a window of the document's REDACTED text and what is known about the rest.
101
+ #
102
+ # Redacted, since §52 (RB1). Two things follow and both are wanted: no page
103
+ # cut from this text can split a credential, and the process no longer holds
104
+ # unredacted remote content for the life of the entry.
105
+ #
106
+ # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6), and
107
+ # each entry by HELD_CHARS.
108
+ _body_cache = TTLCache(CACHE_TTL_S)
109
+
110
+
111
+ class FetchURLParams(BaseModel):
112
+ # `MAX_URL_CHARS`, not 400 spelled again: batch 156 derived the
113
+ # bound the two SEARCH tools use from this one, because a URL
114
+ # they offer past it is a route this gate refuses. Two files
115
+ # holding one number is the state a pair is in just before it
116
+ # stops being a pair, and the relation is what matters here.
117
+ url: str = Field(..., description="URL to be fetched", min_length=1, max_length=MAX_URL_CHARS)
118
+ offset: int = Field(
119
+ 0,
120
+ ge=0,
121
+ description=(
122
+ "Number of characters to skip before reading (0-based). Use the "
123
+ "offset named in a truncated result's message to read the next "
124
+ "page of a long document."
125
+ ),
126
+ )
127
+
128
+
129
+ TOOL_SCHEMA = {
130
+ "name": "fetch_url",
131
+ "description": _TOOL_DESCRIPTION,
132
+ "input_schema": FetchURLParams.model_json_schema(),
133
+ }
134
+
135
+
136
+ @dataclass
137
+ class _Held:
138
+ """A window of one document's text, and what is known about the rest.
139
+
140
+ `start` and `total` are positions in the document's REDACTED text, the
141
+ only position space this tool speaks (§52, RB3). `more` says the byte
142
+ bound stopped the read with the server still sending, so text past
143
+ `total` was never retrieved at all.
144
+ """
145
+
146
+ url: str # the URL that answered
147
+ text: str # [start, start + len(text)) of the document
148
+ start: int
149
+ total: int
150
+ more: bool
151
+ kind: str = "text" # "html", "pdf" or "text"
152
+ read_limit: int = MAX_CONTENT_BYTES
153
+ digest: str = ""
154
+ links: dict = field(default_factory=dict)
155
+ title: Optional[str] = None
156
+ pages: Optional[int] = None
157
+ no_text: bool = False
158
+
159
+ def covers(self, offset: int) -> bool:
160
+ """Whether a page at `offset` can be served without a fetch."""
161
+ if offset >= self.total:
162
+ return True # past the end: the answer is known
163
+ return self.start <= offset < self.start + len(self.text)
164
+
165
+
166
+ class _Refused(Exception):
167
+ """A body this tool will not read, with the sentence that says why."""
168
+
169
+
170
+ # ---- Reading the body ------------------------------------------------------
171
+
172
+ def _header(response, name: str) -> str:
173
+ headers = getattr(response, "headers", None) or {}
174
+ for key, value in headers.items():
175
+ if key.lower() == name:
176
+ return str(value)
177
+ return ""
178
+
179
+
180
+ def _content_type(response) -> str:
181
+ return _header(response, "content-type").split(";", 1)[0].strip().lower()
182
+
183
+
184
+ def _kind_from_type(content_type: str) -> Optional[str]:
185
+ """What a declared Content-Type says the body is, or None to sniff."""
186
+ if not content_type or content_type == "application/octet-stream":
187
+ return None
188
+ if content_type in ("text/html", "application/xhtml+xml"):
189
+ return "html"
190
+ if content_type == "application/pdf":
191
+ return "pdf"
192
+ if content_type.startswith("text/") or content_type.endswith(
193
+ ("+json", "+xml")) or content_type in (
194
+ "application/json", "application/xml", "application/javascript",
195
+ "application/x-javascript", "application/ld+json",
196
+ "application/x-yaml", "application/yaml", "application/toml",
197
+ "application/x-ndjson", "application/rss+xml",
198
+ "application/atom+xml"):
199
+ return "text"
200
+ return "binary"
201
+
202
+
203
+ def _sniff(head: bytes, declared: Optional[str]) -> str:
204
+ """The body's kind, from its first bytes where the type did not say.
205
+
206
+ A PDF announces itself whatever its header claims: servers send them as
207
+ `application/octet-stream`, and some as `text/plain`.
208
+ """
209
+ if head.lstrip()[:5] == b"%PDF-":
210
+ return "pdf"
211
+ if declared is not None:
212
+ return declared
213
+ sample = head[:1024].lower()
214
+ if b"<html" in sample or b"<!doctype html" in sample:
215
+ return "html"
216
+ if b"\x00" in sample:
217
+ return "binary"
218
+ return "text"
219
+
220
+
221
+ def _limit_for(kind: str) -> int:
222
+ if kind == "html":
223
+ return int(round(config.FETCH_MAX_HTML_MB * 1_000_000))
224
+ if kind == "pdf":
225
+ return int(round(config.FETCH_MAX_PDF_MB * 1_000_000))
226
+ return MAX_CONTENT_BYTES
227
+
228
+
229
+ def _too_large(url: str, size: int, limit: int) -> str:
230
+ return (f"{url} is a PDF of {size:,} bytes, larger than the "
231
+ f"{limit:,}-byte limit (fetch_max_pdf_mb in config.yaml), so it "
232
+ f"was not read.")
233
+
234
+
235
+ def _bounded_body(response, url: str) -> tuple:
236
+ """(kind, raw bytes, whether the server still had more to send).
237
+
238
+ Reads only as far as the body's kind allows: an HTML page to
239
+ `fetch_max_html_mb`, a PDF to `fetch_max_pdf_mb`, anything else that is
240
+ text to MAX_CONTENT_BYTES. The extra chunk beyond a cap is what makes
241
+ `more` honest: without it, a body that happens to end exactly at the
242
+ cap is indistinguishable from one that was cut off.
243
+
244
+ A PDF is the exception to "stop and say so": half a PDF has no index
245
+ and cannot be read, so one past its limit is refused whole -- before
246
+ any of it is read when its Content-Length says so. A binary body is
247
+ refused as soon as its first bytes show what it is.
248
+
249
+ NOT THE PLACE TO REDACT, though it is the first place the body exists
250
+ (§52). This reads BYTES and the redactor works on decoded strings: a
251
+ credential spanning the boundary between two `iter_bytes` chunks would
252
+ be as invisible here as one spanning two pages is downstream. Redaction
253
+ runs once, on the whole decoded text, above both the cache and the
254
+ slice.
255
+ """
256
+ declared = _kind_from_type(_content_type(response))
257
+ if declared == "pdf":
258
+ length = _header(response, "content-length")
259
+ if length.isdigit() and int(length) > _limit_for("pdf"):
260
+ raise _Refused(_too_large(url, int(length), _limit_for("pdf")))
261
+ if declared == "binary":
262
+ raise _Refused(_binary(url, _content_type(response)))
263
+ chunks, total, kind, limit = [], 0, None, MAX_CONTENT_BYTES
264
+ for chunk in response.iter_bytes():
265
+ chunks.append(chunk)
266
+ total += len(chunk)
267
+ if kind is None and (total >= 1024 or not chunk):
268
+ kind = _sniff(b"".join(chunks), declared)
269
+ if kind == "binary":
270
+ raise _Refused(_binary(url, _content_type(response)))
271
+ limit = _limit_for(kind)
272
+ if kind is not None and total > limit:
273
+ if kind == "pdf":
274
+ raise _Refused(_too_large(url, total, limit))
275
+ raw = b"".join(chunks)[:limit]
276
+ return kind, raw, True
277
+ raw = b"".join(chunks)
278
+ if kind is None:
279
+ # A body shorter than the sniff window is typed here, after the
280
+ # loop -- and must still meet its limit, which a short limit can
281
+ # make shorter than the window (the test that moves the constants).
282
+ kind = _sniff(raw, declared)
283
+ if kind == "binary":
284
+ raise _Refused(_binary(url, _content_type(response)))
285
+ limit = _limit_for(kind)
286
+ if len(raw) > limit:
287
+ if kind == "pdf":
288
+ raise _Refused(_too_large(url, len(raw), limit))
289
+ return kind, raw[:limit], True
290
+ return kind, raw, False
291
+
292
+
293
+ def _binary(url: str, content_type: str) -> str:
294
+ named = content_type or "a binary type"
295
+ return (f"{url} answered with {named}, which is not text, HTML or a PDF, "
296
+ f"so fetch_url did not read it. To keep the file itself, "
297
+ f"download it with a shell command.")
298
+
299
+
300
+ def _decode(raw: bytes, response) -> str:
301
+ encoding = getattr(response, "encoding", None) or "utf-8"
302
+ try:
303
+ return raw.decode(encoding, errors="replace")
304
+ except LookupError:
305
+ # A server may name an encoding this build of Python does not
306
+ # have. That is not a reason to lose the page.
307
+ return raw.decode("utf-8", errors="replace")
308
+
309
+
310
+ # ---- Turning a body into a document ----------------------------------------
311
+
312
+ def _keep_link(third: ThirdPartyText):
313
+ """The link filter `_page_text` applies: a URL this tool would refuse,
314
+ or one too long for it to accept, is not offered as a route (A58)."""
315
+ def keep(url: str) -> Optional[str]:
316
+ kept, withheld = third.url(url)
317
+ if kept is None or withheld:
318
+ return None
319
+ if is_url_permitted(kept, resolve=False):
320
+ return None
321
+ return kept
322
+ return keep
323
+
324
+
325
+ def _window_start(want: int) -> int:
326
+ """Where a held window begins for a page at `want`."""
327
+ return 0 if want < HELD_CHARS else want
328
+
329
+
330
+ def _held(url: str, kind: str, raw: bytes, more: bool, response,
331
+ want: int):
332
+ """The `_Held` for a fetched body, or an error dict for a PDF that could
333
+ not be read."""
334
+ start = _window_start(want)
335
+ if kind == "pdf":
336
+ return _held_pdf(url, raw, start)
337
+ text = _decode(raw, response)
338
+ links: dict = {}
339
+ title = None
340
+ if kind == "html":
341
+ third = ThirdPartyText("fetch_url")
342
+ document = _page_text.html_document(text, url, _keep_link(third))
343
+ text, links, title = document.text, document.links, document.title
344
+ if title is not None:
345
+ title = redact_output_text(title)
346
+ # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position is
347
+ # the fix: the redactor sees the whole document exactly once, while it
348
+ # is still one string. Everything below -- the held window, page 1, and
349
+ # every page a later offset cuts out of it -- is a slice of clean text.
350
+ text = _redacted(url, text)
351
+ return _Held(url=url, text=text[start:start + HELD_CHARS], start=start,
352
+ total=len(text), more=more, kind=kind,
353
+ read_limit=_limit_for(kind),
354
+ digest=_page_text.text_digest(text), links=links,
355
+ title=title)
356
+
357
+
358
+ def _held_pdf(url: str, raw: bytes, start: int):
359
+ """A PDF's window, read in the killable child (the owner's choice)."""
360
+ from tools import isolation
361
+ from tools.builtin import _pdf_text
362
+
363
+ result = isolation.run_isolated(
364
+ _pdf_text.pdf_window,
365
+ {"data": _pdf_text.encode(raw), "start": start, "limit": HELD_CHARS},
366
+ config.TOOL_COMPUTE_TIMEOUT_S, tool_name="fetch_url")
367
+ if "error" in result:
368
+ return {"error": f"Could not read the PDF at {url}: {result['error']}"}
369
+ if result.get("redacted"):
370
+ logger.warning(
371
+ "Output policy altered fetch_url result: credentials replaced "
372
+ "in the text of %s before it was paged.", url)
373
+ title = result.get("title")
374
+ return _Held(url=url, text=result.get("text", ""), start=start,
375
+ total=int(result.get("total", 0)), more=False, kind="pdf",
376
+ read_limit=_limit_for("pdf"),
377
+ digest=result.get("digest", ""),
378
+ title=redact_output_text(title) if title else None,
379
+ pages=result.get("pages"),
380
+ no_text=bool(result.get("no_text")))
381
+
382
+
383
+ # ---- One page of a held document -------------------------------------------
384
+
385
+ def _page(url: str, body: str, more: bool, offset: int) -> dict:
386
+ """One page of a whole held body, plus how to reach the next (§51).
387
+
388
+ The shape every caller before batch 240 used, kept for them: it is a
389
+ held document whose window is the whole body.
390
+ """
391
+ return _held_page(_Held(url=url, text=body, start=0, total=len(body),
392
+ more=more, read_limit=MAX_CONTENT_BYTES), offset)
393
+
394
+
395
+ def _held_page(held: _Held, offset: int, changed: bool = False) -> dict:
396
+ """One page of a held document, plus how to reach the next (§51, PG1/PG4).
397
+
398
+ `read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
399
+ a message naming the offset to use next. One paging idiom across the
400
+ tools that have one, rather than a second one to learn here.
401
+
402
+ THE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS. `truncated` is
403
+ still "there is more than you were given". `source_corpus` and the
404
+ grounding passes read `url` and `content`; the register's constraint is
405
+ that added keys are safe and changed ones are not. Batch 240 adds
406
+ `title` on the first page, `links` for the markers a page carries, and
407
+ `format`/`pages` for a PDF.
408
+
409
+ EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because the
410
+ text arrives redacted. That is the only position space this tool
411
+ speaks, and it is self-consistent: a caller only ever passes back an
412
+ offset this function named.
413
+ """
414
+ total = held.total
415
+ local = offset - held.start
416
+ end = _page_text.split_marker_end(held.text, local + MAX_CONTENT_CHARS)
417
+ chunk = held.text[local:end] if 0 <= local < len(held.text) else ""
418
+ next_offset = offset + len(chunk)
419
+ beyond = next_offset < total
420
+
421
+ result = {
422
+ "url": held.url,
423
+ "content": chunk,
424
+ "truncated": beyond or held.more,
425
+ "offset": offset,
426
+ "chars_available": total,
427
+ }
428
+ if offset == 0 and held.title:
429
+ result["title"] = held.title
430
+ if held.kind == "pdf":
431
+ result["format"] = "pdf"
432
+ result["pages"] = held.pages
433
+ cited = _page_text.markers_in(chunk)
434
+ if cited:
435
+ listed = [n for n in cited if n in held.links][:LINKS_PER_PAGE]
436
+ result["links"] = {f"L{n}": held.links[n] for n in listed}
437
+ omitted = len([n for n in cited if n in held.links]) - len(listed)
438
+ if omitted:
439
+ result["links_omitted"] = omitted
440
+
441
+ notes = []
442
+ if changed:
443
+ notes.append(
444
+ "The document changed since an earlier page of it was read; "
445
+ "this page comes from the version fetched now.")
446
+ if held.no_text:
447
+ notes.append(
448
+ f"This PDF's {held.pages} pages have no text layer -- it is "
449
+ f"probably scanned images -- so there is nothing to read.")
450
+ if offset >= total and total:
451
+ # Past the end is not an error, for `read`'s reason: a model that
452
+ # walked one page too far should be told where the end was, not
453
+ # handed a failure it has to reason about.
454
+ notes.append(
455
+ f"Offset {offset} is past the end of this document, which has "
456
+ f"{total} characters. Nothing was returned.")
457
+ elif beyond:
458
+ notes.append(
459
+ f"Returned characters {offset}-{next_offset} of {total}. "
460
+ f"Use offset={next_offset} to read further.")
461
+ elif held.more:
462
+ # PG6's honesty, kept: the remainder was never retrieved, so name
463
+ # no offset that would return nothing.
464
+ notes.append(
465
+ f"Returned characters {offset}-{next_offset}, the end of the "
466
+ f"readable window. This document was cut at the "
467
+ f"{held.read_limit}-byte read limit, so the rest of it was "
468
+ f"never retrieved and no further page exists.")
469
+ elif offset:
470
+ notes.append(
471
+ f"Returned characters {offset}-{next_offset} of {total}. "
472
+ f"This is the last page.")
473
+ if notes:
474
+ result["message"] = " ".join(notes)
475
+ return result
476
+
477
+
478
+ def _redacted(url: str, body: str) -> str:
479
+ """The text with credentials replaced, said once (§52, RB1 and RB2).
480
+
481
+ ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
482
+ `check_output_policy`, so this text was always redacted -- but it was
483
+ redacted AFTER the page had been cut out of it, and a pattern only
484
+ matches what it can see whole. Measured (batch 106), a `ghp_` token
485
+ straddling character 5,000 survived both pages untouched, was rejoined by
486
+ the corpus's PG7 append, and landed in `sources/<sha256>.txt` and in the
487
+ window sent to the embedding provider. Every one of the ten patterns is
488
+ defeated by some cut; the fixed-length tokens and both credential shapes
489
+ are defeated by EVERY interior cut, because a prefix of a fixed-length
490
+ token is not that token.
491
+
492
+ This repo had already locked the rule twice. `param_digest` "redacts
493
+ BEFORE truncating" because "truncating first cuts a credential below the
494
+ 20 characters its pattern needs", and `tui/app.py`'s diff block runs on
495
+ whole texts because a secret "would be just as readable split across two
496
+ wrapped rows as on one". §51's paging is a third cut, and the first to
497
+ get the order wrong.
498
+
499
+ RB2, the warning. Redacting here means `check_output_policy` finds
500
+ nothing left to alter, so #49's "nothing this function does is silent"
501
+ would stop covering this tool without anybody noticing. One line at the
502
+ fetch replaces up to fourteen at the pages, and it keeps that function's
503
+ own rule: the match is never echoed, because saying what was found would
504
+ put it in app.log and every sink a WARNING reaches. The URL is named --
505
+ it is what makes the line actionable, and `logging_setup`'s formatter
506
+ redaction is the sink-side guard for a URL carrying userinfo, exactly as
507
+ it is for the `fetch_url failed for %s` line below.
508
+ """
509
+ clean = redact_output_text(body)
510
+ if clean != body:
511
+ logger.warning(
512
+ "Output policy altered fetch_url result: credentials replaced "
513
+ "in the body of %s before it was paged.", url)
514
+ return clean
515
+
516
+
517
+ def _cached(url: str):
518
+ """The held document for `url`, or None. Policy is the CALLER's job."""
519
+ return _body_cache.get(url)
520
+
521
+
522
+ def _hold(requested: str, held: _Held) -> None:
523
+ _body_cache.set(requested, held)
524
+ if held.url != requested:
525
+ _body_cache.set(held.url, held)
526
+
527
+
528
+ def run(params: dict) -> dict:
529
+ """Fetch one URL, checking policy BEFORE every request it issues.
530
+
531
+ Redirects are followed manually (#53, #54). The version this replaced
532
+ passed `follow_redirects=True` and checked the blocklist once, against
533
+ the argument -- so one 302 walked past the control, and the blocked
534
+ host had already served its body by the time anything could object.
535
+ §25 R5's dispatch-wide check has the same blind spot for the same
536
+ reason: it scans the argument, and the argument was clean.
537
+
538
+ Checking each hop BEFORE issuing it, rather than checking the history
539
+ afterwards, is the whole point. For a malware host the difference is
540
+ whether it was contacted; for `169.254.169.254` the request IS the
541
+ disclosure, so a post-hoc check does not fix #54 at all.
542
+
543
+ The returned `url` is the one that ANSWERED, not the one that was
544
+ asked for. That half is a correctness fix as much as a security one:
545
+ the grounding passes attribute fetched text to the URL in this field,
546
+ and output_writer builds each run's sources/ directory from it -- so
547
+ a redirect chain used to silently rewrite what a claim was grounded
548
+ in, with nothing in the result, the transcript or the MessageLog
549
+ recording where the content actually came from.
550
+ """
551
+ parsed = FetchURLParams(**params)
552
+ want = parsed.offset
553
+
554
+ # §51 (PG5). A page after the first is served from the document the
555
+ # first one held, so that every page of it comes from ONE fetch and
556
+ # page 2 continues page 1 rather than continuing whatever the server is
557
+ # serving now. A page past the held window (batch 240) is the one
558
+ # exception, and the version check below is what keeps it honest.
559
+ #
560
+ # THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
561
+ # consulted before the blocklist is a blocklist with a hole in it: the
562
+ # body was permitted when it was fetched, and `is_url_permitted` is not
563
+ # a constant -- it resolves names, and a run's own configuration can
564
+ # change under it. Skipping it here would rebuild #54 one layer up,
565
+ # where the control is bypassed by a second call rather than a
566
+ # redirect.
567
+ #
568
+ # offset 0 never reads the cache: a fresh read of a page is a fresh
569
+ # read, which is what every caller that passes no offset already
570
+ # expects and what makes the held copy current for the pages after it.
571
+ earlier: Optional[_Held] = None
572
+ if want:
573
+ refusal = is_url_permitted(parsed.url)
574
+ if refusal:
575
+ return {"error": refusal}
576
+ earlier = _cached(parsed.url)
577
+ if earlier is not None and earlier.covers(want):
578
+ return _held_page(earlier, want)
579
+
580
+ url = parsed.url
581
+ for _ in range(MAX_REDIRECTS):
582
+ refusal = is_url_permitted(url)
583
+ if refusal:
584
+ return {"error": refusal}
585
+
586
+ try:
587
+ # §31 (H7): streamed, so the headers are available before
588
+ # any body is pulled. Two consequences, both wanted -- a
589
+ # redirect now costs no body at all (the old path downloaded
590
+ # every hop's), and the body that IS wanted stops at its
591
+ # bound instead of being buffered whole and then sliced.
592
+ # ROADMAP_v3 §53: `_net_common.stream` is `httpx.stream`,
593
+ # admitted and counted against the tool meter, once per hop.
594
+ # Batch 240: a descriptive User-Agent, because Wikimedia
595
+ # answers httpx's default with 403 (measured).
596
+ with _net_common.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
597
+ follow_redirects=False,
598
+ headers={"User-Agent":
599
+ _net_common.USER_AGENT,
600
+ "Accept": _ACCEPT}) as response:
601
+ if response.is_redirect:
602
+ # .next_request is None on a redirect with no usable
603
+ # Location; treat that as the end of the chain rather
604
+ # than following nothing.
605
+ if response.next_request is None:
606
+ break
607
+ url = str(response.next_request.url)
608
+ continue
609
+ response.raise_for_status()
610
+ kind, raw, more = _bounded_body(response, url)
611
+ except _Refused as refused:
612
+ return {"error": str(refused)}
613
+ except httpx.HTTPError as e:
614
+ # Interpolated, NOT extra={}: the default formatter renders only
615
+ # %(message)s, so every field passed via extra was silently
616
+ # dropped -- fifteen consecutive "fetch_url failed" lines with
617
+ # no URL and no reason, which is what made a run of ordinary
618
+ # 404s indistinguishable from a broken tool.
619
+ logger.warning("fetch_url failed for %s: %s", url, e)
620
+ return {"error": f"Could not fetch URL: {e}"}
621
+
622
+ held = _held(url, kind, raw, more, response, want)
623
+ if isinstance(held, dict):
624
+ return held
625
+ # Held before the page is cut, so a later offset reads the document
626
+ # this fetch actually retrieved (PG5). Keyed by both spellings,
627
+ # because `url` here is the one that ANSWERED and the model may
628
+ # page with the one it asked for.
629
+ _hold(parsed.url, held)
630
+ changed = (earlier is not None and bool(earlier.digest)
631
+ and earlier.digest != held.digest)
632
+
633
+ # `truncated` is still true when EITHER bound bit: more characters
634
+ # were decoded than are returned, or the byte cap stopped the read
635
+ # with the server still sending. A silently short page is how a
636
+ # grounding pass concludes a source does not say something.
637
+ return _held_page(held, want, changed)
638
+
639
+ return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}