@yahyahammad/venastine 0.3.9 → 0.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,27 +1,34 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import logging
4
+ from dataclasses import dataclass, field
5
+ from typing import Optional
4
6
 
5
7
  import httpx
6
8
  from pydantic import BaseModel, Field
7
9
 
10
+ import config
8
11
  from safety.policy_enforcement import is_url_permitted, redact_output_text
9
- from tools.builtin._net_common import MAX_URL_CHARS, TTLCache
12
+ from tools.builtin import _net_common, _page_text
13
+ from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
10
14
 
11
15
  logger = logging.getLogger(__name__)
12
16
 
13
17
  _TOOL_DESCRIPTION = (
14
- "Retrieve the contents of a web page using a URL. Long pages are "
15
- "returned one page at a time -- when a result says it was truncated, "
16
- "its message names the offset to pass for the next page."
18
+ "Retrieve the readable text of a web page or PDF by URL. Pages are "
19
+ "returned as text without the site's navigation; a link in it is marked "
20
+ "like [L3], and the result lists the URL for each marker on that page. "
21
+ "Long documents are returned one page at a time -- when a result says "
22
+ "it was truncated, its message names the offset to pass for the next "
23
+ "page."
17
24
  )
18
25
  MAX_CONTENT_CHARS = 5000
19
26
 
20
- # ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
21
- # before the connection is dropped, as distinct from how many characters
22
- # are returned. Both are needed and they bound different things:
23
- # MAX_CONTENT_CHARS is what the model reads, this is what the process
24
- # holds.
27
+ # ROADMAP_v2 §31 (H7), #55. How many BYTES of a TEXT body -- JSON, plain
28
+ # text, XML, source -- may be pulled off the wire before the connection is
29
+ # dropped, as distinct from how many characters are returned. Both are
30
+ # needed and they bound different things: MAX_CONTENT_CHARS is what the
31
+ # model reads, this is what the process holds.
25
32
  #
26
33
  # Generous next to 5000 characters -- a multi-byte encoding can spend
27
34
  # four bytes on one of them -- because the point is not to be tight. It
@@ -30,19 +37,41 @@ MAX_CONTENT_CHARS = 5000
30
37
  # whatever the URL points at. A Content-Length check is not the fix
31
38
  # either; a hostile server can omit it or lie about it, so the only
32
39
  # thing that actually bounds this is refusing to read past a cap.
40
+ #
41
+ # HTML AND PDF HAVE THEIR OWN BOUNDS SINCE BATCH 240 (`fetch_max_html_mb`,
42
+ # `fetch_max_pdf_mb`), the owner's amendment of PG6. Measured, an HTML page
43
+ # is 75-99% markup, so this cap stopped Wikipedia before its article's
44
+ # first sentence; what is held now is the EXTRACTED text, bounded by
45
+ # HELD_CHARS, so memory no longer follows the markup.
33
46
  MAX_CONTENT_BYTES = 65_536
34
47
  REQUEST_TIMEOUT_S = 8.0
35
48
 
49
+ # Batch 240, the owner's choice: how much extracted text one fetch holds to
50
+ # be paged through. A Wikipedia article is ~90,000 characters and the Llama
51
+ # 2 report ~260,000. An offset past it re-fetches and serves the window it
52
+ # asks for, so a longer document is still readable to its end; it is not
53
+ # all held at once.
54
+ HELD_CHARS = 200_000
55
+
56
+ # How many link markers one page of text lists URLs for. Past it the page
57
+ # says how many more it cited, so the cut is visible.
58
+ LINKS_PER_PAGE = 30
59
+
36
60
  # Redirects are followed BY HAND (see run()), so this is the loop bound,
37
61
  # not httpx's. httpx's own default is 20; this is lower because a
38
62
  # research fetch that needs more than five hops is not a fetch worth
39
63
  # making, and every hop costs a policy check and a DNS resolution.
40
64
  MAX_REDIRECTS = 5
41
65
 
42
- # ROADMAP_v3 §51 (PG5). How long a fetched body stays available to be paged
43
- # through. The same 300 s `arxiv.py` and `web_search.py` each use, and for a
44
- # weaker reason than theirs: they cache to avoid repeating a query, this
45
- # caches so that page 2 IS page 2.
66
+ # What fetch_url asks for. A server choosing between representations then
67
+ # prefers the ones this tool can read.
68
+ _ACCEPT = ("text/html,application/xhtml+xml,application/pdf;q=0.9,"
69
+ "text/plain;q=0.8,*/*;q=0.5")
70
+
71
+ # ROADMAP_v3 §51 (PG5). How long a fetched document stays available to be
72
+ # paged through. The same 300 s `arxiv.py` and `web_search.py` each use, and
73
+ # for a weaker reason than theirs: they cache to avoid repeating a query,
74
+ # this caches so that page 2 IS page 2.
46
75
  #
47
76
  # MEASURED (batch 105), which is why this is a cache and not a re-fetch.
48
77
  # Five real URLs fetched twice 45 seconds apart -- the interval a model
@@ -55,25 +84,27 @@ MAX_REDIRECTS = 5
55
84
  # result able to say so. A news front page is exactly what a research run
56
85
  # reads.
57
86
  #
87
+ # Since batch 240 a page PAST the held window does re-fetch, because the
88
+ # owner asked for the whole document to be readable. The splice that
89
+ # measurement found is then named rather than hidden: the held entry
90
+ # carries a digest of the whole document's text, and a re-fetch whose
91
+ # digest differs says the document changed.
92
+ #
58
93
  # The second measured reason is cost. Paging an 88 KB document by re-fetch
59
94
  # is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
60
- # 5.3 MB for `bbc.com/news`. The cache holds at most the byte cap per URL.
95
+ # 5.3 MB for `bbc.com/news`.
61
96
  CACHE_TTL_S = 300
62
97
 
63
98
  # Keyed by URL -- BOTH the requested spelling and the one that answered,
64
- # because a redirect means the model may page with either. Holds
65
- # (answering url, decoded and REDACTED body, whether the byte cap stopped
66
- # the read).
99
+ # because a redirect means the model may page with either. Holds a `_Held`:
100
+ # a window of the document's REDACTED text and what is known about the rest.
67
101
  #
68
102
  # Redacted, since §52 (RB1). Two things follow and both are wanted: no page
69
- # cut from this body can split a credential, and the process no longer holds
70
- # unredacted remote content for the life of the entry -- before §51 the body
71
- # was a local that died with the call, and holding it extended the lifetime
72
- # of text nobody here wrote from one function call to 300 seconds.
103
+ # cut from this text can split a credential, and the process no longer holds
104
+ # unredacted remote content for the life of the entry.
73
105
  #
74
- # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6).
75
- # Measured (batch 106): 100 fetches held 6.3 MB and nothing was ever dropped,
76
- # because `TTLCache` expired on read alone and nothing read those keys again.
106
+ # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6), and
107
+ # each entry by HELD_CHARS.
77
108
  _body_cache = TTLCache(CACHE_TTL_S)
78
109
 
79
110
 
@@ -102,109 +133,350 @@ TOOL_SCHEMA = {
102
133
  }
103
134
 
104
135
 
105
- def _bounded_body(response) -> tuple:
106
- """(decoded text, whether the server still had more to send).
136
+ @dataclass
137
+ class _Held:
138
+ """A window of one document's text, and what is known about the rest.
107
139
 
108
- Stops after MAX_CONTENT_BYTES rather than reading to the end. The
109
- extra chunk beyond the cap is what makes `more` honest: without it,
110
- a body that happens to end exactly at the cap is indistinguishable
111
- from one that was cut off.
140
+ `start` and `total` are positions in the document's REDACTED text, the
141
+ only position space this tool speaks (§52, RB3). `more` says the byte
142
+ bound stopped the read with the server still sending, so text past
143
+ `total` was never retrieved at all.
144
+ """
145
+
146
+ url: str # the URL that answered
147
+ text: str # [start, start + len(text)) of the document
148
+ start: int
149
+ total: int
150
+ more: bool
151
+ kind: str = "text" # "html", "pdf" or "text"
152
+ read_limit: int = MAX_CONTENT_BYTES
153
+ digest: str = ""
154
+ links: dict = field(default_factory=dict)
155
+ title: Optional[str] = None
156
+ pages: Optional[int] = None
157
+ no_text: bool = False
158
+
159
+ def covers(self, offset: int) -> bool:
160
+ """Whether a page at `offset` can be served without a fetch."""
161
+ if offset >= self.total:
162
+ return True # past the end: the answer is known
163
+ return self.start <= offset < self.start + len(self.text)
164
+
165
+
166
+ class _Refused(Exception):
167
+ """A body this tool will not read, with the sentence that says why."""
168
+
169
+
170
+ # ---- Reading the body ------------------------------------------------------
171
+
172
+ def _header(response, name: str) -> str:
173
+ headers = getattr(response, "headers", None) or {}
174
+ for key, value in headers.items():
175
+ if key.lower() == name:
176
+ return str(value)
177
+ return ""
178
+
179
+
180
+ def _content_type(response) -> str:
181
+ return _header(response, "content-type").split(";", 1)[0].strip().lower()
182
+
183
+
184
+ def _kind_from_type(content_type: str) -> Optional[str]:
185
+ """What a declared Content-Type says the body is, or None to sniff."""
186
+ if not content_type or content_type == "application/octet-stream":
187
+ return None
188
+ if content_type in ("text/html", "application/xhtml+xml"):
189
+ return "html"
190
+ if content_type == "application/pdf":
191
+ return "pdf"
192
+ if content_type.startswith("text/") or content_type.endswith(
193
+ ("+json", "+xml")) or content_type in (
194
+ "application/json", "application/xml", "application/javascript",
195
+ "application/x-javascript", "application/ld+json",
196
+ "application/x-yaml", "application/yaml", "application/toml",
197
+ "application/x-ndjson", "application/rss+xml",
198
+ "application/atom+xml"):
199
+ return "text"
200
+ return "binary"
201
+
202
+
203
+ def _sniff(head: bytes, declared: Optional[str]) -> str:
204
+ """The body's kind, from its first bytes where the type did not say.
205
+
206
+ A PDF announces itself whatever its header claims: servers send them as
207
+ `application/octet-stream`, and some as `text/plain`.
208
+ """
209
+ if head.lstrip()[:5] == b"%PDF-":
210
+ return "pdf"
211
+ if declared is not None:
212
+ return declared
213
+ sample = head[:1024].lower()
214
+ if b"<html" in sample or b"<!doctype html" in sample:
215
+ return "html"
216
+ if b"\x00" in sample:
217
+ return "binary"
218
+ return "text"
219
+
220
+
221
+ def _limit_for(kind: str) -> int:
222
+ if kind == "html":
223
+ return int(round(config.FETCH_MAX_HTML_MB * 1_000_000))
224
+ if kind == "pdf":
225
+ return int(round(config.FETCH_MAX_PDF_MB * 1_000_000))
226
+ return MAX_CONTENT_BYTES
227
+
228
+
229
+ def _too_large(url: str, size: int, limit: int) -> str:
230
+ return (f"{url} is a PDF of {size:,} bytes, larger than the "
231
+ f"{limit:,}-byte limit (fetch_max_pdf_mb in config.yaml), so it "
232
+ f"was not read.")
233
+
234
+
235
+ def _bounded_body(response, url: str) -> tuple:
236
+ """(kind, raw bytes, whether the server still had more to send).
237
+
238
+ Reads only as far as the body's kind allows: an HTML page to
239
+ `fetch_max_html_mb`, a PDF to `fetch_max_pdf_mb`, anything else that is
240
+ text to MAX_CONTENT_BYTES. The extra chunk beyond a cap is what makes
241
+ `more` honest: without it, a body that happens to end exactly at the
242
+ cap is indistinguishable from one that was cut off.
243
+
244
+ A PDF is the exception to "stop and say so": half a PDF has no index
245
+ and cannot be read, so one past its limit is refused whole -- before
246
+ any of it is read when its Content-Length says so. A binary body is
247
+ refused as soon as its first bytes show what it is.
112
248
 
113
249
  NOT THE PLACE TO REDACT, though it is the first place the body exists
114
250
  (§52). This reads BYTES and the redactor works on decoded strings: a
115
251
  credential spanning the boundary between two `iter_bytes` chunks would
116
- be as invisible here as one spanning two pages is downstream, which is
117
- the very failure §52 exists to close. Redaction runs once in `run()`,
118
- on the whole decoded body, above both the cache and the slice.
252
+ be as invisible here as one spanning two pages is downstream. Redaction
253
+ runs once, on the whole decoded text, above both the cache and the
254
+ slice.
119
255
  """
120
- chunks, total, more = [], 0, False
256
+ declared = _kind_from_type(_content_type(response))
257
+ if declared == "pdf":
258
+ length = _header(response, "content-length")
259
+ if length.isdigit() and int(length) > _limit_for("pdf"):
260
+ raise _Refused(_too_large(url, int(length), _limit_for("pdf")))
261
+ if declared == "binary":
262
+ raise _Refused(_binary(url, _content_type(response)))
263
+ chunks, total, kind, limit = [], 0, None, MAX_CONTENT_BYTES
121
264
  for chunk in response.iter_bytes():
122
265
  chunks.append(chunk)
123
266
  total += len(chunk)
124
- if total > MAX_CONTENT_BYTES:
125
- more = True
126
- break
127
- raw = b"".join(chunks)[:MAX_CONTENT_BYTES]
267
+ if kind is None and (total >= 1024 or not chunk):
268
+ kind = _sniff(b"".join(chunks), declared)
269
+ if kind == "binary":
270
+ raise _Refused(_binary(url, _content_type(response)))
271
+ limit = _limit_for(kind)
272
+ if kind is not None and total > limit:
273
+ if kind == "pdf":
274
+ raise _Refused(_too_large(url, total, limit))
275
+ raw = b"".join(chunks)[:limit]
276
+ return kind, raw, True
277
+ raw = b"".join(chunks)
278
+ if kind is None:
279
+ # A body shorter than the sniff window is typed here, after the
280
+ # loop -- and must still meet its limit, which a short limit can
281
+ # make shorter than the window (the test that moves the constants).
282
+ kind = _sniff(raw, declared)
283
+ if kind == "binary":
284
+ raise _Refused(_binary(url, _content_type(response)))
285
+ limit = _limit_for(kind)
286
+ if len(raw) > limit:
287
+ if kind == "pdf":
288
+ raise _Refused(_too_large(url, len(raw), limit))
289
+ return kind, raw[:limit], True
290
+ return kind, raw, False
291
+
292
+
293
+ def _binary(url: str, content_type: str) -> str:
294
+ named = content_type or "a binary type"
295
+ return (f"{url} answered with {named}, which is not text, HTML or a PDF, "
296
+ f"so fetch_url did not read it. To keep the file itself, "
297
+ f"download it with a shell command.")
298
+
299
+
300
+ def _decode(raw: bytes, response) -> str:
128
301
  encoding = getattr(response, "encoding", None) or "utf-8"
129
302
  try:
130
- return raw.decode(encoding, errors="replace"), more
303
+ return raw.decode(encoding, errors="replace")
131
304
  except LookupError:
132
305
  # A server may name an encoding this build of Python does not
133
306
  # have. That is not a reason to lose the page.
134
- return raw.decode("utf-8", errors="replace"), more
307
+ return raw.decode("utf-8", errors="replace")
308
+
309
+
310
+ # ---- Turning a body into a document ----------------------------------------
311
+
312
+ def _keep_link(third: ThirdPartyText):
313
+ """The link filter `_page_text` applies: a URL this tool would refuse,
314
+ or one too long for it to accept, is not offered as a route (A58)."""
315
+ def keep(url: str) -> Optional[str]:
316
+ kept, withheld = third.url(url)
317
+ if kept is None or withheld:
318
+ return None
319
+ if is_url_permitted(kept, resolve=False):
320
+ return None
321
+ return kept
322
+ return keep
323
+
324
+
325
+ def _window_start(want: int) -> int:
326
+ """Where a held window begins for a page at `want`."""
327
+ return 0 if want < HELD_CHARS else want
328
+
329
+
330
+ def _held(url: str, kind: str, raw: bytes, more: bool, response,
331
+ want: int):
332
+ """The `_Held` for a fetched body, or an error dict for a PDF that could
333
+ not be read."""
334
+ start = _window_start(want)
335
+ if kind == "pdf":
336
+ return _held_pdf(url, raw, start)
337
+ text = _decode(raw, response)
338
+ links: dict = {}
339
+ title = None
340
+ if kind == "html":
341
+ third = ThirdPartyText("fetch_url")
342
+ document = _page_text.html_document(text, url, _keep_link(third))
343
+ text, links, title = document.text, document.links, document.title
344
+ if title is not None:
345
+ title = redact_output_text(title)
346
+ # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position is
347
+ # the fix: the redactor sees the whole document exactly once, while it
348
+ # is still one string. Everything below -- the held window, page 1, and
349
+ # every page a later offset cuts out of it -- is a slice of clean text.
350
+ text = _redacted(url, text)
351
+ return _Held(url=url, text=text[start:start + HELD_CHARS], start=start,
352
+ total=len(text), more=more, kind=kind,
353
+ read_limit=_limit_for(kind),
354
+ digest=_page_text.text_digest(text), links=links,
355
+ title=title)
356
+
357
+
358
+ def _held_pdf(url: str, raw: bytes, start: int):
359
+ """A PDF's window, read in the killable child (the owner's choice)."""
360
+ from tools import isolation
361
+ from tools.builtin import _pdf_text
362
+
363
+ result = isolation.run_isolated(
364
+ _pdf_text.pdf_window,
365
+ {"data": _pdf_text.encode(raw), "start": start, "limit": HELD_CHARS},
366
+ config.TOOL_COMPUTE_TIMEOUT_S, tool_name="fetch_url")
367
+ if "error" in result:
368
+ return {"error": f"Could not read the PDF at {url}: {result['error']}"}
369
+ if result.get("redacted"):
370
+ logger.warning(
371
+ "Output policy altered fetch_url result: credentials replaced "
372
+ "in the text of %s before it was paged.", url)
373
+ title = result.get("title")
374
+ return _Held(url=url, text=result.get("text", ""), start=start,
375
+ total=int(result.get("total", 0)), more=False, kind="pdf",
376
+ read_limit=_limit_for("pdf"),
377
+ digest=result.get("digest", ""),
378
+ title=redact_output_text(title) if title else None,
379
+ pages=result.get("pages"),
380
+ no_text=bool(result.get("no_text")))
381
+
135
382
 
383
+ # ---- One page of a held document -------------------------------------------
136
384
 
137
385
  def _page(url: str, body: str, more: bool, offset: int) -> dict:
138
- """One page of a held body, plus how to reach the next (§51, PG1/PG4).
386
+ """One page of a whole held body, plus how to reach the next (§51).
387
+
388
+ The shape every caller before batch 240 used, kept for them: it is a
389
+ held document whose window is the whole body.
390
+ """
391
+ return _held_page(_Held(url=url, text=body, start=0, total=len(body),
392
+ more=more, read_limit=MAX_CONTENT_BYTES), offset)
393
+
394
+
395
+ def _held_page(held: _Held, offset: int, changed: bool = False) -> dict:
396
+ """One page of a held document, plus how to reach the next (§51, PG1/PG4).
139
397
 
140
398
  `read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
141
399
  a message naming the offset to use next. One paging idiom across the
142
400
  tools that have one, rather than a second one to learn here.
143
401
 
144
- THE THREE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS.
145
- `truncated` is still "there is more than you were given", and at
146
- offset 0 it computes to precisely what it did before §51 -- the byte
147
- cap bit, or the body is longer than one page. `source_corpus` and the
402
+ THE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS. `truncated` is
403
+ still "there is more than you were given". `source_corpus` and the
148
404
  grounding passes read `url` and `content`; the register's constraint is
149
- that added keys are safe and changed ones are not.
405
+ that added keys are safe and changed ones are not. Batch 240 adds
406
+ `title` on the first page, `links` for the markers a page carries, and
407
+ `format`/`pages` for a PDF.
150
408
 
151
- EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because
152
- `body` arrives redacted. That is the only position space this tool
409
+ EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because the
410
+ text arrives redacted. That is the only position space this tool
153
411
  speaks, and it is self-consistent: a caller only ever passes back an
154
- offset this function named. Two consequences, stated rather than left to
155
- be discovered -- a document containing credentials reports a smaller
156
- `chars_available` than its source has, and the same URL reports
157
- different totals with `REDACT_TOOL_OUTPUTS` on and off.
412
+ offset this function named.
158
413
  """
159
- total = len(body)
160
- chunk = body[offset:offset + MAX_CONTENT_CHARS]
414
+ total = held.total
415
+ local = offset - held.start
416
+ end = _page_text.split_marker_end(held.text, local + MAX_CONTENT_CHARS)
417
+ chunk = held.text[local:end] if 0 <= local < len(held.text) else ""
161
418
  next_offset = offset + len(chunk)
162
419
  beyond = next_offset < total
163
420
 
164
421
  result = {
165
- "url": url,
422
+ "url": held.url,
166
423
  "content": chunk,
167
- "truncated": beyond or more,
424
+ "truncated": beyond or held.more,
168
425
  "offset": offset,
169
426
  "chars_available": total,
170
427
  }
171
-
428
+ if offset == 0 and held.title:
429
+ result["title"] = held.title
430
+ if held.kind == "pdf":
431
+ result["format"] = "pdf"
432
+ result["pages"] = held.pages
433
+ cited = _page_text.markers_in(chunk)
434
+ if cited:
435
+ listed = [n for n in cited if n in held.links][:LINKS_PER_PAGE]
436
+ result["links"] = {f"L{n}": held.links[n] for n in listed}
437
+ omitted = len([n for n in cited if n in held.links]) - len(listed)
438
+ if omitted:
439
+ result["links_omitted"] = omitted
440
+
441
+ notes = []
442
+ if changed:
443
+ notes.append(
444
+ "The document changed since an earlier page of it was read; "
445
+ "this page comes from the version fetched now.")
446
+ if held.no_text:
447
+ notes.append(
448
+ f"This PDF's {held.pages} pages have no text layer -- it is "
449
+ f"probably scanned images -- so there is nothing to read.")
172
450
  if offset >= total and total:
173
451
  # Past the end is not an error, for `read`'s reason: a model that
174
452
  # walked one page too far should be told where the end was, not
175
453
  # handed a failure it has to reason about.
176
- result["message"] = (
454
+ notes.append(
177
455
  f"Offset {offset} is past the end of this document, which has "
178
- f"{total} characters. Nothing was returned."
179
- )
456
+ f"{total} characters. Nothing was returned.")
180
457
  elif beyond:
181
- result["message"] = (
458
+ notes.append(
182
459
  f"Returned characters {offset}-{next_offset} of {total}. "
183
- f"Use offset={next_offset} to read further."
184
- )
185
- elif more:
186
- # PG6, and the whole of it. The cap is NOT raised, so the honest
187
- # thing is to say the remainder was never retrieved rather than
188
- # name an offset that would return nothing. Measured (batch 105):
189
- # an ordinary Python docs page is 88 KB, so roughly a quarter of it
190
- # is past this bound, and `bbc.com/news` is 382 KB.
191
- result["message"] = (
460
+ f"Use offset={next_offset} to read further.")
461
+ elif held.more:
462
+ # PG6's honesty, kept: the remainder was never retrieved, so name
463
+ # no offset that would return nothing.
464
+ notes.append(
192
465
  f"Returned characters {offset}-{next_offset}, the end of the "
193
466
  f"readable window. This document was cut at the "
194
- f"{MAX_CONTENT_BYTES}-byte read limit, so the rest of it was "
195
- f"never retrieved and no further page exists."
196
- )
467
+ f"{held.read_limit}-byte read limit, so the rest of it was "
468
+ f"never retrieved and no further page exists.")
197
469
  elif offset:
198
- result["message"] = (
470
+ notes.append(
199
471
  f"Returned characters {offset}-{next_offset} of {total}. "
200
- f"This is the last page."
201
- )
202
-
472
+ f"This is the last page.")
473
+ if notes:
474
+ result["message"] = " ".join(notes)
203
475
  return result
204
476
 
205
477
 
206
478
  def _redacted(url: str, body: str) -> str:
207
- """The body with credentials replaced, said once (§52, RB1 and RB2).
479
+ """The text with credentials replaced, said once (§52, RB1 and RB2).
208
480
 
209
481
  ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
210
482
  `check_output_policy`, so this text was always redacted -- but it was
@@ -243,14 +515,14 @@ def _redacted(url: str, body: str) -> str:
243
515
 
244
516
 
245
517
  def _cached(url: str):
246
- """The held body for `url`, or None. Policy is the CALLER's job."""
518
+ """The held document for `url`, or None. Policy is the CALLER's job."""
247
519
  return _body_cache.get(url)
248
520
 
249
521
 
250
- def _hold(requested: str, answering: str, body: str, more: bool) -> None:
251
- _body_cache.set(requested, (answering, body, more))
252
- if answering != requested:
253
- _body_cache.set(answering, (answering, body, more))
522
+ def _hold(requested: str, held: _Held) -> None:
523
+ _body_cache.set(requested, held)
524
+ if held.url != requested:
525
+ _body_cache.set(held.url, held)
254
526
 
255
527
 
256
528
  def run(params: dict) -> dict:
@@ -277,11 +549,13 @@ def run(params: dict) -> dict:
277
549
  recording where the content actually came from.
278
550
  """
279
551
  parsed = FetchURLParams(**params)
552
+ want = parsed.offset
280
553
 
281
- # §51 (PG5). A page after the first is served from the body the first
282
- # one held, so that every page of a document comes from ONE fetch and
283
- # page 2 continues page 1 rather than continuing whatever the server
284
- # is serving now.
554
+ # §51 (PG5). A page after the first is served from the document the
555
+ # first one held, so that every page of it comes from ONE fetch and
556
+ # page 2 continues page 1 rather than continuing whatever the server is
557
+ # serving now. A page past the held window (batch 240) is the one
558
+ # exception, and the version check below is what keeps it honest.
285
559
  #
286
560
  # THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
287
561
  # consulted before the blocklist is a blocklist with a hole in it: the
@@ -294,14 +568,14 @@ def run(params: dict) -> dict:
294
568
  # offset 0 never reads the cache: a fresh read of a page is a fresh
295
569
  # read, which is what every caller that passes no offset already
296
570
  # expects and what makes the held copy current for the pages after it.
297
- if parsed.offset:
571
+ earlier: Optional[_Held] = None
572
+ if want:
298
573
  refusal = is_url_permitted(parsed.url)
299
574
  if refusal:
300
575
  return {"error": refusal}
301
- held = _cached(parsed.url)
302
- if held is not None:
303
- answering, body, more = held
304
- return _page(answering, body, more, parsed.offset)
576
+ earlier = _cached(parsed.url)
577
+ if earlier is not None and earlier.covers(want):
578
+ return _held_page(earlier, want)
305
579
 
306
580
  url = parsed.url
307
581
  for _ in range(MAX_REDIRECTS):
@@ -313,12 +587,17 @@ def run(params: dict) -> dict:
313
587
  # §31 (H7): streamed, so the headers are available before
314
588
  # any body is pulled. Two consequences, both wanted -- a
315
589
  # redirect now costs no body at all (the old path downloaded
316
- # every hop's), and the body that IS wanted stops at
317
- # MAX_CONTENT_BYTES instead of being buffered whole and then
318
- # sliced. read_run's guard is the model: bound before, not
319
- # after.
320
- with httpx.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
321
- follow_redirects=False) as response:
590
+ # every hop's), and the body that IS wanted stops at its
591
+ # bound instead of being buffered whole and then sliced.
592
+ # ROADMAP_v3 §53: `_net_common.stream` is `httpx.stream`,
593
+ # admitted and counted against the tool meter, once per hop.
594
+ # Batch 240: a descriptive User-Agent, because Wikimedia
595
+ # answers httpx's default with 403 (measured).
596
+ with _net_common.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
597
+ follow_redirects=False,
598
+ headers={"User-Agent":
599
+ _net_common.USER_AGENT,
600
+ "Accept": _ACCEPT}) as response:
322
601
  if response.is_redirect:
323
602
  # .next_request is None on a redirect with no usable
324
603
  # Location; treat that as the end of the chain rather
@@ -328,7 +607,9 @@ def run(params: dict) -> dict:
328
607
  url = str(response.next_request.url)
329
608
  continue
330
609
  response.raise_for_status()
331
- body, more = _bounded_body(response)
610
+ kind, raw, more = _bounded_body(response, url)
611
+ except _Refused as refused:
612
+ return {"error": str(refused)}
332
613
  except httpx.HTTPError as e:
333
614
  # Interpolated, NOT extra={}: the default formatter renders only
334
615
  # %(message)s, so every field passed via extra was silently
@@ -338,34 +619,21 @@ def run(params: dict) -> dict:
338
619
  logger.warning("fetch_url failed for %s: %s", url, e)
339
620
  return {"error": f"Could not fetch URL: {e}"}
340
621
 
341
- # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position
342
- # is the fix: the redactor sees the whole document exactly once,
343
- # while it is still one string. Everything below this line -- the
344
- # held copy, page 1, and every page a later offset cuts out of it --
345
- # is a slice of already-clean text.
346
- body = _redacted(url, body)
347
-
348
- # Held before the page is cut, so a later offset reads the body
622
+ held = _held(url, kind, raw, more, response, want)
623
+ if isinstance(held, dict):
624
+ return held
625
+ # Held before the page is cut, so a later offset reads the document
349
626
  # this fetch actually retrieved (PG5). Keyed by both spellings,
350
627
  # because `url` here is the one that ANSWERED and the model may
351
628
  # page with the one it asked for.
352
- _hold(parsed.url, url, body, more)
629
+ _hold(parsed.url, held)
630
+ changed = (earlier is not None and bool(earlier.digest)
631
+ and earlier.digest != held.digest)
353
632
 
354
633
  # `truncated` is still true when EITHER bound bit: more characters
355
634
  # were decoded than are returned, or the byte cap stopped the read
356
635
  # with the server still sending. A silently short page is how a
357
636
  # grounding pass concludes a source does not say something.
358
- #
359
- # `or more` CANNOT FIRE with the shipped constants, and that is
360
- # deliberate rather than an oversight: 65,536 bytes is at least
361
- # 16,384 characters even at four bytes each, so the first term is
362
- # always already true when the byte cap bites. It is here for the
363
- # day someone raises MAX_CONTENT_CHARS or lowers
364
- # MAX_CONTENT_BYTES, which is exactly when a silent under-report
365
- # would appear. A mutation deleting it survived until a test moved
366
- # the constants to reach it. §51 moved the expression into
367
- # `_page` as `beyond or more`, where `beyond` is the same question
368
- # asked of an arbitrary offset instead of only the first page.
369
- return _page(url, body, more, parsed.offset)
637
+ return _held_page(held, want, changed)
370
638
 
371
639
  return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}