@yahyahammad/venastine 0.3.9 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +109 -0
- package/README.md +13 -4
- package/bin/venastine.mjs +62 -6
- package/bin/venastine.py +3 -1
- package/cli_commands.py +10 -0
- package/config.yaml +51 -0
- package/config_schema.py +16 -0
- package/core/client.py +45 -4
- package/core/commands.py +4 -0
- package/core/provider_errors.py +40 -0
- package/core/reasoning/scholar.py +14 -5
- package/docs/SECURITY.md +6 -0
- package/mcp_client/client.py +49 -2
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/requirements.txt +19 -0
- package/tools/builtin/_net_common.py +878 -0
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +5 -1
- package/tools/builtin/fetch_url.py +388 -120
- package/tools/builtin/web_search.py +81 -7
- package/tools/registry.py +10 -0
- package/tui/app.py +54 -0
- package/tui/app.tcss +7 -0
- package/tui/widgets.py +48 -0
|
@@ -1,27 +1,34 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Optional
|
|
4
6
|
|
|
5
7
|
import httpx
|
|
6
8
|
from pydantic import BaseModel, Field
|
|
7
9
|
|
|
10
|
+
import config
|
|
8
11
|
from safety.policy_enforcement import is_url_permitted, redact_output_text
|
|
9
|
-
from tools.builtin
|
|
12
|
+
from tools.builtin import _net_common, _page_text
|
|
13
|
+
from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
|
|
10
14
|
|
|
11
15
|
logger = logging.getLogger(__name__)
|
|
12
16
|
|
|
13
17
|
_TOOL_DESCRIPTION = (
|
|
14
|
-
"Retrieve the
|
|
15
|
-
"returned
|
|
16
|
-
"
|
|
18
|
+
"Retrieve the readable text of a web page or PDF by URL. Pages are "
|
|
19
|
+
"returned as text without the site's navigation; a link in it is marked "
|
|
20
|
+
"like [L3], and the result lists the URL for each marker on that page. "
|
|
21
|
+
"Long documents are returned one page at a time -- when a result says "
|
|
22
|
+
"it was truncated, its message names the offset to pass for the next "
|
|
23
|
+
"page."
|
|
17
24
|
)
|
|
18
25
|
MAX_CONTENT_CHARS = 5000
|
|
19
26
|
|
|
20
|
-
# ROADMAP_v2 §31 (H7), #55. How many BYTES
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
# holds.
|
|
27
|
+
# ROADMAP_v2 §31 (H7), #55. How many BYTES of a TEXT body -- JSON, plain
|
|
28
|
+
# text, XML, source -- may be pulled off the wire before the connection is
|
|
29
|
+
# dropped, as distinct from how many characters are returned. Both are
|
|
30
|
+
# needed and they bound different things: MAX_CONTENT_CHARS is what the
|
|
31
|
+
# model reads, this is what the process holds.
|
|
25
32
|
#
|
|
26
33
|
# Generous next to 5000 characters -- a multi-byte encoding can spend
|
|
27
34
|
# four bytes on one of them -- because the point is not to be tight. It
|
|
@@ -30,19 +37,41 @@ MAX_CONTENT_CHARS = 5000
|
|
|
30
37
|
# whatever the URL points at. A Content-Length check is not the fix
|
|
31
38
|
# either; a hostile server can omit it or lie about it, so the only
|
|
32
39
|
# thing that actually bounds this is refusing to read past a cap.
|
|
40
|
+
#
|
|
41
|
+
# HTML AND PDF HAVE THEIR OWN BOUNDS SINCE BATCH 240 (`fetch_max_html_mb`,
|
|
42
|
+
# `fetch_max_pdf_mb`), the owner's amendment of PG6. Measured, an HTML page
|
|
43
|
+
# is 75-99% markup, so this cap stopped Wikipedia before its article's
|
|
44
|
+
# first sentence; what is held now is the EXTRACTED text, bounded by
|
|
45
|
+
# HELD_CHARS, so memory no longer follows the markup.
|
|
33
46
|
MAX_CONTENT_BYTES = 65_536
|
|
34
47
|
REQUEST_TIMEOUT_S = 8.0
|
|
35
48
|
|
|
49
|
+
# Batch 240, the owner's choice: how much extracted text one fetch holds to
|
|
50
|
+
# be paged through. A Wikipedia article is ~90,000 characters and the Llama
|
|
51
|
+
# 2 report ~260,000. An offset past it re-fetches and serves the window it
|
|
52
|
+
# asks for, so a longer document is still readable to its end; it is not
|
|
53
|
+
# all held at once.
|
|
54
|
+
HELD_CHARS = 200_000
|
|
55
|
+
|
|
56
|
+
# How many link markers one page of text lists URLs for. Past it the page
|
|
57
|
+
# says how many more it cited, so the cut is visible.
|
|
58
|
+
LINKS_PER_PAGE = 30
|
|
59
|
+
|
|
36
60
|
# Redirects are followed BY HAND (see run()), so this is the loop bound,
|
|
37
61
|
# not httpx's. httpx's own default is 20; this is lower because a
|
|
38
62
|
# research fetch that needs more than five hops is not a fetch worth
|
|
39
63
|
# making, and every hop costs a policy check and a DNS resolution.
|
|
40
64
|
MAX_REDIRECTS = 5
|
|
41
65
|
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
|
|
45
|
-
|
|
66
|
+
# What fetch_url asks for. A server choosing between representations then
|
|
67
|
+
# prefers the ones this tool can read.
|
|
68
|
+
_ACCEPT = ("text/html,application/xhtml+xml,application/pdf;q=0.9,"
|
|
69
|
+
"text/plain;q=0.8,*/*;q=0.5")
|
|
70
|
+
|
|
71
|
+
# ROADMAP_v3 §51 (PG5). How long a fetched document stays available to be
|
|
72
|
+
# paged through. The same 300 s `arxiv.py` and `web_search.py` each use, and
|
|
73
|
+
# for a weaker reason than theirs: they cache to avoid repeating a query,
|
|
74
|
+
# this caches so that page 2 IS page 2.
|
|
46
75
|
#
|
|
47
76
|
# MEASURED (batch 105), which is why this is a cache and not a re-fetch.
|
|
48
77
|
# Five real URLs fetched twice 45 seconds apart -- the interval a model
|
|
@@ -55,25 +84,27 @@ MAX_REDIRECTS = 5
|
|
|
55
84
|
# result able to say so. A news front page is exactly what a research run
|
|
56
85
|
# reads.
|
|
57
86
|
#
|
|
87
|
+
# Since batch 240 a page PAST the held window does re-fetch, because the
|
|
88
|
+
# owner asked for the whole document to be readable. The splice that
|
|
89
|
+
# measurement found is then named rather than hidden: the held entry
|
|
90
|
+
# carries a digest of the whole document's text, and a re-fetch whose
|
|
91
|
+
# digest differs says the document changed.
|
|
92
|
+
#
|
|
58
93
|
# The second measured reason is cost. Paging an 88 KB document by re-fetch
|
|
59
94
|
# is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
|
|
60
|
-
# 5.3 MB for `bbc.com/news`.
|
|
95
|
+
# 5.3 MB for `bbc.com/news`.
|
|
61
96
|
CACHE_TTL_S = 300
|
|
62
97
|
|
|
63
98
|
# Keyed by URL -- BOTH the requested spelling and the one that answered,
|
|
64
|
-
# because a redirect means the model may page with either. Holds
|
|
65
|
-
#
|
|
66
|
-
# the read).
|
|
99
|
+
# because a redirect means the model may page with either. Holds a `_Held`:
|
|
100
|
+
# a window of the document's REDACTED text and what is known about the rest.
|
|
67
101
|
#
|
|
68
102
|
# Redacted, since §52 (RB1). Two things follow and both are wanted: no page
|
|
69
|
-
# cut from this
|
|
70
|
-
# unredacted remote content for the life of the entry
|
|
71
|
-
# was a local that died with the call, and holding it extended the lifetime
|
|
72
|
-
# of text nobody here wrote from one function call to 300 seconds.
|
|
103
|
+
# cut from this text can split a credential, and the process no longer holds
|
|
104
|
+
# unredacted remote content for the life of the entry.
|
|
73
105
|
#
|
|
74
|
-
# The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6)
|
|
75
|
-
#
|
|
76
|
-
# because `TTLCache` expired on read alone and nothing read those keys again.
|
|
106
|
+
# The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6), and
|
|
107
|
+
# each entry by HELD_CHARS.
|
|
77
108
|
_body_cache = TTLCache(CACHE_TTL_S)
|
|
78
109
|
|
|
79
110
|
|
|
@@ -102,109 +133,350 @@ TOOL_SCHEMA = {
|
|
|
102
133
|
}
|
|
103
134
|
|
|
104
135
|
|
|
105
|
-
|
|
106
|
-
|
|
136
|
+
@dataclass
|
|
137
|
+
class _Held:
|
|
138
|
+
"""A window of one document's text, and what is known about the rest.
|
|
107
139
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
140
|
+
`start` and `total` are positions in the document's REDACTED text, the
|
|
141
|
+
only position space this tool speaks (§52, RB3). `more` says the byte
|
|
142
|
+
bound stopped the read with the server still sending, so text past
|
|
143
|
+
`total` was never retrieved at all.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
url: str # the URL that answered
|
|
147
|
+
text: str # [start, start + len(text)) of the document
|
|
148
|
+
start: int
|
|
149
|
+
total: int
|
|
150
|
+
more: bool
|
|
151
|
+
kind: str = "text" # "html", "pdf" or "text"
|
|
152
|
+
read_limit: int = MAX_CONTENT_BYTES
|
|
153
|
+
digest: str = ""
|
|
154
|
+
links: dict = field(default_factory=dict)
|
|
155
|
+
title: Optional[str] = None
|
|
156
|
+
pages: Optional[int] = None
|
|
157
|
+
no_text: bool = False
|
|
158
|
+
|
|
159
|
+
def covers(self, offset: int) -> bool:
|
|
160
|
+
"""Whether a page at `offset` can be served without a fetch."""
|
|
161
|
+
if offset >= self.total:
|
|
162
|
+
return True # past the end: the answer is known
|
|
163
|
+
return self.start <= offset < self.start + len(self.text)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class _Refused(Exception):
|
|
167
|
+
"""A body this tool will not read, with the sentence that says why."""
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---- Reading the body ------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
def _header(response, name: str) -> str:
|
|
173
|
+
headers = getattr(response, "headers", None) or {}
|
|
174
|
+
for key, value in headers.items():
|
|
175
|
+
if key.lower() == name:
|
|
176
|
+
return str(value)
|
|
177
|
+
return ""
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _content_type(response) -> str:
|
|
181
|
+
return _header(response, "content-type").split(";", 1)[0].strip().lower()
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _kind_from_type(content_type: str) -> Optional[str]:
|
|
185
|
+
"""What a declared Content-Type says the body is, or None to sniff."""
|
|
186
|
+
if not content_type or content_type == "application/octet-stream":
|
|
187
|
+
return None
|
|
188
|
+
if content_type in ("text/html", "application/xhtml+xml"):
|
|
189
|
+
return "html"
|
|
190
|
+
if content_type == "application/pdf":
|
|
191
|
+
return "pdf"
|
|
192
|
+
if content_type.startswith("text/") or content_type.endswith(
|
|
193
|
+
("+json", "+xml")) or content_type in (
|
|
194
|
+
"application/json", "application/xml", "application/javascript",
|
|
195
|
+
"application/x-javascript", "application/ld+json",
|
|
196
|
+
"application/x-yaml", "application/yaml", "application/toml",
|
|
197
|
+
"application/x-ndjson", "application/rss+xml",
|
|
198
|
+
"application/atom+xml"):
|
|
199
|
+
return "text"
|
|
200
|
+
return "binary"
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _sniff(head: bytes, declared: Optional[str]) -> str:
|
|
204
|
+
"""The body's kind, from its first bytes where the type did not say.
|
|
205
|
+
|
|
206
|
+
A PDF announces itself whatever its header claims: servers send them as
|
|
207
|
+
`application/octet-stream`, and some as `text/plain`.
|
|
208
|
+
"""
|
|
209
|
+
if head.lstrip()[:5] == b"%PDF-":
|
|
210
|
+
return "pdf"
|
|
211
|
+
if declared is not None:
|
|
212
|
+
return declared
|
|
213
|
+
sample = head[:1024].lower()
|
|
214
|
+
if b"<html" in sample or b"<!doctype html" in sample:
|
|
215
|
+
return "html"
|
|
216
|
+
if b"\x00" in sample:
|
|
217
|
+
return "binary"
|
|
218
|
+
return "text"
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _limit_for(kind: str) -> int:
|
|
222
|
+
if kind == "html":
|
|
223
|
+
return int(round(config.FETCH_MAX_HTML_MB * 1_000_000))
|
|
224
|
+
if kind == "pdf":
|
|
225
|
+
return int(round(config.FETCH_MAX_PDF_MB * 1_000_000))
|
|
226
|
+
return MAX_CONTENT_BYTES
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _too_large(url: str, size: int, limit: int) -> str:
|
|
230
|
+
return (f"{url} is a PDF of {size:,} bytes, larger than the "
|
|
231
|
+
f"{limit:,}-byte limit (fetch_max_pdf_mb in config.yaml), so it "
|
|
232
|
+
f"was not read.")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _bounded_body(response, url: str) -> tuple:
|
|
236
|
+
"""(kind, raw bytes, whether the server still had more to send).
|
|
237
|
+
|
|
238
|
+
Reads only as far as the body's kind allows: an HTML page to
|
|
239
|
+
`fetch_max_html_mb`, a PDF to `fetch_max_pdf_mb`, anything else that is
|
|
240
|
+
text to MAX_CONTENT_BYTES. The extra chunk beyond a cap is what makes
|
|
241
|
+
`more` honest: without it, a body that happens to end exactly at the
|
|
242
|
+
cap is indistinguishable from one that was cut off.
|
|
243
|
+
|
|
244
|
+
A PDF is the exception to "stop and say so": half a PDF has no index
|
|
245
|
+
and cannot be read, so one past its limit is refused whole -- before
|
|
246
|
+
any of it is read when its Content-Length says so. A binary body is
|
|
247
|
+
refused as soon as its first bytes show what it is.
|
|
112
248
|
|
|
113
249
|
NOT THE PLACE TO REDACT, though it is the first place the body exists
|
|
114
250
|
(§52). This reads BYTES and the redactor works on decoded strings: a
|
|
115
251
|
credential spanning the boundary between two `iter_bytes` chunks would
|
|
116
|
-
be as invisible here as one spanning two pages is downstream
|
|
117
|
-
|
|
118
|
-
|
|
252
|
+
be as invisible here as one spanning two pages is downstream. Redaction
|
|
253
|
+
runs once, on the whole decoded text, above both the cache and the
|
|
254
|
+
slice.
|
|
119
255
|
"""
|
|
120
|
-
|
|
256
|
+
declared = _kind_from_type(_content_type(response))
|
|
257
|
+
if declared == "pdf":
|
|
258
|
+
length = _header(response, "content-length")
|
|
259
|
+
if length.isdigit() and int(length) > _limit_for("pdf"):
|
|
260
|
+
raise _Refused(_too_large(url, int(length), _limit_for("pdf")))
|
|
261
|
+
if declared == "binary":
|
|
262
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
263
|
+
chunks, total, kind, limit = [], 0, None, MAX_CONTENT_BYTES
|
|
121
264
|
for chunk in response.iter_bytes():
|
|
122
265
|
chunks.append(chunk)
|
|
123
266
|
total += len(chunk)
|
|
124
|
-
if total
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
267
|
+
if kind is None and (total >= 1024 or not chunk):
|
|
268
|
+
kind = _sniff(b"".join(chunks), declared)
|
|
269
|
+
if kind == "binary":
|
|
270
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
271
|
+
limit = _limit_for(kind)
|
|
272
|
+
if kind is not None and total > limit:
|
|
273
|
+
if kind == "pdf":
|
|
274
|
+
raise _Refused(_too_large(url, total, limit))
|
|
275
|
+
raw = b"".join(chunks)[:limit]
|
|
276
|
+
return kind, raw, True
|
|
277
|
+
raw = b"".join(chunks)
|
|
278
|
+
if kind is None:
|
|
279
|
+
# A body shorter than the sniff window is typed here, after the
|
|
280
|
+
# loop -- and must still meet its limit, which a short limit can
|
|
281
|
+
# make shorter than the window (the test that moves the constants).
|
|
282
|
+
kind = _sniff(raw, declared)
|
|
283
|
+
if kind == "binary":
|
|
284
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
285
|
+
limit = _limit_for(kind)
|
|
286
|
+
if len(raw) > limit:
|
|
287
|
+
if kind == "pdf":
|
|
288
|
+
raise _Refused(_too_large(url, len(raw), limit))
|
|
289
|
+
return kind, raw[:limit], True
|
|
290
|
+
return kind, raw, False
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _binary(url: str, content_type: str) -> str:
|
|
294
|
+
named = content_type or "a binary type"
|
|
295
|
+
return (f"{url} answered with {named}, which is not text, HTML or a PDF, "
|
|
296
|
+
f"so fetch_url did not read it. To keep the file itself, "
|
|
297
|
+
f"download it with a shell command.")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _decode(raw: bytes, response) -> str:
|
|
128
301
|
encoding = getattr(response, "encoding", None) or "utf-8"
|
|
129
302
|
try:
|
|
130
|
-
return raw.decode(encoding, errors="replace")
|
|
303
|
+
return raw.decode(encoding, errors="replace")
|
|
131
304
|
except LookupError:
|
|
132
305
|
# A server may name an encoding this build of Python does not
|
|
133
306
|
# have. That is not a reason to lose the page.
|
|
134
|
-
return raw.decode("utf-8", errors="replace")
|
|
307
|
+
return raw.decode("utf-8", errors="replace")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
# ---- Turning a body into a document ----------------------------------------
|
|
311
|
+
|
|
312
|
+
def _keep_link(third: ThirdPartyText):
|
|
313
|
+
"""The link filter `_page_text` applies: a URL this tool would refuse,
|
|
314
|
+
or one too long for it to accept, is not offered as a route (A58)."""
|
|
315
|
+
def keep(url: str) -> Optional[str]:
|
|
316
|
+
kept, withheld = third.url(url)
|
|
317
|
+
if kept is None or withheld:
|
|
318
|
+
return None
|
|
319
|
+
if is_url_permitted(kept, resolve=False):
|
|
320
|
+
return None
|
|
321
|
+
return kept
|
|
322
|
+
return keep
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _window_start(want: int) -> int:
|
|
326
|
+
"""Where a held window begins for a page at `want`."""
|
|
327
|
+
return 0 if want < HELD_CHARS else want
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _held(url: str, kind: str, raw: bytes, more: bool, response,
|
|
331
|
+
want: int):
|
|
332
|
+
"""The `_Held` for a fetched body, or an error dict for a PDF that could
|
|
333
|
+
not be read."""
|
|
334
|
+
start = _window_start(want)
|
|
335
|
+
if kind == "pdf":
|
|
336
|
+
return _held_pdf(url, raw, start)
|
|
337
|
+
text = _decode(raw, response)
|
|
338
|
+
links: dict = {}
|
|
339
|
+
title = None
|
|
340
|
+
if kind == "html":
|
|
341
|
+
third = ThirdPartyText("fetch_url")
|
|
342
|
+
document = _page_text.html_document(text, url, _keep_link(third))
|
|
343
|
+
text, links, title = document.text, document.links, document.title
|
|
344
|
+
if title is not None:
|
|
345
|
+
title = redact_output_text(title)
|
|
346
|
+
# §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position is
|
|
347
|
+
# the fix: the redactor sees the whole document exactly once, while it
|
|
348
|
+
# is still one string. Everything below -- the held window, page 1, and
|
|
349
|
+
# every page a later offset cuts out of it -- is a slice of clean text.
|
|
350
|
+
text = _redacted(url, text)
|
|
351
|
+
return _Held(url=url, text=text[start:start + HELD_CHARS], start=start,
|
|
352
|
+
total=len(text), more=more, kind=kind,
|
|
353
|
+
read_limit=_limit_for(kind),
|
|
354
|
+
digest=_page_text.text_digest(text), links=links,
|
|
355
|
+
title=title)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _held_pdf(url: str, raw: bytes, start: int):
|
|
359
|
+
"""A PDF's window, read in the killable child (the owner's choice)."""
|
|
360
|
+
from tools import isolation
|
|
361
|
+
from tools.builtin import _pdf_text
|
|
362
|
+
|
|
363
|
+
result = isolation.run_isolated(
|
|
364
|
+
_pdf_text.pdf_window,
|
|
365
|
+
{"data": _pdf_text.encode(raw), "start": start, "limit": HELD_CHARS},
|
|
366
|
+
config.TOOL_COMPUTE_TIMEOUT_S, tool_name="fetch_url")
|
|
367
|
+
if "error" in result:
|
|
368
|
+
return {"error": f"Could not read the PDF at {url}: {result['error']}"}
|
|
369
|
+
if result.get("redacted"):
|
|
370
|
+
logger.warning(
|
|
371
|
+
"Output policy altered fetch_url result: credentials replaced "
|
|
372
|
+
"in the text of %s before it was paged.", url)
|
|
373
|
+
title = result.get("title")
|
|
374
|
+
return _Held(url=url, text=result.get("text", ""), start=start,
|
|
375
|
+
total=int(result.get("total", 0)), more=False, kind="pdf",
|
|
376
|
+
read_limit=_limit_for("pdf"),
|
|
377
|
+
digest=result.get("digest", ""),
|
|
378
|
+
title=redact_output_text(title) if title else None,
|
|
379
|
+
pages=result.get("pages"),
|
|
380
|
+
no_text=bool(result.get("no_text")))
|
|
381
|
+
|
|
135
382
|
|
|
383
|
+
# ---- One page of a held document -------------------------------------------
|
|
136
384
|
|
|
137
385
|
def _page(url: str, body: str, more: bool, offset: int) -> dict:
|
|
138
|
-
"""One page of a held body, plus how to reach the next (§51
|
|
386
|
+
"""One page of a whole held body, plus how to reach the next (§51).
|
|
387
|
+
|
|
388
|
+
The shape every caller before batch 240 used, kept for them: it is a
|
|
389
|
+
held document whose window is the whole body.
|
|
390
|
+
"""
|
|
391
|
+
return _held_page(_Held(url=url, text=body, start=0, total=len(body),
|
|
392
|
+
more=more, read_limit=MAX_CONTENT_BYTES), offset)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _held_page(held: _Held, offset: int, changed: bool = False) -> dict:
|
|
396
|
+
"""One page of a held document, plus how to reach the next (§51, PG1/PG4).
|
|
139
397
|
|
|
140
398
|
`read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
|
|
141
399
|
a message naming the offset to use next. One paging idiom across the
|
|
142
400
|
tools that have one, rather than a second one to learn here.
|
|
143
401
|
|
|
144
|
-
THE
|
|
145
|
-
|
|
146
|
-
offset 0 it computes to precisely what it did before §51 -- the byte
|
|
147
|
-
cap bit, or the body is longer than one page. `source_corpus` and the
|
|
402
|
+
THE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS. `truncated` is
|
|
403
|
+
still "there is more than you were given". `source_corpus` and the
|
|
148
404
|
grounding passes read `url` and `content`; the register's constraint is
|
|
149
|
-
that added keys are safe and changed ones are not.
|
|
405
|
+
that added keys are safe and changed ones are not. Batch 240 adds
|
|
406
|
+
`title` on the first page, `links` for the markers a page carries, and
|
|
407
|
+
`format`/`pages` for a PDF.
|
|
150
408
|
|
|
151
|
-
EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because
|
|
152
|
-
|
|
409
|
+
EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because the
|
|
410
|
+
text arrives redacted. That is the only position space this tool
|
|
153
411
|
speaks, and it is self-consistent: a caller only ever passes back an
|
|
154
|
-
offset this function named.
|
|
155
|
-
be discovered -- a document containing credentials reports a smaller
|
|
156
|
-
`chars_available` than its source has, and the same URL reports
|
|
157
|
-
different totals with `REDACT_TOOL_OUTPUTS` on and off.
|
|
412
|
+
offset this function named.
|
|
158
413
|
"""
|
|
159
|
-
total =
|
|
160
|
-
|
|
414
|
+
total = held.total
|
|
415
|
+
local = offset - held.start
|
|
416
|
+
end = _page_text.split_marker_end(held.text, local + MAX_CONTENT_CHARS)
|
|
417
|
+
chunk = held.text[local:end] if 0 <= local < len(held.text) else ""
|
|
161
418
|
next_offset = offset + len(chunk)
|
|
162
419
|
beyond = next_offset < total
|
|
163
420
|
|
|
164
421
|
result = {
|
|
165
|
-
"url": url,
|
|
422
|
+
"url": held.url,
|
|
166
423
|
"content": chunk,
|
|
167
|
-
"truncated": beyond or more,
|
|
424
|
+
"truncated": beyond or held.more,
|
|
168
425
|
"offset": offset,
|
|
169
426
|
"chars_available": total,
|
|
170
427
|
}
|
|
171
|
-
|
|
428
|
+
if offset == 0 and held.title:
|
|
429
|
+
result["title"] = held.title
|
|
430
|
+
if held.kind == "pdf":
|
|
431
|
+
result["format"] = "pdf"
|
|
432
|
+
result["pages"] = held.pages
|
|
433
|
+
cited = _page_text.markers_in(chunk)
|
|
434
|
+
if cited:
|
|
435
|
+
listed = [n for n in cited if n in held.links][:LINKS_PER_PAGE]
|
|
436
|
+
result["links"] = {f"L{n}": held.links[n] for n in listed}
|
|
437
|
+
omitted = len([n for n in cited if n in held.links]) - len(listed)
|
|
438
|
+
if omitted:
|
|
439
|
+
result["links_omitted"] = omitted
|
|
440
|
+
|
|
441
|
+
notes = []
|
|
442
|
+
if changed:
|
|
443
|
+
notes.append(
|
|
444
|
+
"The document changed since an earlier page of it was read; "
|
|
445
|
+
"this page comes from the version fetched now.")
|
|
446
|
+
if held.no_text:
|
|
447
|
+
notes.append(
|
|
448
|
+
f"This PDF's {held.pages} pages have no text layer -- it is "
|
|
449
|
+
f"probably scanned images -- so there is nothing to read.")
|
|
172
450
|
if offset >= total and total:
|
|
173
451
|
# Past the end is not an error, for `read`'s reason: a model that
|
|
174
452
|
# walked one page too far should be told where the end was, not
|
|
175
453
|
# handed a failure it has to reason about.
|
|
176
|
-
|
|
454
|
+
notes.append(
|
|
177
455
|
f"Offset {offset} is past the end of this document, which has "
|
|
178
|
-
f"{total} characters. Nothing was returned."
|
|
179
|
-
)
|
|
456
|
+
f"{total} characters. Nothing was returned.")
|
|
180
457
|
elif beyond:
|
|
181
|
-
|
|
458
|
+
notes.append(
|
|
182
459
|
f"Returned characters {offset}-{next_offset} of {total}. "
|
|
183
|
-
f"Use offset={next_offset} to read further."
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
#
|
|
187
|
-
|
|
188
|
-
# name an offset that would return nothing. Measured (batch 105):
|
|
189
|
-
# an ordinary Python docs page is 88 KB, so roughly a quarter of it
|
|
190
|
-
# is past this bound, and `bbc.com/news` is 382 KB.
|
|
191
|
-
result["message"] = (
|
|
460
|
+
f"Use offset={next_offset} to read further.")
|
|
461
|
+
elif held.more:
|
|
462
|
+
# PG6's honesty, kept: the remainder was never retrieved, so name
|
|
463
|
+
# no offset that would return nothing.
|
|
464
|
+
notes.append(
|
|
192
465
|
f"Returned characters {offset}-{next_offset}, the end of the "
|
|
193
466
|
f"readable window. This document was cut at the "
|
|
194
|
-
f"{
|
|
195
|
-
f"never retrieved and no further page exists."
|
|
196
|
-
)
|
|
467
|
+
f"{held.read_limit}-byte read limit, so the rest of it was "
|
|
468
|
+
f"never retrieved and no further page exists.")
|
|
197
469
|
elif offset:
|
|
198
|
-
|
|
470
|
+
notes.append(
|
|
199
471
|
f"Returned characters {offset}-{next_offset} of {total}. "
|
|
200
|
-
f"This is the last page."
|
|
201
|
-
|
|
202
|
-
|
|
472
|
+
f"This is the last page.")
|
|
473
|
+
if notes:
|
|
474
|
+
result["message"] = " ".join(notes)
|
|
203
475
|
return result
|
|
204
476
|
|
|
205
477
|
|
|
206
478
|
def _redacted(url: str, body: str) -> str:
|
|
207
|
-
"""The
|
|
479
|
+
"""The text with credentials replaced, said once (§52, RB1 and RB2).
|
|
208
480
|
|
|
209
481
|
ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
|
|
210
482
|
`check_output_policy`, so this text was always redacted -- but it was
|
|
@@ -243,14 +515,14 @@ def _redacted(url: str, body: str) -> str:
|
|
|
243
515
|
|
|
244
516
|
|
|
245
517
|
def _cached(url: str):
|
|
246
|
-
"""The held
|
|
518
|
+
"""The held document for `url`, or None. Policy is the CALLER's job."""
|
|
247
519
|
return _body_cache.get(url)
|
|
248
520
|
|
|
249
521
|
|
|
250
|
-
def _hold(requested: str,
|
|
251
|
-
_body_cache.set(requested,
|
|
252
|
-
if
|
|
253
|
-
_body_cache.set(
|
|
522
|
+
def _hold(requested: str, held: _Held) -> None:
|
|
523
|
+
_body_cache.set(requested, held)
|
|
524
|
+
if held.url != requested:
|
|
525
|
+
_body_cache.set(held.url, held)
|
|
254
526
|
|
|
255
527
|
|
|
256
528
|
def run(params: dict) -> dict:
|
|
@@ -277,11 +549,13 @@ def run(params: dict) -> dict:
|
|
|
277
549
|
recording where the content actually came from.
|
|
278
550
|
"""
|
|
279
551
|
parsed = FetchURLParams(**params)
|
|
552
|
+
want = parsed.offset
|
|
280
553
|
|
|
281
|
-
# §51 (PG5). A page after the first is served from the
|
|
282
|
-
# one held, so that every page of
|
|
283
|
-
# page 2 continues page 1 rather than continuing whatever the server
|
|
284
|
-
#
|
|
554
|
+
# §51 (PG5). A page after the first is served from the document the
|
|
555
|
+
# first one held, so that every page of it comes from ONE fetch and
|
|
556
|
+
# page 2 continues page 1 rather than continuing whatever the server is
|
|
557
|
+
# serving now. A page past the held window (batch 240) is the one
|
|
558
|
+
# exception, and the version check below is what keeps it honest.
|
|
285
559
|
#
|
|
286
560
|
# THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
|
|
287
561
|
# consulted before the blocklist is a blocklist with a hole in it: the
|
|
@@ -294,14 +568,14 @@ def run(params: dict) -> dict:
|
|
|
294
568
|
# offset 0 never reads the cache: a fresh read of a page is a fresh
|
|
295
569
|
# read, which is what every caller that passes no offset already
|
|
296
570
|
# expects and what makes the held copy current for the pages after it.
|
|
297
|
-
|
|
571
|
+
earlier: Optional[_Held] = None
|
|
572
|
+
if want:
|
|
298
573
|
refusal = is_url_permitted(parsed.url)
|
|
299
574
|
if refusal:
|
|
300
575
|
return {"error": refusal}
|
|
301
|
-
|
|
302
|
-
if
|
|
303
|
-
|
|
304
|
-
return _page(answering, body, more, parsed.offset)
|
|
576
|
+
earlier = _cached(parsed.url)
|
|
577
|
+
if earlier is not None and earlier.covers(want):
|
|
578
|
+
return _held_page(earlier, want)
|
|
305
579
|
|
|
306
580
|
url = parsed.url
|
|
307
581
|
for _ in range(MAX_REDIRECTS):
|
|
@@ -313,12 +587,17 @@ def run(params: dict) -> dict:
|
|
|
313
587
|
# §31 (H7): streamed, so the headers are available before
|
|
314
588
|
# any body is pulled. Two consequences, both wanted -- a
|
|
315
589
|
# redirect now costs no body at all (the old path downloaded
|
|
316
|
-
# every hop's), and the body that IS wanted stops at
|
|
317
|
-
#
|
|
318
|
-
#
|
|
319
|
-
#
|
|
320
|
-
|
|
321
|
-
|
|
590
|
+
# every hop's), and the body that IS wanted stops at its
|
|
591
|
+
# bound instead of being buffered whole and then sliced.
|
|
592
|
+
# ROADMAP_v3 §53: `_net_common.stream` is `httpx.stream`,
|
|
593
|
+
# admitted and counted against the tool meter, once per hop.
|
|
594
|
+
# Batch 240: a descriptive User-Agent, because Wikimedia
|
|
595
|
+
# answers httpx's default with 403 (measured).
|
|
596
|
+
with _net_common.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
|
|
597
|
+
follow_redirects=False,
|
|
598
|
+
headers={"User-Agent":
|
|
599
|
+
_net_common.USER_AGENT,
|
|
600
|
+
"Accept": _ACCEPT}) as response:
|
|
322
601
|
if response.is_redirect:
|
|
323
602
|
# .next_request is None on a redirect with no usable
|
|
324
603
|
# Location; treat that as the end of the chain rather
|
|
@@ -328,7 +607,9 @@ def run(params: dict) -> dict:
|
|
|
328
607
|
url = str(response.next_request.url)
|
|
329
608
|
continue
|
|
330
609
|
response.raise_for_status()
|
|
331
|
-
|
|
610
|
+
kind, raw, more = _bounded_body(response, url)
|
|
611
|
+
except _Refused as refused:
|
|
612
|
+
return {"error": str(refused)}
|
|
332
613
|
except httpx.HTTPError as e:
|
|
333
614
|
# Interpolated, NOT extra={}: the default formatter renders only
|
|
334
615
|
# %(message)s, so every field passed via extra was silently
|
|
@@ -338,34 +619,21 @@ def run(params: dict) -> dict:
|
|
|
338
619
|
logger.warning("fetch_url failed for %s: %s", url, e)
|
|
339
620
|
return {"error": f"Could not fetch URL: {e}"}
|
|
340
621
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
#
|
|
345
|
-
# is a slice of already-clean text.
|
|
346
|
-
body = _redacted(url, body)
|
|
347
|
-
|
|
348
|
-
# Held before the page is cut, so a later offset reads the body
|
|
622
|
+
held = _held(url, kind, raw, more, response, want)
|
|
623
|
+
if isinstance(held, dict):
|
|
624
|
+
return held
|
|
625
|
+
# Held before the page is cut, so a later offset reads the document
|
|
349
626
|
# this fetch actually retrieved (PG5). Keyed by both spellings,
|
|
350
627
|
# because `url` here is the one that ANSWERED and the model may
|
|
351
628
|
# page with the one it asked for.
|
|
352
|
-
_hold(parsed.url,
|
|
629
|
+
_hold(parsed.url, held)
|
|
630
|
+
changed = (earlier is not None and bool(earlier.digest)
|
|
631
|
+
and earlier.digest != held.digest)
|
|
353
632
|
|
|
354
633
|
# `truncated` is still true when EITHER bound bit: more characters
|
|
355
634
|
# were decoded than are returned, or the byte cap stopped the read
|
|
356
635
|
# with the server still sending. A silently short page is how a
|
|
357
636
|
# grounding pass concludes a source does not say something.
|
|
358
|
-
|
|
359
|
-
# `or more` CANNOT FIRE with the shipped constants, and that is
|
|
360
|
-
# deliberate rather than an oversight: 65,536 bytes is at least
|
|
361
|
-
# 16,384 characters even at four bytes each, so the first term is
|
|
362
|
-
# always already true when the byte cap bites. It is here for the
|
|
363
|
-
# day someone raises MAX_CONTENT_CHARS or lowers
|
|
364
|
-
# MAX_CONTENT_BYTES, which is exactly when a silent under-report
|
|
365
|
-
# would appear. A mutation deleting it survived until a test moved
|
|
366
|
-
# the constants to reach it. §51 moved the expression into
|
|
367
|
-
# `_page` as `beyond or more`, where `beyond` is the same question
|
|
368
|
-
# asked of an arbitrary offset instead of only the first page.
|
|
369
|
-
return _page(url, body, more, parsed.offset)
|
|
637
|
+
return _held_page(held, want, changed)
|
|
370
638
|
|
|
371
639
|
return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}
|