@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,371 +1,639 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import logging
|
|
4
|
-
|
|
5
|
-
import
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
from
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
#
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
#
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
#
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
"""
|
|
139
|
-
|
|
140
|
-
`
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
"""
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
import config
|
|
11
|
+
from safety.policy_enforcement import is_url_permitted, redact_output_text
|
|
12
|
+
from tools.builtin import _net_common, _page_text
|
|
13
|
+
from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
_TOOL_DESCRIPTION = (
|
|
18
|
+
"Retrieve the readable text of a web page or PDF by URL. Pages are "
|
|
19
|
+
"returned as text without the site's navigation; a link in it is marked "
|
|
20
|
+
"like [L3], and the result lists the URL for each marker on that page. "
|
|
21
|
+
"Long documents are returned one page at a time -- when a result says "
|
|
22
|
+
"it was truncated, its message names the offset to pass for the next "
|
|
23
|
+
"page."
|
|
24
|
+
)
|
|
25
|
+
MAX_CONTENT_CHARS = 5000
|
|
26
|
+
|
|
27
|
+
# ROADMAP_v2 §31 (H7), #55. How many BYTES of a TEXT body -- JSON, plain
|
|
28
|
+
# text, XML, source -- may be pulled off the wire before the connection is
|
|
29
|
+
# dropped, as distinct from how many characters are returned. Both are
|
|
30
|
+
# needed and they bound different things: MAX_CONTENT_CHARS is what the
|
|
31
|
+
# model reads, this is what the process holds.
|
|
32
|
+
#
|
|
33
|
+
# Generous next to 5000 characters -- a multi-byte encoding can spend
|
|
34
|
+
# four bytes on one of them -- because the point is not to be tight. It
|
|
35
|
+
# is that `response.text` had no ceiling AT ALL: httpx.get buffers the
|
|
36
|
+
# entire body before the slice happens, and the body is chosen by
|
|
37
|
+
# whatever the URL points at. A Content-Length check is not the fix
|
|
38
|
+
# either; a hostile server can omit it or lie about it, so the only
|
|
39
|
+
# thing that actually bounds this is refusing to read past a cap.
|
|
40
|
+
#
|
|
41
|
+
# HTML AND PDF HAVE THEIR OWN BOUNDS SINCE BATCH 240 (`fetch_max_html_mb`,
|
|
42
|
+
# `fetch_max_pdf_mb`), the owner's amendment of PG6. Measured, an HTML page
|
|
43
|
+
# is 75-99% markup, so this cap stopped Wikipedia before its article's
|
|
44
|
+
# first sentence; what is held now is the EXTRACTED text, bounded by
|
|
45
|
+
# HELD_CHARS, so memory no longer follows the markup.
|
|
46
|
+
MAX_CONTENT_BYTES = 65_536
|
|
47
|
+
REQUEST_TIMEOUT_S = 8.0
|
|
48
|
+
|
|
49
|
+
# Batch 240, the owner's choice: how much extracted text one fetch holds to
|
|
50
|
+
# be paged through. A Wikipedia article is ~90,000 characters and the Llama
|
|
51
|
+
# 2 report ~260,000. An offset past it re-fetches and serves the window it
|
|
52
|
+
# asks for, so a longer document is still readable to its end; it is not
|
|
53
|
+
# all held at once.
|
|
54
|
+
HELD_CHARS = 200_000
|
|
55
|
+
|
|
56
|
+
# How many link markers one page of text lists URLs for. Past it the page
|
|
57
|
+
# says how many more it cited, so the cut is visible.
|
|
58
|
+
LINKS_PER_PAGE = 30
|
|
59
|
+
|
|
60
|
+
# Redirects are followed BY HAND (see run()), so this is the loop bound,
|
|
61
|
+
# not httpx's. httpx's own default is 20; this is lower because a
|
|
62
|
+
# research fetch that needs more than five hops is not a fetch worth
|
|
63
|
+
# making, and every hop costs a policy check and a DNS resolution.
|
|
64
|
+
MAX_REDIRECTS = 5
|
|
65
|
+
|
|
66
|
+
# What fetch_url asks for. A server choosing between representations then
|
|
67
|
+
# prefers the ones this tool can read.
|
|
68
|
+
_ACCEPT = ("text/html,application/xhtml+xml,application/pdf;q=0.9,"
|
|
69
|
+
"text/plain;q=0.8,*/*;q=0.5")
|
|
70
|
+
|
|
71
|
+
# ROADMAP_v3 §51 (PG5). How long a fetched document stays available to be
|
|
72
|
+
# paged through. The same 300 s `arxiv.py` and `web_search.py` each use, and
|
|
73
|
+
# for a weaker reason than theirs: they cache to avoid repeating a query,
|
|
74
|
+
# this caches so that page 2 IS page 2.
|
|
75
|
+
#
|
|
76
|
+
# MEASURED (batch 105), which is why this is a cache and not a re-fetch.
|
|
77
|
+
# Five real URLs fetched twice 45 seconds apart -- the interval a model
|
|
78
|
+
# actually pages over, not the milliseconds an eager probe measures:
|
|
79
|
+
# four were byte-identical and `news.ycombinator.com` was not. Its first
|
|
80
|
+
# difference landed at byte 2,019, INSIDE page 1, and the text at
|
|
81
|
+
# offset 5000..10000 differed between the two fetches. So a model that read
|
|
82
|
+
# page 1 and asked for page 2 would have been handed two halves of two
|
|
83
|
+
# different documents, spliced at an arbitrary point, with nothing in the
|
|
84
|
+
# result able to say so. A news front page is exactly what a research run
|
|
85
|
+
# reads.
|
|
86
|
+
#
|
|
87
|
+
# Since batch 240 a page PAST the held window does re-fetch, because the
|
|
88
|
+
# owner asked for the whole document to be readable. The splice that
|
|
89
|
+
# measurement found is then named rather than hidden: the held entry
|
|
90
|
+
# carries a digest of the whole document's text, and a re-fetch whose
|
|
91
|
+
# digest differs says the document changed.
|
|
92
|
+
#
|
|
93
|
+
# The second measured reason is cost. Paging an 88 KB document by re-fetch
|
|
94
|
+
# is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
|
|
95
|
+
# 5.3 MB for `bbc.com/news`.
|
|
96
|
+
CACHE_TTL_S = 300
|
|
97
|
+
|
|
98
|
+
# Keyed by URL -- BOTH the requested spelling and the one that answered,
|
|
99
|
+
# because a redirect means the model may page with either. Holds a `_Held`:
|
|
100
|
+
# a window of the document's REDACTED text and what is known about the rest.
|
|
101
|
+
#
|
|
102
|
+
# Redacted, since §52 (RB1). Two things follow and both are wanted: no page
|
|
103
|
+
# cut from this text can split a credential, and the process no longer holds
|
|
104
|
+
# unredacted remote content for the life of the entry.
|
|
105
|
+
#
|
|
106
|
+
# The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6), and
|
|
107
|
+
# each entry by HELD_CHARS.
|
|
108
|
+
_body_cache = TTLCache(CACHE_TTL_S)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class FetchURLParams(BaseModel):
|
|
112
|
+
# `MAX_URL_CHARS`, not 400 spelled again: batch 156 derived the
|
|
113
|
+
# bound the two SEARCH tools use from this one, because a URL
|
|
114
|
+
# they offer past it is a route this gate refuses. Two files
|
|
115
|
+
# holding one number is the state a pair is in just before it
|
|
116
|
+
# stops being a pair, and the relation is what matters here.
|
|
117
|
+
url: str = Field(..., description="URL to be fetched", min_length=1, max_length=MAX_URL_CHARS)
|
|
118
|
+
offset: int = Field(
|
|
119
|
+
0,
|
|
120
|
+
ge=0,
|
|
121
|
+
description=(
|
|
122
|
+
"Number of characters to skip before reading (0-based). Use the "
|
|
123
|
+
"offset named in a truncated result's message to read the next "
|
|
124
|
+
"page of a long document."
|
|
125
|
+
),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
TOOL_SCHEMA = {
|
|
130
|
+
"name": "fetch_url",
|
|
131
|
+
"description": _TOOL_DESCRIPTION,
|
|
132
|
+
"input_schema": FetchURLParams.model_json_schema(),
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass
|
|
137
|
+
class _Held:
|
|
138
|
+
"""A window of one document's text, and what is known about the rest.
|
|
139
|
+
|
|
140
|
+
`start` and `total` are positions in the document's REDACTED text, the
|
|
141
|
+
only position space this tool speaks (§52, RB3). `more` says the byte
|
|
142
|
+
bound stopped the read with the server still sending, so text past
|
|
143
|
+
`total` was never retrieved at all.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
url: str # the URL that answered
|
|
147
|
+
text: str # [start, start + len(text)) of the document
|
|
148
|
+
start: int
|
|
149
|
+
total: int
|
|
150
|
+
more: bool
|
|
151
|
+
kind: str = "text" # "html", "pdf" or "text"
|
|
152
|
+
read_limit: int = MAX_CONTENT_BYTES
|
|
153
|
+
digest: str = ""
|
|
154
|
+
links: dict = field(default_factory=dict)
|
|
155
|
+
title: Optional[str] = None
|
|
156
|
+
pages: Optional[int] = None
|
|
157
|
+
no_text: bool = False
|
|
158
|
+
|
|
159
|
+
def covers(self, offset: int) -> bool:
|
|
160
|
+
"""Whether a page at `offset` can be served without a fetch."""
|
|
161
|
+
if offset >= self.total:
|
|
162
|
+
return True # past the end: the answer is known
|
|
163
|
+
return self.start <= offset < self.start + len(self.text)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class _Refused(Exception):
|
|
167
|
+
"""A body this tool will not read, with the sentence that says why."""
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---- Reading the body ------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
def _header(response, name: str) -> str:
|
|
173
|
+
headers = getattr(response, "headers", None) or {}
|
|
174
|
+
for key, value in headers.items():
|
|
175
|
+
if key.lower() == name:
|
|
176
|
+
return str(value)
|
|
177
|
+
return ""
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _content_type(response) -> str:
|
|
181
|
+
return _header(response, "content-type").split(";", 1)[0].strip().lower()
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _kind_from_type(content_type: str) -> Optional[str]:
|
|
185
|
+
"""What a declared Content-Type says the body is, or None to sniff."""
|
|
186
|
+
if not content_type or content_type == "application/octet-stream":
|
|
187
|
+
return None
|
|
188
|
+
if content_type in ("text/html", "application/xhtml+xml"):
|
|
189
|
+
return "html"
|
|
190
|
+
if content_type == "application/pdf":
|
|
191
|
+
return "pdf"
|
|
192
|
+
if content_type.startswith("text/") or content_type.endswith(
|
|
193
|
+
("+json", "+xml")) or content_type in (
|
|
194
|
+
"application/json", "application/xml", "application/javascript",
|
|
195
|
+
"application/x-javascript", "application/ld+json",
|
|
196
|
+
"application/x-yaml", "application/yaml", "application/toml",
|
|
197
|
+
"application/x-ndjson", "application/rss+xml",
|
|
198
|
+
"application/atom+xml"):
|
|
199
|
+
return "text"
|
|
200
|
+
return "binary"
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _sniff(head: bytes, declared: Optional[str]) -> str:
|
|
204
|
+
"""The body's kind, from its first bytes where the type did not say.
|
|
205
|
+
|
|
206
|
+
A PDF announces itself whatever its header claims: servers send them as
|
|
207
|
+
`application/octet-stream`, and some as `text/plain`.
|
|
208
|
+
"""
|
|
209
|
+
if head.lstrip()[:5] == b"%PDF-":
|
|
210
|
+
return "pdf"
|
|
211
|
+
if declared is not None:
|
|
212
|
+
return declared
|
|
213
|
+
sample = head[:1024].lower()
|
|
214
|
+
if b"<html" in sample or b"<!doctype html" in sample:
|
|
215
|
+
return "html"
|
|
216
|
+
if b"\x00" in sample:
|
|
217
|
+
return "binary"
|
|
218
|
+
return "text"
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _limit_for(kind: str) -> int:
|
|
222
|
+
if kind == "html":
|
|
223
|
+
return int(round(config.FETCH_MAX_HTML_MB * 1_000_000))
|
|
224
|
+
if kind == "pdf":
|
|
225
|
+
return int(round(config.FETCH_MAX_PDF_MB * 1_000_000))
|
|
226
|
+
return MAX_CONTENT_BYTES
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _too_large(url: str, size: int, limit: int) -> str:
|
|
230
|
+
return (f"{url} is a PDF of {size:,} bytes, larger than the "
|
|
231
|
+
f"{limit:,}-byte limit (fetch_max_pdf_mb in config.yaml), so it "
|
|
232
|
+
f"was not read.")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _bounded_body(response, url: str) -> tuple:
|
|
236
|
+
"""(kind, raw bytes, whether the server still had more to send).
|
|
237
|
+
|
|
238
|
+
Reads only as far as the body's kind allows: an HTML page to
|
|
239
|
+
`fetch_max_html_mb`, a PDF to `fetch_max_pdf_mb`, anything else that is
|
|
240
|
+
text to MAX_CONTENT_BYTES. The extra chunk beyond a cap is what makes
|
|
241
|
+
`more` honest: without it, a body that happens to end exactly at the
|
|
242
|
+
cap is indistinguishable from one that was cut off.
|
|
243
|
+
|
|
244
|
+
A PDF is the exception to "stop and say so": half a PDF has no index
|
|
245
|
+
and cannot be read, so one past its limit is refused whole -- before
|
|
246
|
+
any of it is read when its Content-Length says so. A binary body is
|
|
247
|
+
refused as soon as its first bytes show what it is.
|
|
248
|
+
|
|
249
|
+
NOT THE PLACE TO REDACT, though it is the first place the body exists
|
|
250
|
+
(§52). This reads BYTES and the redactor works on decoded strings: a
|
|
251
|
+
credential spanning the boundary between two `iter_bytes` chunks would
|
|
252
|
+
be as invisible here as one spanning two pages is downstream. Redaction
|
|
253
|
+
runs once, on the whole decoded text, above both the cache and the
|
|
254
|
+
slice.
|
|
255
|
+
"""
|
|
256
|
+
declared = _kind_from_type(_content_type(response))
|
|
257
|
+
if declared == "pdf":
|
|
258
|
+
length = _header(response, "content-length")
|
|
259
|
+
if length.isdigit() and int(length) > _limit_for("pdf"):
|
|
260
|
+
raise _Refused(_too_large(url, int(length), _limit_for("pdf")))
|
|
261
|
+
if declared == "binary":
|
|
262
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
263
|
+
chunks, total, kind, limit = [], 0, None, MAX_CONTENT_BYTES
|
|
264
|
+
for chunk in response.iter_bytes():
|
|
265
|
+
chunks.append(chunk)
|
|
266
|
+
total += len(chunk)
|
|
267
|
+
if kind is None and (total >= 1024 or not chunk):
|
|
268
|
+
kind = _sniff(b"".join(chunks), declared)
|
|
269
|
+
if kind == "binary":
|
|
270
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
271
|
+
limit = _limit_for(kind)
|
|
272
|
+
if kind is not None and total > limit:
|
|
273
|
+
if kind == "pdf":
|
|
274
|
+
raise _Refused(_too_large(url, total, limit))
|
|
275
|
+
raw = b"".join(chunks)[:limit]
|
|
276
|
+
return kind, raw, True
|
|
277
|
+
raw = b"".join(chunks)
|
|
278
|
+
if kind is None:
|
|
279
|
+
# A body shorter than the sniff window is typed here, after the
|
|
280
|
+
# loop -- and must still meet its limit, which a short limit can
|
|
281
|
+
# make shorter than the window (the test that moves the constants).
|
|
282
|
+
kind = _sniff(raw, declared)
|
|
283
|
+
if kind == "binary":
|
|
284
|
+
raise _Refused(_binary(url, _content_type(response)))
|
|
285
|
+
limit = _limit_for(kind)
|
|
286
|
+
if len(raw) > limit:
|
|
287
|
+
if kind == "pdf":
|
|
288
|
+
raise _Refused(_too_large(url, len(raw), limit))
|
|
289
|
+
return kind, raw[:limit], True
|
|
290
|
+
return kind, raw, False
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _binary(url: str, content_type: str) -> str:
|
|
294
|
+
named = content_type or "a binary type"
|
|
295
|
+
return (f"{url} answered with {named}, which is not text, HTML or a PDF, "
|
|
296
|
+
f"so fetch_url did not read it. To keep the file itself, "
|
|
297
|
+
f"download it with a shell command.")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _decode(raw: bytes, response) -> str:
|
|
301
|
+
encoding = getattr(response, "encoding", None) or "utf-8"
|
|
302
|
+
try:
|
|
303
|
+
return raw.decode(encoding, errors="replace")
|
|
304
|
+
except LookupError:
|
|
305
|
+
# A server may name an encoding this build of Python does not
|
|
306
|
+
# have. That is not a reason to lose the page.
|
|
307
|
+
return raw.decode("utf-8", errors="replace")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
# ---- Turning a body into a document ----------------------------------------
|
|
311
|
+
|
|
312
|
+
def _keep_link(third: ThirdPartyText):
|
|
313
|
+
"""The link filter `_page_text` applies: a URL this tool would refuse,
|
|
314
|
+
or one too long for it to accept, is not offered as a route (A58)."""
|
|
315
|
+
def keep(url: str) -> Optional[str]:
|
|
316
|
+
kept, withheld = third.url(url)
|
|
317
|
+
if kept is None or withheld:
|
|
318
|
+
return None
|
|
319
|
+
if is_url_permitted(kept, resolve=False):
|
|
320
|
+
return None
|
|
321
|
+
return kept
|
|
322
|
+
return keep
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _window_start(want: int) -> int:
|
|
326
|
+
"""Where a held window begins for a page at `want`."""
|
|
327
|
+
return 0 if want < HELD_CHARS else want
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _held(url: str, kind: str, raw: bytes, more: bool, response,
|
|
331
|
+
want: int):
|
|
332
|
+
"""The `_Held` for a fetched body, or an error dict for a PDF that could
|
|
333
|
+
not be read."""
|
|
334
|
+
start = _window_start(want)
|
|
335
|
+
if kind == "pdf":
|
|
336
|
+
return _held_pdf(url, raw, start)
|
|
337
|
+
text = _decode(raw, response)
|
|
338
|
+
links: dict = {}
|
|
339
|
+
title = None
|
|
340
|
+
if kind == "html":
|
|
341
|
+
third = ThirdPartyText("fetch_url")
|
|
342
|
+
document = _page_text.html_document(text, url, _keep_link(third))
|
|
343
|
+
text, links, title = document.text, document.links, document.title
|
|
344
|
+
if title is not None:
|
|
345
|
+
title = redact_output_text(title)
|
|
346
|
+
# §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position is
|
|
347
|
+
# the fix: the redactor sees the whole document exactly once, while it
|
|
348
|
+
# is still one string. Everything below -- the held window, page 1, and
|
|
349
|
+
# every page a later offset cuts out of it -- is a slice of clean text.
|
|
350
|
+
text = _redacted(url, text)
|
|
351
|
+
return _Held(url=url, text=text[start:start + HELD_CHARS], start=start,
|
|
352
|
+
total=len(text), more=more, kind=kind,
|
|
353
|
+
read_limit=_limit_for(kind),
|
|
354
|
+
digest=_page_text.text_digest(text), links=links,
|
|
355
|
+
title=title)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _held_pdf(url: str, raw: bytes, start: int):
|
|
359
|
+
"""A PDF's window, read in the killable child (the owner's choice)."""
|
|
360
|
+
from tools import isolation
|
|
361
|
+
from tools.builtin import _pdf_text
|
|
362
|
+
|
|
363
|
+
result = isolation.run_isolated(
|
|
364
|
+
_pdf_text.pdf_window,
|
|
365
|
+
{"data": _pdf_text.encode(raw), "start": start, "limit": HELD_CHARS},
|
|
366
|
+
config.TOOL_COMPUTE_TIMEOUT_S, tool_name="fetch_url")
|
|
367
|
+
if "error" in result:
|
|
368
|
+
return {"error": f"Could not read the PDF at {url}: {result['error']}"}
|
|
369
|
+
if result.get("redacted"):
|
|
370
|
+
logger.warning(
|
|
371
|
+
"Output policy altered fetch_url result: credentials replaced "
|
|
372
|
+
"in the text of %s before it was paged.", url)
|
|
373
|
+
title = result.get("title")
|
|
374
|
+
return _Held(url=url, text=result.get("text", ""), start=start,
|
|
375
|
+
total=int(result.get("total", 0)), more=False, kind="pdf",
|
|
376
|
+
read_limit=_limit_for("pdf"),
|
|
377
|
+
digest=result.get("digest", ""),
|
|
378
|
+
title=redact_output_text(title) if title else None,
|
|
379
|
+
pages=result.get("pages"),
|
|
380
|
+
no_text=bool(result.get("no_text")))
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
# ---- One page of a held document -------------------------------------------
|
|
384
|
+
|
|
385
|
+
def _page(url: str, body: str, more: bool, offset: int) -> dict:
|
|
386
|
+
"""One page of a whole held body, plus how to reach the next (§51).
|
|
387
|
+
|
|
388
|
+
The shape every caller before batch 240 used, kept for them: it is a
|
|
389
|
+
held document whose window is the whole body.
|
|
390
|
+
"""
|
|
391
|
+
return _held_page(_Held(url=url, text=body, start=0, total=len(body),
|
|
392
|
+
more=more, read_limit=MAX_CONTENT_BYTES), offset)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _held_page(held: _Held, offset: int, changed: bool = False) -> dict:
|
|
396
|
+
"""One page of a held document, plus how to reach the next (§51, PG1/PG4).
|
|
397
|
+
|
|
398
|
+
`read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
|
|
399
|
+
a message naming the offset to use next. One paging idiom across the
|
|
400
|
+
tools that have one, rather than a second one to learn here.
|
|
401
|
+
|
|
402
|
+
THE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS. `truncated` is
|
|
403
|
+
still "there is more than you were given". `source_corpus` and the
|
|
404
|
+
grounding passes read `url` and `content`; the register's constraint is
|
|
405
|
+
that added keys are safe and changed ones are not. Batch 240 adds
|
|
406
|
+
`title` on the first page, `links` for the markers a page carries, and
|
|
407
|
+
`format`/`pages` for a PDF.
|
|
408
|
+
|
|
409
|
+
EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because the
|
|
410
|
+
text arrives redacted. That is the only position space this tool
|
|
411
|
+
speaks, and it is self-consistent: a caller only ever passes back an
|
|
412
|
+
offset this function named.
|
|
413
|
+
"""
|
|
414
|
+
total = held.total
|
|
415
|
+
local = offset - held.start
|
|
416
|
+
end = _page_text.split_marker_end(held.text, local + MAX_CONTENT_CHARS)
|
|
417
|
+
chunk = held.text[local:end] if 0 <= local < len(held.text) else ""
|
|
418
|
+
next_offset = offset + len(chunk)
|
|
419
|
+
beyond = next_offset < total
|
|
420
|
+
|
|
421
|
+
result = {
|
|
422
|
+
"url": held.url,
|
|
423
|
+
"content": chunk,
|
|
424
|
+
"truncated": beyond or held.more,
|
|
425
|
+
"offset": offset,
|
|
426
|
+
"chars_available": total,
|
|
427
|
+
}
|
|
428
|
+
if offset == 0 and held.title:
|
|
429
|
+
result["title"] = held.title
|
|
430
|
+
if held.kind == "pdf":
|
|
431
|
+
result["format"] = "pdf"
|
|
432
|
+
result["pages"] = held.pages
|
|
433
|
+
cited = _page_text.markers_in(chunk)
|
|
434
|
+
if cited:
|
|
435
|
+
listed = [n for n in cited if n in held.links][:LINKS_PER_PAGE]
|
|
436
|
+
result["links"] = {f"L{n}": held.links[n] for n in listed}
|
|
437
|
+
omitted = len([n for n in cited if n in held.links]) - len(listed)
|
|
438
|
+
if omitted:
|
|
439
|
+
result["links_omitted"] = omitted
|
|
440
|
+
|
|
441
|
+
notes = []
|
|
442
|
+
if changed:
|
|
443
|
+
notes.append(
|
|
444
|
+
"The document changed since an earlier page of it was read; "
|
|
445
|
+
"this page comes from the version fetched now.")
|
|
446
|
+
if held.no_text:
|
|
447
|
+
notes.append(
|
|
448
|
+
f"This PDF's {held.pages} pages have no text layer -- it is "
|
|
449
|
+
f"probably scanned images -- so there is nothing to read.")
|
|
450
|
+
if offset >= total and total:
|
|
451
|
+
# Past the end is not an error, for `read`'s reason: a model that
|
|
452
|
+
# walked one page too far should be told where the end was, not
|
|
453
|
+
# handed a failure it has to reason about.
|
|
454
|
+
notes.append(
|
|
455
|
+
f"Offset {offset} is past the end of this document, which has "
|
|
456
|
+
f"{total} characters. Nothing was returned.")
|
|
457
|
+
elif beyond:
|
|
458
|
+
notes.append(
|
|
459
|
+
f"Returned characters {offset}-{next_offset} of {total}. "
|
|
460
|
+
f"Use offset={next_offset} to read further.")
|
|
461
|
+
elif held.more:
|
|
462
|
+
# PG6's honesty, kept: the remainder was never retrieved, so name
|
|
463
|
+
# no offset that would return nothing.
|
|
464
|
+
notes.append(
|
|
465
|
+
f"Returned characters {offset}-{next_offset}, the end of the "
|
|
466
|
+
f"readable window. This document was cut at the "
|
|
467
|
+
f"{held.read_limit}-byte read limit, so the rest of it was "
|
|
468
|
+
f"never retrieved and no further page exists.")
|
|
469
|
+
elif offset:
|
|
470
|
+
notes.append(
|
|
471
|
+
f"Returned characters {offset}-{next_offset} of {total}. "
|
|
472
|
+
f"This is the last page.")
|
|
473
|
+
if notes:
|
|
474
|
+
result["message"] = " ".join(notes)
|
|
475
|
+
return result
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _redacted(url: str, body: str) -> str:
|
|
479
|
+
"""The text with credentials replaced, said once (§52, RB1 and RB2).
|
|
480
|
+
|
|
481
|
+
ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
|
|
482
|
+
`check_output_policy`, so this text was always redacted -- but it was
|
|
483
|
+
redacted AFTER the page had been cut out of it, and a pattern only
|
|
484
|
+
matches what it can see whole. Measured (batch 106), a `ghp_` token
|
|
485
|
+
straddling character 5,000 survived both pages untouched, was rejoined by
|
|
486
|
+
the corpus's PG7 append, and landed in `sources/<sha256>.txt` and in the
|
|
487
|
+
window sent to the embedding provider. Every one of the ten patterns is
|
|
488
|
+
defeated by some cut; the fixed-length tokens and both credential shapes
|
|
489
|
+
are defeated by EVERY interior cut, because a prefix of a fixed-length
|
|
490
|
+
token is not that token.
|
|
491
|
+
|
|
492
|
+
This repo had already locked the rule twice. `param_digest` "redacts
|
|
493
|
+
BEFORE truncating" because "truncating first cuts a credential below the
|
|
494
|
+
20 characters its pattern needs", and `tui/app.py`'s diff block runs on
|
|
495
|
+
whole texts because a secret "would be just as readable split across two
|
|
496
|
+
wrapped rows as on one". §51's paging is a third cut, and the first to
|
|
497
|
+
get the order wrong.
|
|
498
|
+
|
|
499
|
+
RB2, the warning. Redacting here means `check_output_policy` finds
|
|
500
|
+
nothing left to alter, so #49's "nothing this function does is silent"
|
|
501
|
+
would stop covering this tool without anybody noticing. One line at the
|
|
502
|
+
fetch replaces up to fourteen at the pages, and it keeps that function's
|
|
503
|
+
own rule: the match is never echoed, because saying what was found would
|
|
504
|
+
put it in app.log and every sink a WARNING reaches. The URL is named --
|
|
505
|
+
it is what makes the line actionable, and `logging_setup`'s formatter
|
|
506
|
+
redaction is the sink-side guard for a URL carrying userinfo, exactly as
|
|
507
|
+
it is for the `fetch_url failed for %s` line below.
|
|
508
|
+
"""
|
|
509
|
+
clean = redact_output_text(body)
|
|
510
|
+
if clean != body:
|
|
511
|
+
logger.warning(
|
|
512
|
+
"Output policy altered fetch_url result: credentials replaced "
|
|
513
|
+
"in the body of %s before it was paged.", url)
|
|
514
|
+
return clean
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _cached(url: str):
|
|
518
|
+
"""The held document for `url`, or None. Policy is the CALLER's job."""
|
|
519
|
+
return _body_cache.get(url)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _hold(requested: str, held: _Held) -> None:
|
|
523
|
+
_body_cache.set(requested, held)
|
|
524
|
+
if held.url != requested:
|
|
525
|
+
_body_cache.set(held.url, held)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def run(params: dict) -> dict:
|
|
529
|
+
"""Fetch one URL, checking policy BEFORE every request it issues.
|
|
530
|
+
|
|
531
|
+
Redirects are followed manually (#53, #54). The version this replaced
|
|
532
|
+
passed `follow_redirects=True` and checked the blocklist once, against
|
|
533
|
+
the argument -- so one 302 walked past the control, and the blocked
|
|
534
|
+
host had already served its body by the time anything could object.
|
|
535
|
+
§25 R5's dispatch-wide check has the same blind spot for the same
|
|
536
|
+
reason: it scans the argument, and the argument was clean.
|
|
537
|
+
|
|
538
|
+
Checking each hop BEFORE issuing it, rather than checking the history
|
|
539
|
+
afterwards, is the whole point. For a malware host the difference is
|
|
540
|
+
whether it was contacted; for `169.254.169.254` the request IS the
|
|
541
|
+
disclosure, so a post-hoc check does not fix #54 at all.
|
|
542
|
+
|
|
543
|
+
The returned `url` is the one that ANSWERED, not the one that was
|
|
544
|
+
asked for. That half is a correctness fix as much as a security one:
|
|
545
|
+
the grounding passes attribute fetched text to the URL in this field,
|
|
546
|
+
and output_writer builds each run's sources/ directory from it -- so
|
|
547
|
+
a redirect chain used to silently rewrite what a claim was grounded
|
|
548
|
+
in, with nothing in the result, the transcript or the MessageLog
|
|
549
|
+
recording where the content actually came from.
|
|
550
|
+
"""
|
|
551
|
+
parsed = FetchURLParams(**params)
|
|
552
|
+
want = parsed.offset
|
|
553
|
+
|
|
554
|
+
# §51 (PG5). A page after the first is served from the document the
|
|
555
|
+
# first one held, so that every page of it comes from ONE fetch and
|
|
556
|
+
# page 2 continues page 1 rather than continuing whatever the server is
|
|
557
|
+
# serving now. A page past the held window (batch 240) is the one
|
|
558
|
+
# exception, and the version check below is what keeps it honest.
|
|
559
|
+
#
|
|
560
|
+
# THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
|
|
561
|
+
# consulted before the blocklist is a blocklist with a hole in it: the
|
|
562
|
+
# body was permitted when it was fetched, and `is_url_permitted` is not
|
|
563
|
+
# a constant -- it resolves names, and a run's own configuration can
|
|
564
|
+
# change under it. Skipping it here would rebuild #54 one layer up,
|
|
565
|
+
# where the control is bypassed by a second call rather than a
|
|
566
|
+
# redirect.
|
|
567
|
+
#
|
|
568
|
+
# offset 0 never reads the cache: a fresh read of a page is a fresh
|
|
569
|
+
# read, which is what every caller that passes no offset already
|
|
570
|
+
# expects and what makes the held copy current for the pages after it.
|
|
571
|
+
earlier: Optional[_Held] = None
|
|
572
|
+
if want:
|
|
573
|
+
refusal = is_url_permitted(parsed.url)
|
|
574
|
+
if refusal:
|
|
575
|
+
return {"error": refusal}
|
|
576
|
+
earlier = _cached(parsed.url)
|
|
577
|
+
if earlier is not None and earlier.covers(want):
|
|
578
|
+
return _held_page(earlier, want)
|
|
579
|
+
|
|
580
|
+
url = parsed.url
|
|
581
|
+
for _ in range(MAX_REDIRECTS):
|
|
582
|
+
refusal = is_url_permitted(url)
|
|
583
|
+
if refusal:
|
|
584
|
+
return {"error": refusal}
|
|
585
|
+
|
|
586
|
+
try:
|
|
587
|
+
# §31 (H7): streamed, so the headers are available before
|
|
588
|
+
# any body is pulled. Two consequences, both wanted -- a
|
|
589
|
+
# redirect now costs no body at all (the old path downloaded
|
|
590
|
+
# every hop's), and the body that IS wanted stops at its
|
|
591
|
+
# bound instead of being buffered whole and then sliced.
|
|
592
|
+
# ROADMAP_v3 §53: `_net_common.stream` is `httpx.stream`,
|
|
593
|
+
# admitted and counted against the tool meter, once per hop.
|
|
594
|
+
# Batch 240: a descriptive User-Agent, because Wikimedia
|
|
595
|
+
# answers httpx's default with 403 (measured).
|
|
596
|
+
with _net_common.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
|
|
597
|
+
follow_redirects=False,
|
|
598
|
+
headers={"User-Agent":
|
|
599
|
+
_net_common.USER_AGENT,
|
|
600
|
+
"Accept": _ACCEPT}) as response:
|
|
601
|
+
if response.is_redirect:
|
|
602
|
+
# .next_request is None on a redirect with no usable
|
|
603
|
+
# Location; treat that as the end of the chain rather
|
|
604
|
+
# than following nothing.
|
|
605
|
+
if response.next_request is None:
|
|
606
|
+
break
|
|
607
|
+
url = str(response.next_request.url)
|
|
608
|
+
continue
|
|
609
|
+
response.raise_for_status()
|
|
610
|
+
kind, raw, more = _bounded_body(response, url)
|
|
611
|
+
except _Refused as refused:
|
|
612
|
+
return {"error": str(refused)}
|
|
613
|
+
except httpx.HTTPError as e:
|
|
614
|
+
# Interpolated, NOT extra={}: the default formatter renders only
|
|
615
|
+
# %(message)s, so every field passed via extra was silently
|
|
616
|
+
# dropped -- fifteen consecutive "fetch_url failed" lines with
|
|
617
|
+
# no URL and no reason, which is what made a run of ordinary
|
|
618
|
+
# 404s indistinguishable from a broken tool.
|
|
619
|
+
logger.warning("fetch_url failed for %s: %s", url, e)
|
|
620
|
+
return {"error": f"Could not fetch URL: {e}"}
|
|
621
|
+
|
|
622
|
+
held = _held(url, kind, raw, more, response, want)
|
|
623
|
+
if isinstance(held, dict):
|
|
624
|
+
return held
|
|
625
|
+
# Held before the page is cut, so a later offset reads the document
|
|
626
|
+
# this fetch actually retrieved (PG5). Keyed by both spellings,
|
|
627
|
+
# because `url` here is the one that ANSWERED and the model may
|
|
628
|
+
# page with the one it asked for.
|
|
629
|
+
_hold(parsed.url, held)
|
|
630
|
+
changed = (earlier is not None and bool(earlier.digest)
|
|
631
|
+
and earlier.digest != held.digest)
|
|
632
|
+
|
|
633
|
+
# `truncated` is still true when EITHER bound bit: more characters
|
|
634
|
+
# were decoded than are returned, or the byte cap stopped the read
|
|
635
|
+
# with the server still sending. A silently short page is how a
|
|
636
|
+
# grounding pass concludes a source does not say something.
|
|
637
|
+
return _held_page(held, want, changed)
|
|
638
|
+
|
|
639
|
+
return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}
|