@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,153 +1,371 @@
1
- from __future__ import annotations
2
-
3
- import logging
4
-
5
- import httpx
6
- from pydantic import BaseModel, Field
7
-
8
- from safety.policy_enforcement import is_url_permitted
9
-
10
- logger = logging.getLogger(__name__)
11
-
12
- _TOOL_DESCRIPTION = "Retrieve the contents of a web page using a URL."
13
- MAX_CONTENT_CHARS = 5000
14
-
15
- # ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
16
- # before the connection is dropped, as distinct from how many characters
17
- # are returned. Both are needed and they bound different things:
18
- # MAX_CONTENT_CHARS is what the model reads, this is what the process
19
- # holds.
20
- #
21
- # Generous next to 5000 characters -- a multi-byte encoding can spend
22
- # four bytes on one of them -- because the point is not to be tight. It
23
- # is that `response.text` had no ceiling AT ALL: httpx.get buffers the
24
- # entire body before the slice happens, and the body is chosen by
25
- # whatever the URL points at. A Content-Length check is not the fix
26
- # either; a hostile server can omit it or lie about it, so the only
27
- # thing that actually bounds this is refusing to read past a cap.
28
- MAX_CONTENT_BYTES = 65_536
29
- REQUEST_TIMEOUT_S = 8.0
30
-
31
- # Redirects are followed BY HAND (see run()), so this is the loop bound,
32
- # not httpx's. httpx's own default is 20; this is lower because a
33
- # research fetch that needs more than five hops is not a fetch worth
34
- # making, and every hop costs a policy check and a DNS resolution.
35
- MAX_REDIRECTS = 5
36
-
37
-
38
- class FetchURLParams(BaseModel):
39
- url: str = Field(..., description="URL to be fetched", min_length=1, max_length=400)
40
-
41
-
42
- TOOL_SCHEMA = {
43
- "name": "fetch_url",
44
- "description": _TOOL_DESCRIPTION,
45
- "input_schema": FetchURLParams.model_json_schema(),
46
- }
47
-
48
-
49
- def _bounded_body(response) -> tuple:
50
- """(decoded text, whether the server still had more to send).
51
-
52
- Stops after MAX_CONTENT_BYTES rather than reading to the end. The
53
- extra chunk beyond the cap is what makes `more` honest: without it,
54
- a body that happens to end exactly at the cap is indistinguishable
55
- from one that was cut off.
56
- """
57
- chunks, total, more = [], 0, False
58
- for chunk in response.iter_bytes():
59
- chunks.append(chunk)
60
- total += len(chunk)
61
- if total > MAX_CONTENT_BYTES:
62
- more = True
63
- break
64
- raw = b"".join(chunks)[:MAX_CONTENT_BYTES]
65
- encoding = getattr(response, "encoding", None) or "utf-8"
66
- try:
67
- return raw.decode(encoding, errors="replace"), more
68
- except LookupError:
69
- # A server may name an encoding this build of Python does not
70
- # have. That is not a reason to lose the page.
71
- return raw.decode("utf-8", errors="replace"), more
72
-
73
-
74
- def run(params: dict) -> dict:
75
- """Fetch one URL, checking policy BEFORE every request it issues.
76
-
77
- Redirects are followed manually (#53, #54). The version this replaced
78
- passed `follow_redirects=True` and checked the blocklist once, against
79
- the argument -- so one 302 walked past the control, and the blocked
80
- host had already served its body by the time anything could object.
81
- §25 R5's dispatch-wide check has the same blind spot for the same
82
- reason: it scans the argument, and the argument was clean.
83
-
84
- Checking each hop BEFORE issuing it, rather than checking the history
85
- afterwards, is the whole point. For a malware host the difference is
86
- whether it was contacted; for `169.254.169.254` the request IS the
87
- disclosure, so a post-hoc check does not fix #54 at all.
88
-
89
- The returned `url` is the one that ANSWERED, not the one that was
90
- asked for. That half is a correctness fix as much as a security one:
91
- the grounding passes attribute fetched text to the URL in this field,
92
- and output_writer builds each run's sources/ directory from it -- so
93
- a redirect chain used to silently rewrite what a claim was grounded
94
- in, with nothing in the result, the transcript or the MessageLog
95
- recording where the content actually came from.
96
- """
97
- parsed = FetchURLParams(**params)
98
-
99
- url = parsed.url
100
- for _ in range(MAX_REDIRECTS):
101
- refusal = is_url_permitted(url)
102
- if refusal:
103
- return {"error": refusal}
104
-
105
- try:
106
- # §31 (H7): streamed, so the headers are available before
107
- # any body is pulled. Two consequences, both wanted -- a
108
- # redirect now costs no body at all (the old path downloaded
109
- # every hop's), and the body that IS wanted stops at
110
- # MAX_CONTENT_BYTES instead of being buffered whole and then
111
- # sliced. read_run's guard is the model: bound before, not
112
- # after.
113
- with httpx.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
114
- follow_redirects=False) as response:
115
- if response.is_redirect:
116
- # .next_request is None on a redirect with no usable
117
- # Location; treat that as the end of the chain rather
118
- # than following nothing.
119
- if response.next_request is None:
120
- break
121
- url = str(response.next_request.url)
122
- continue
123
- response.raise_for_status()
124
- body, more = _bounded_body(response)
125
- except httpx.HTTPError as e:
126
- # Interpolated, NOT extra={}: the default formatter renders only
127
- # %(message)s, so every field passed via extra was silently
128
- # dropped -- fifteen consecutive "fetch_url failed" lines with
129
- # no URL and no reason, which is what made a run of ordinary
130
- # 404s indistinguishable from a broken tool.
131
- logger.warning("fetch_url failed for %s: %s", url, e)
132
- return {"error": f"Could not fetch URL: {e}"}
133
-
134
- return {
135
- "url": url,
136
- "content": body[:MAX_CONTENT_CHARS],
137
- # True when EITHER bound bit: more characters were decoded
138
- # than are returned, or the byte cap stopped the read with
139
- # the server still sending. A silently short page is how a
140
- # grounding pass concludes a source does not say something.
141
- #
142
- # `or more` CANNOT FIRE with the shipped constants, and that
143
- # is deliberate rather than an oversight: 65,536 bytes is at
144
- # least 16,384 characters even at four bytes each, so the
145
- # first term is always already true when the byte cap bites.
146
- # It is here for the day someone raises MAX_CONTENT_CHARS or
147
- # lowers MAX_CONTENT_BYTES, which is exactly when a silent
148
- # under-report would appear. A mutation deleting it survived
149
- # until a test moved the constants to reach it.
150
- "truncated": len(body) > MAX_CONTENT_CHARS or more,
151
- }
152
-
153
- return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+
5
+ import httpx
6
+ from pydantic import BaseModel, Field
7
+
8
+ from safety.policy_enforcement import is_url_permitted, redact_output_text
9
+ from tools.builtin._net_common import MAX_URL_CHARS, TTLCache
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ _TOOL_DESCRIPTION = (
14
+ "Retrieve the contents of a web page using a URL. Long pages are "
15
+ "returned one page at a time -- when a result says it was truncated, "
16
+ "its message names the offset to pass for the next page."
17
+ )
18
+ MAX_CONTENT_CHARS = 5000
19
+
20
+ # ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
21
+ # before the connection is dropped, as distinct from how many characters
22
+ # are returned. Both are needed and they bound different things:
23
+ # MAX_CONTENT_CHARS is what the model reads, this is what the process
24
+ # holds.
25
+ #
26
+ # Generous next to 5000 characters -- a multi-byte encoding can spend
27
+ # four bytes on one of them -- because the point is not to be tight. It
28
+ # is that `response.text` had no ceiling AT ALL: httpx.get buffers the
29
+ # entire body before the slice happens, and the body is chosen by
30
+ # whatever the URL points at. A Content-Length check is not the fix
31
+ # either; a hostile server can omit it or lie about it, so the only
32
+ # thing that actually bounds this is refusing to read past a cap.
33
+ MAX_CONTENT_BYTES = 65_536
34
+ REQUEST_TIMEOUT_S = 8.0
35
+
36
+ # Redirects are followed BY HAND (see run()), so this is the loop bound,
37
+ # not httpx's. httpx's own default is 20; this is lower because a
38
+ # research fetch that needs more than five hops is not a fetch worth
39
+ # making, and every hop costs a policy check and a DNS resolution.
40
+ MAX_REDIRECTS = 5
41
+
42
+ # ROADMAP_v3 §51 (PG5). How long a fetched body stays available to be paged
43
+ # through. The same 300 s `arxiv.py` and `web_search.py` each use, and for a
44
+ # weaker reason than theirs: they cache to avoid repeating a query, this
45
+ # caches so that page 2 IS page 2.
46
+ #
47
+ # MEASURED (batch 105), which is why this is a cache and not a re-fetch.
48
+ # Five real URLs fetched twice 45 seconds apart -- the interval a model
49
+ # actually pages over, not the milliseconds an eager probe measures:
50
+ # four were byte-identical and `news.ycombinator.com` was not. Its first
51
+ # difference landed at byte 2,019, INSIDE page 1, and the text at
52
+ # offset 5000..10000 differed between the two fetches. So a model that read
53
+ # page 1 and asked for page 2 would have been handed two halves of two
54
+ # different documents, spliced at an arbitrary point, with nothing in the
55
+ # result able to say so. A news front page is exactly what a research run
56
+ # reads.
57
+ #
58
+ # The second measured reason is cost. Paging an 88 KB document by re-fetch
59
+ # is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
60
+ # 5.3 MB for `bbc.com/news`. The cache holds at most the byte cap per URL.
61
+ CACHE_TTL_S = 300
62
+
63
+ # Keyed by URL -- BOTH the requested spelling and the one that answered,
64
+ # because a redirect means the model may page with either. Holds
65
+ # (answering url, decoded and REDACTED body, whether the byte cap stopped
66
+ # the read).
67
+ #
68
+ # Redacted, since §52 (RB1). Two things follow and both are wanted: no page
69
+ # cut from this body can split a credential, and the process no longer holds
70
+ # unredacted remote content for the life of the entry -- before §51 the body
71
+ # was a local that died with the call, and holding it extended the lifetime
72
+ # of text nobody here wrote from one function call to 300 seconds.
73
+ #
74
+ # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6).
75
+ # Measured (batch 106): 100 fetches held 6.3 MB and nothing was ever dropped,
76
+ # because `TTLCache` expired on read alone and nothing read those keys again.
77
+ _body_cache = TTLCache(CACHE_TTL_S)
78
+
79
+
80
+ class FetchURLParams(BaseModel):
81
+ # `MAX_URL_CHARS`, not 400 spelled again: batch 156 derived the
82
+ # bound the two SEARCH tools use from this one, because a URL
83
+ # they offer past it is a route this gate refuses. Two files
84
+ # holding one number is the state a pair is in just before it
85
+ # stops being a pair, and the relation is what matters here.
86
+ url: str = Field(..., description="URL to be fetched", min_length=1, max_length=MAX_URL_CHARS)
87
+ offset: int = Field(
88
+ 0,
89
+ ge=0,
90
+ description=(
91
+ "Number of characters to skip before reading (0-based). Use the "
92
+ "offset named in a truncated result's message to read the next "
93
+ "page of a long document."
94
+ ),
95
+ )
96
+
97
+
98
+ TOOL_SCHEMA = {
99
+ "name": "fetch_url",
100
+ "description": _TOOL_DESCRIPTION,
101
+ "input_schema": FetchURLParams.model_json_schema(),
102
+ }
103
+
104
+
105
+ def _bounded_body(response) -> tuple:
106
+ """(decoded text, whether the server still had more to send).
107
+
108
+ Stops after MAX_CONTENT_BYTES rather than reading to the end. The
109
+ extra chunk beyond the cap is what makes `more` honest: without it,
110
+ a body that happens to end exactly at the cap is indistinguishable
111
+ from one that was cut off.
112
+
113
+ NOT THE PLACE TO REDACT, though it is the first place the body exists
114
+ (§52). This reads BYTES and the redactor works on decoded strings: a
115
+ credential spanning the boundary between two `iter_bytes` chunks would
116
+ be as invisible here as one spanning two pages is downstream, which is
117
+ the very failure §52 exists to close. Redaction runs once in `run()`,
118
+ on the whole decoded body, above both the cache and the slice.
119
+ """
120
+ chunks, total, more = [], 0, False
121
+ for chunk in response.iter_bytes():
122
+ chunks.append(chunk)
123
+ total += len(chunk)
124
+ if total > MAX_CONTENT_BYTES:
125
+ more = True
126
+ break
127
+ raw = b"".join(chunks)[:MAX_CONTENT_BYTES]
128
+ encoding = getattr(response, "encoding", None) or "utf-8"
129
+ try:
130
+ return raw.decode(encoding, errors="replace"), more
131
+ except LookupError:
132
+ # A server may name an encoding this build of Python does not
133
+ # have. That is not a reason to lose the page.
134
+ return raw.decode("utf-8", errors="replace"), more
135
+
136
+
137
+ def _page(url: str, body: str, more: bool, offset: int) -> dict:
138
+ """One page of a held body, plus how to reach the next (§51, PG1/PG4).
139
+
140
+ `read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
141
+ a message naming the offset to use next. One paging idiom across the
142
+ tools that have one, rather than a second one to learn here.
143
+
144
+ THE THREE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS.
145
+ `truncated` is still "there is more than you were given", and at
146
+ offset 0 it computes to precisely what it did before §51 -- the byte
147
+ cap bit, or the body is longer than one page. `source_corpus` and the
148
+ grounding passes read `url` and `content`; the register's constraint is
149
+ that added keys are safe and changed ones are not.
150
+
151
+ EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because
152
+ `body` arrives redacted. That is the only position space this tool
153
+ speaks, and it is self-consistent: a caller only ever passes back an
154
+ offset this function named. Two consequences, stated rather than left to
155
+ be discovered -- a document containing credentials reports a smaller
156
+ `chars_available` than its source has, and the same URL reports
157
+ different totals with `REDACT_TOOL_OUTPUTS` on and off.
158
+ """
159
+ total = len(body)
160
+ chunk = body[offset:offset + MAX_CONTENT_CHARS]
161
+ next_offset = offset + len(chunk)
162
+ beyond = next_offset < total
163
+
164
+ result = {
165
+ "url": url,
166
+ "content": chunk,
167
+ "truncated": beyond or more,
168
+ "offset": offset,
169
+ "chars_available": total,
170
+ }
171
+
172
+ if offset >= total and total:
173
+ # Past the end is not an error, for `read`'s reason: a model that
174
+ # walked one page too far should be told where the end was, not
175
+ # handed a failure it has to reason about.
176
+ result["message"] = (
177
+ f"Offset {offset} is past the end of this document, which has "
178
+ f"{total} characters. Nothing was returned."
179
+ )
180
+ elif beyond:
181
+ result["message"] = (
182
+ f"Returned characters {offset}-{next_offset} of {total}. "
183
+ f"Use offset={next_offset} to read further."
184
+ )
185
+ elif more:
186
+ # PG6, and the whole of it. The cap is NOT raised, so the honest
187
+ # thing is to say the remainder was never retrieved rather than
188
+ # name an offset that would return nothing. Measured (batch 105):
189
+ # an ordinary Python docs page is 88 KB, so roughly a quarter of it
190
+ # is past this bound, and `bbc.com/news` is 382 KB.
191
+ result["message"] = (
192
+ f"Returned characters {offset}-{next_offset}, the end of the "
193
+ f"readable window. This document was cut at the "
194
+ f"{MAX_CONTENT_BYTES}-byte read limit, so the rest of it was "
195
+ f"never retrieved and no further page exists."
196
+ )
197
+ elif offset:
198
+ result["message"] = (
199
+ f"Returned characters {offset}-{next_offset} of {total}. "
200
+ f"This is the last page."
201
+ )
202
+
203
+ return result
204
+
205
+
206
+ def _redacted(url: str, body: str) -> str:
207
+ """The body with credentials replaced, said once (§52, RB1 and RB2).
208
+
209
+ ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
210
+ `check_output_policy`, so this text was always redacted -- but it was
211
+ redacted AFTER the page had been cut out of it, and a pattern only
212
+ matches what it can see whole. Measured (batch 106), a `ghp_` token
213
+ straddling character 5,000 survived both pages untouched, was rejoined by
214
+ the corpus's PG7 append, and landed in `sources/<sha256>.txt` and in the
215
+ window sent to the embedding provider. Every one of the ten patterns is
216
+ defeated by some cut; the fixed-length tokens and both credential shapes
217
+ are defeated by EVERY interior cut, because a prefix of a fixed-length
218
+ token is not that token.
219
+
220
+ This repo had already locked the rule twice. `param_digest` "redacts
221
+ BEFORE truncating" because "truncating first cuts a credential below the
222
+ 20 characters its pattern needs", and `tui/app.py`'s diff block runs on
223
+ whole texts because a secret "would be just as readable split across two
224
+ wrapped rows as on one". §51's paging is a third cut, and the first to
225
+ get the order wrong.
226
+
227
+ RB2, the warning. Redacting here means `check_output_policy` finds
228
+ nothing left to alter, so #49's "nothing this function does is silent"
229
+ would stop covering this tool without anybody noticing. One line at the
230
+ fetch replaces up to fourteen at the pages, and it keeps that function's
231
+ own rule: the match is never echoed, because saying what was found would
232
+ put it in app.log and every sink a WARNING reaches. The URL is named --
233
+ it is what makes the line actionable, and `logging_setup`'s formatter
234
+ redaction is the sink-side guard for a URL carrying userinfo, exactly as
235
+ it is for the `fetch_url failed for %s` line below.
236
+ """
237
+ clean = redact_output_text(body)
238
+ if clean != body:
239
+ logger.warning(
240
+ "Output policy altered fetch_url result: credentials replaced "
241
+ "in the body of %s before it was paged.", url)
242
+ return clean
243
+
244
+
245
+ def _cached(url: str):
246
+ """The held body for `url`, or None. Policy is the CALLER's job."""
247
+ return _body_cache.get(url)
248
+
249
+
250
+ def _hold(requested: str, answering: str, body: str, more: bool) -> None:
251
+ _body_cache.set(requested, (answering, body, more))
252
+ if answering != requested:
253
+ _body_cache.set(answering, (answering, body, more))
254
+
255
+
256
+ def run(params: dict) -> dict:
257
+ """Fetch one URL, checking policy BEFORE every request it issues.
258
+
259
+ Redirects are followed manually (#53, #54). The version this replaced
260
+ passed `follow_redirects=True` and checked the blocklist once, against
261
+ the argument -- so one 302 walked past the control, and the blocked
262
+ host had already served its body by the time anything could object.
263
+ §25 R5's dispatch-wide check has the same blind spot for the same
264
+ reason: it scans the argument, and the argument was clean.
265
+
266
+ Checking each hop BEFORE issuing it, rather than checking the history
267
+ afterwards, is the whole point. For a malware host the difference is
268
+ whether it was contacted; for `169.254.169.254` the request IS the
269
+ disclosure, so a post-hoc check does not fix #54 at all.
270
+
271
+ The returned `url` is the one that ANSWERED, not the one that was
272
+ asked for. That half is a correctness fix as much as a security one:
273
+ the grounding passes attribute fetched text to the URL in this field,
274
+ and output_writer builds each run's sources/ directory from it -- so
275
+ a redirect chain used to silently rewrite what a claim was grounded
276
+ in, with nothing in the result, the transcript or the MessageLog
277
+ recording where the content actually came from.
278
+ """
279
+ parsed = FetchURLParams(**params)
280
+
281
+ # §51 (PG5). A page after the first is served from the body the first
282
+ # one held, so that every page of a document comes from ONE fetch and
283
+ # page 2 continues page 1 rather than continuing whatever the server
284
+ # is serving now.
285
+ #
286
+ # THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
287
+ # consulted before the blocklist is a blocklist with a hole in it: the
288
+ # body was permitted when it was fetched, and `is_url_permitted` is not
289
+ # a constant -- it resolves names, and a run's own configuration can
290
+ # change under it. Skipping it here would rebuild #54 one layer up,
291
+ # where the control is bypassed by a second call rather than a
292
+ # redirect.
293
+ #
294
+ # offset 0 never reads the cache: a fresh read of a page is a fresh
295
+ # read, which is what every caller that passes no offset already
296
+ # expects and what makes the held copy current for the pages after it.
297
+ if parsed.offset:
298
+ refusal = is_url_permitted(parsed.url)
299
+ if refusal:
300
+ return {"error": refusal}
301
+ held = _cached(parsed.url)
302
+ if held is not None:
303
+ answering, body, more = held
304
+ return _page(answering, body, more, parsed.offset)
305
+
306
+ url = parsed.url
307
+ for _ in range(MAX_REDIRECTS):
308
+ refusal = is_url_permitted(url)
309
+ if refusal:
310
+ return {"error": refusal}
311
+
312
+ try:
313
+ # §31 (H7): streamed, so the headers are available before
314
+ # any body is pulled. Two consequences, both wanted -- a
315
+ # redirect now costs no body at all (the old path downloaded
316
+ # every hop's), and the body that IS wanted stops at
317
+ # MAX_CONTENT_BYTES instead of being buffered whole and then
318
+ # sliced. read_run's guard is the model: bound before, not
319
+ # after.
320
+ with httpx.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
321
+ follow_redirects=False) as response:
322
+ if response.is_redirect:
323
+ # .next_request is None on a redirect with no usable
324
+ # Location; treat that as the end of the chain rather
325
+ # than following nothing.
326
+ if response.next_request is None:
327
+ break
328
+ url = str(response.next_request.url)
329
+ continue
330
+ response.raise_for_status()
331
+ body, more = _bounded_body(response)
332
+ except httpx.HTTPError as e:
333
+ # Interpolated, NOT extra={}: the default formatter renders only
334
+ # %(message)s, so every field passed via extra was silently
335
+ # dropped -- fifteen consecutive "fetch_url failed" lines with
336
+ # no URL and no reason, which is what made a run of ordinary
337
+ # 404s indistinguishable from a broken tool.
338
+ logger.warning("fetch_url failed for %s: %s", url, e)
339
+ return {"error": f"Could not fetch URL: {e}"}
340
+
341
+ # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position
342
+ # is the fix: the redactor sees the whole document exactly once,
343
+ # while it is still one string. Everything below this line -- the
344
+ # held copy, page 1, and every page a later offset cuts out of it --
345
+ # is a slice of already-clean text.
346
+ body = _redacted(url, body)
347
+
348
+ # Held before the page is cut, so a later offset reads the body
349
+ # this fetch actually retrieved (PG5). Keyed by both spellings,
350
+ # because `url` here is the one that ANSWERED and the model may
351
+ # page with the one it asked for.
352
+ _hold(parsed.url, url, body, more)
353
+
354
+ # `truncated` is still true when EITHER bound bit: more characters
355
+ # were decoded than are returned, or the byte cap stopped the read
356
+ # with the server still sending. A silently short page is how a
357
+ # grounding pass concludes a source does not say something.
358
+ #
359
+ # `or more` CANNOT FIRE with the shipped constants, and that is
360
+ # deliberate rather than an oversight: 65,536 bytes is at least
361
+ # 16,384 characters even at four bytes each, so the first term is
362
+ # always already true when the byte cap bites. It is here for the
363
+ # day someone raises MAX_CONTENT_CHARS or lowers
364
+ # MAX_CONTENT_BYTES, which is exactly when a silent under-report
365
+ # would appear. A mutation deleting it survived until a test moved
366
+ # the constants to reach it. §51 moved the expression into
367
+ # `_page` as `beyond or more`, where `beyond` is the same question
368
+ # asked of an arbitrary offset instead of only the first page.
369
+ return _page(url, body, more, parsed.offset)
370
+
371
+ return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}