@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,1082 +1,1082 @@
1
- """
2
- safety/policy_enforcement.py
3
-
4
- Content-level policy, applied AFTER a tool runs, to its output — NOT
5
- access control (that's security/permissions.py's job, applied BEFORE a
6
- tool runs, deciding whether it runs at all). registry.py's dispatch()
7
- calls check_output_policy() on every tool result, regardless of which
8
- tool produced it.
9
-
10
- Three responsibilities:
11
- 1. Centralized URL policy — is_url_permitted() answers "may this URL be
12
- requested at all", composing the blocked-domain list with a
13
- non-public-address guard. web_search.py and fetch_url.py both call
14
- it, so a URL cannot be permitted for one and refused by the other.
15
- is_domain_blocked() remains the domain half on its own.
16
- 2. Secret redaction — defense-in-depth: if a tool's output accidentally
17
- contains something that looks like a leaked API key, it is replaced
18
- with [REDACTED] before the content reaches storage or the model.
19
- 3. Argument policy (§25) — the symmetric half of (2), applied BEFORE a
20
- tool runs, to what it was asked to do rather than to what it returned.
21
- Refuses rather than redacts, and refuses on the blocked-domain list
22
- too, which until then only the two tools that opted in enforced.
23
-
24
- (1) and (3) overlap deliberately: (3) is the choke point every tool
25
- passes through, (1) is what a tool checks about a value only it can
26
- interpret. Neither makes the other redundant.
27
- """
28
-
29
- from __future__ import annotations
30
-
31
- import ipaddress
32
- import json
33
- import logging
34
- import re
35
- import socket
36
- from urllib.parse import urlparse
37
-
38
- # safety -> security, and never the reverse. `security.posture` imports
39
- # `config` and the stdlib only, so this adds no cycle -- which is exactly
40
- # why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
41
- # this module, and the edge only stays acyclic while it runs one way.
42
- from security import posture
43
- from security import secrets as harness_secrets
44
-
45
- logger = logging.getLogger(__name__)
46
-
47
- # ---------------------------------------------------------------------------
48
- # ---- Blocked domains ------------------------------------------------------
49
- # ---------------------------------------------------------------------------
50
-
51
- # Centralized blocked-domain list. web_search.py and fetch_url.py import
52
- # is_domain_blocked() to enforce this. Add domains hosting malware,
53
- # phishing, or other harmful content based on your threat model.
54
- #
55
- # MAINTENANCE: domains change ownership, get taken down, or are repurposed.
56
- # Review this list periodically. Remove domains that are no longer harmful;
57
- # add new ones as threats emerge. An empty set disables domain filtering.
58
- BLOCKED_DOMAINS: set[str] = {
59
- # Well-known harmful domains — examples only, maintain as needed.
60
- "malware-traffic-analysis.net", # malware sample distribution
61
- "exploit-db.com", # exploit code repository
62
- "packetstormsecurity.com", # exploit/tool archive
63
- }
64
-
65
-
66
- def _hostname(url: str) -> str:
67
- """The host of a full URL or a bare domain, normalised for comparison.
68
-
69
- ONE normaliser, because the two branches below had drifted apart and
70
- the difference was a bypass (#48). `.hostname` already lowercases,
71
- strips the port and strips userinfo; the bare-domain branch did none
72
- of that except by accident, so `exploit-db.com:443` passed it while
73
- the URL form of the same host was caught.
74
-
75
- The trailing dot defeated BOTH branches. `exploit-db.com.` is a legal,
76
- resolvable FQDN naming exactly the same host, `urlparse` keeps the
77
- dot, and no amount of lowercasing makes the strings equal. Stripped
78
- here rather than at each call site, since every caller compares.
79
- """
80
- if "://" in url:
81
- try:
82
- host = urlparse(url).hostname or ""
83
- except Exception:
84
- host = url
85
- else:
86
- # Strip path, then userinfo, then port -- the three things
87
- # .hostname does for the URL branch and this one used not to.
88
- host = url.split("/")[0]
89
- host = host.rpartition("@")[2]
90
- if host.startswith("["): # [::1]:8080
91
- host = host.partition("]")[0].lstrip("[")
92
- else:
93
- host = host.partition(":")[0]
94
- return host.lower().rstrip(".")
95
-
96
-
97
- def is_domain_blocked(url: str) -> bool:
98
- """Return True if the URL's host is in, or under, BLOCKED_DOMAINS.
99
-
100
- Handles full URLs (https://example.com/path) and bare domains
101
- (example.com), through one normaliser -- see `_hostname`.
102
-
103
- Subdomains ARE matched: blocking ``evil.com`` also blocks
104
- ``sub.evil.com``. This reverses a documented carve-out, and the
105
- reversal is the point. A blocklist an attacker leaves by prepending
106
- one label is close to decorative, and the old docstring's advice
107
- ("block ``evil.com`` explicitly if you also want ``sub.evil.com``")
108
- asked a maintainer to enumerate an infinite set. Matching a suffix
109
- can only ever block MORE than the list's author wrote down, and this
110
- list's whole purpose is refusal, so over-blocking is the safe
111
- direction to be wrong in.
112
-
113
- The match is label-wise (`host == blocked` or `host` ends with
114
- `"." + blocked`), never a bare string suffix: `notevil.com` must not
115
- match `evil.com`.
116
- """
117
- if not BLOCKED_DOMAINS:
118
- return False
119
- host = _hostname(url)
120
- return any(host == blocked or host.endswith("." + blocked)
121
- for blocked in (b.lower().rstrip(".") for b in BLOCKED_DOMAINS))
122
-
123
-
124
- def is_url_permitted(url: str, resolve: bool = True) -> str | None:
125
- """Refusal reason for a URL, or None if it may be requested.
126
-
127
- ONE checker, two callers -- `fetch_url` before every hop it issues,
128
- and `web_search` over the results it is about to hand the model. They
129
- were asking different questions about the same policy, which is how
130
- #53 and #54 stayed open while #48 looked like the whole of it.
131
-
132
- Three gates, cheapest first:
133
-
134
- 1. Scheme. http/https only. Moved here from fetch_url so the answer
135
- does not depend on which caller asked.
136
- 2. `is_domain_blocked`.
137
- 3. Address. Rejects anything that is not a GLOBAL address --
138
- loopback, private, link-local, reserved and multicast in one
139
- predicate. `169.254.169.254` is the one that matters most: on a
140
- cloud instance with IMDSv1 it returns role credentials, and the
141
- route to it is a fetched page saying "for full details see
142
- <that URL>" to a grounding pass that obliges.
143
-
144
- `resolve=False` answers from the literal text only -- an IP literal
145
- is still checked, in every spelling the platform's resolver reads as
146
- a number (#207: `2130706433` and `0x7f.0.0.1` too, not only dotted
147
- quads), and a NAME is taken on trust. That is for `web_search`,
148
- which filters results rather than fetching them: a DNS lookup per
149
- search hit is real latency for a check whose failure mode is "the
150
- model is shown a URL that fetch_url will refuse anyway". The caller
151
- that actually opens a socket always resolves.
152
-
153
- KNOWN LIMITATION -- DNS rebinding. When we resolve, httpx then
154
- resolves again independently, so a host whose records change between
155
- this check and the connect can still be reached. Closing it means
156
- connecting to the pinned IP and passing the original host in a
157
- `Host:` header, which on HTTPS fights SNI and certificate
158
- verification -- a subtly wrong implementation there is a worse hole
159
- than this one. Recorded rather than silently accepted; the attacker
160
- needs control of DNS for a host the model was already induced to
161
- fetch.
162
- """
163
- if not (url.startswith("http://") or url.startswith("https://")):
164
- return "URL must start with http:// or https://"
165
-
166
- if is_domain_blocked(url):
167
- return f"Domain is blocked by policy: {url}"
168
-
169
- host = _hostname(url)
170
- if not host:
171
- return f"URL has no host: {url}"
172
-
173
- try:
174
- literal = ipaddress.ip_address(host)
175
- except ValueError:
176
- literal = None
177
-
178
- if literal is None and not resolve:
179
- # #207, batch 213. `ipaddress` reads dotted quads and IPv6 only, so
180
- # `2130706433`, `0x7f.0.0.1` and `127.1` -- all 127.0.0.1 to the
181
- # platform's resolver -- were taken on trust as NAMES here, and
182
- # only fetch_url refused them. AI_NUMERICHOST asks that resolver to
183
- # read the text as a number and never to look anything up, so it
184
- # costs no network, which was the whole reason for resolve=False.
185
- # Measured on Windows and in WSL: the same spellings parse, and a
186
- # name fails with gaierror.
187
- try:
188
- literal = ipaddress.ip_address(socket.getaddrinfo(
189
- host, None, flags=socket.AI_NUMERICHOST)[0][4][0])
190
- except (socket.gaierror, ValueError, IndexError):
191
- literal = None
192
-
193
- if literal is not None:
194
- addresses = [literal]
195
- elif resolve:
196
- try:
197
- addresses = [
198
- ipaddress.ip_address(info[4][0])
199
- for info in socket.getaddrinfo(host, None)
200
- ]
201
- except (socket.gaierror, ValueError) as e:
202
- # Fail CLOSED. A name we cannot resolve is one we cannot
203
- # clear, and httpx would only fail on it a moment later --
204
- # refusing here costs nothing and keeps "unchecked" from
205
- # ever meaning "allowed".
206
- return f"Could not resolve host for policy check: {host} ({e})"
207
- else:
208
- addresses = []
209
-
210
- # ANY, not all. A name resolving to one public and one private
211
- # address is the rebinding shape, and taking the first answer or
212
- # requiring unanimity would both let it through.
213
- for address in addresses:
214
- if not address.is_global:
215
- return (f"URL resolves to a non-public address, which this "
216
- f"harness will not fetch: {host} -> {address}")
217
-
218
- return None
219
-
220
-
221
- # ---------------------------------------------------------------------------
222
- # ---- Secret redaction -----------------------------------------------------
223
- # ---------------------------------------------------------------------------
224
-
225
- # A PEM body's line break: a real one, or the two-character escape a JSON
226
- # string carries -- a cloud service-account key file is exactly that shape.
227
- # Unambiguous on purpose: every repetition holds a mandatory break, so the
228
- # spaces between two breaks belong to exactly one repetition and a body
229
- # that fails to close backtracks in linear time, not exponential.
230
- _PEM_BREAK = r"(?:[ \t]*(?:\r?\n|\\r\\n|\\n))"
231
- _PEM_SEP = _PEM_BREAK + r"+[ \t]*"
232
- # One body line: base64, at least 16 characters, so a word of prose after a
233
- # quoted header is not one; or the two RFC 1421 headers a legacy encrypted
234
- # key carries before its base64.
235
- _PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
236
-
237
- _SECRET_PATTERNS = [
238
- # OpenAI-style keys
239
- re.compile(r"sk-[a-zA-Z0-9]{20,}"),
240
- # Anthropic-style keys
241
- re.compile(r"sk-ant-[a-zA-Z0-9\-]{20,}"),
242
- # GitHub personal access tokens
243
- re.compile(r"ghp_[a-zA-Z0-9]{36}"),
244
- # AWS access key IDs
245
- re.compile(r"AKIA[0-9A-Z]{16}"),
246
- # Google API keys
247
- re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
248
- # Slack tokens
249
- re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
250
- # PEM private keys: the header, and the WHOLE BLOCK when it is there
251
- # (TECHNICAL_DEBT 52, batch 160). This used to be the header line alone,
252
- # so the base64 body -- the key material -- survived verbatim under a
253
- # `[REDACTED]` that read as the key having been removed. The body is
254
- # OPTIONAL in the pattern on purpose: `_input_leaf` uses `.search()`,
255
- # and anything carrying the header still matches, so input refusals are
256
- # exactly what they were. The body is LINES, each after a line break:
257
- # a block with its END line goes whole, and one a cut ended goes as far
258
- # as its lines do. Lines rather than "anything up to END" because prose
259
- # quotes both markers -- "starts with -----BEGIN ... and ends with
260
- # -----END ..." -- and a lazy match took the sentence between them.
261
- # Measured: that sentence now loses its two markers and keeps its words.
262
- re.compile(
263
- r"-----BEGIN [A-Z ]*PRIVATE KEY-----"
264
- r"(?:(?:" + _PEM_SEP + _PEM_UNIT + r")*"
265
- r"(?:" + _PEM_SEP + r"[A-Za-z0-9+/=]{1,15})?"
266
- r"(?:" + _PEM_BREAK + r"*[ \t]*)-----END [A-Z ]*PRIVATE KEY-----"
267
- r"|(?:" + _PEM_SEP + _PEM_UNIT + r")*)"),
268
- # --- Batch 160, TECHNICAL_DEBT 52: the CURRENT siblings of the vendor
269
- # --- formats above, each of which the older pattern beside it misses
270
- # --- by a character class -- a `_` or a second `-` the 2024 shapes
271
- # --- never had. Measured through the real `redact_output_text` before
272
- # --- this: every one survived. They refuse inputs too, as `ghp_` and
273
- # --- `sk-` always have; one vendor list, one vocabulary.
274
- # GitHub fine-grained PATs, the format `ghp_` is being retired for
275
- re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
276
- # GitHub OAuth, user-to-server, server-to-server and refresh tokens
277
- re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
278
- # OpenAI project, service-account and admin keys: `sk-proj-...`
279
- re.compile(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{20,}"),
280
- # Stripe live secret and restricted keys
281
- re.compile(r"[sr]k_live_[A-Za-z0-9]{16,}"),
282
- # npm access tokens
283
- re.compile(r"npm_[A-Za-z0-9]{36}"),
284
- # --- ROADMAP_v3 §79 (HD5): the three credentials `/connect` hands the
285
- # --- harness that no pattern above can see. A sign-in puts them in a
286
- # --- session the model shares, so they are redacted and refused like
287
- # --- the rest of the list.
288
- # Hugging Face user access tokens and OAuth access tokens. An
289
- # identifier like `hf_hub_download` breaks the run of word characters
290
- # at its second underscore and is not matched.
291
- re.compile(r"(?<![A-Za-z0-9])hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
292
- # OpenRouter keys, which `sk-` above misses at the second hyphen
293
- re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
294
- # ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
295
- # bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
296
- # that merely ENDS a word like `items-<uuid>`.
297
- re.compile(r"(?<![A-Za-z0-9])ms-[0-9a-f]{8}-[0-9a-f]{4}"
298
- r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
299
- ]
300
-
301
-
302
- #: What this module puts where a secret was. Named in batch 65 rather
303
- #: than written out at each of the four substitutions, because a second
304
- #: module now has to RECOGNISE it: `tui/markdown.py` refuses to arm a
305
- #: URL wearing this mark for a click, since a redacted address is a real
306
- #: one and simply not the one that was fetched. That module is pure and
307
- #: may not import policy, so it keeps its own copy and
308
- #: `tests/test_markdown_render.py` holds the two against each other --
309
- #: one string, two readers, and a check rather than a hope.
310
- REDACTION_MARKER = "[REDACTED]"
311
-
312
-
313
- def redact_secrets(text: str) -> str:
314
- """Replace strings matching known secret patterns with [REDACTED]."""
315
- for pattern in _SECRET_PATTERNS:
316
- text = pattern.sub(REDACTION_MARKER, text)
317
- return text
318
-
319
-
320
- # ---------------------------------------------------------------------------
321
- # ---- Credential shapes (#167, batch 20) ------------------------------------
322
- # ---------------------------------------------------------------------------
323
- #
324
- # The patterns above are VENDOR TOKENS -- every one has a prefix some
325
- # company mints, so a match is high-confidence and a bare `[REDACTED]` loses
326
- # nothing structural. A credential that is just a password in a field has no
327
- # prefix, and nothing matched it (#167): batch 14 added five build manifests
328
- # to read_project_doc's allowlist (#94), whose credential conventions are
329
- # exactly the three unscrubbed shapes below.
330
- #
331
- # These are CONTEXT shapes, not tokens: each anchors on a credential FIELD
332
- # NAME (or URL userinfo), so an auth error code, a stack frame or an ARN
333
- # does not match, and the replacement is SURGICAL -- tag names, field names,
334
- # usernames and hosts survive, only the value goes. That structure is why
335
- # the false-positive cost is acceptable: documentation quoting an example
336
- # keeps its shape and loses only its example value.
337
- #
338
- # OUTPUT REDACTION ONLY. These never feed check_input_policy: refusing a
339
- # call because its arguments mention `password = "..."` would break every
340
- # legitimate call that carries build-file content the user asked for --
341
- # _SECRET_PATTERNS stay the input side's whole vocabulary. And the keyword
342
- # set is password|passwd ONLY, deliberately not token/secret/key: those
343
- # appear constantly in ordinary prose ("token bucket", JWT discussion) and
344
- # would redact aggressively.
345
-
346
- # Template placeholders survive untouched: `${PASSWORD}`, `$DB_PASS`,
347
- # `{{ secrets.DB_PW }}`, `<your-password-here>` name where a credential
348
- # WOULD go; they are what docs and templates contain, and redacting them
349
- # destroys the example while protecting nothing.
350
- _TEMPLATE_VALUE_RE = re.compile(
351
- r"^(?:"
352
- r"\$\{[^}]*\}" # ${PASSWORD}, ${secrets.DB}
353
- r"|\$[A-Za-z_][A-Za-z0-9_]*" # $PASSWORD
354
- r"|\{\{[^}]*\}\}" # {{ secrets.DB_PW }}
355
- r"|<[^<>]+>" # <your-password-here>
356
- r")$")
357
-
358
- # scheme://user:secret@host -> scheme://user:[REDACTED]@host. Credentials
359
- # in a URL's userinfo are never anything else, which makes this the cheapest
360
- # of the three; anchoring on `//` keeps bare `user:pass@host` fragments in
361
- # prose alone, and excluding `/ \s : @` from both sides means a host:port
362
- # path (`https://example.com:8080/x`) cannot masquerade as userinfo.
363
- _URL_USERINFO_RE = re.compile(r"(//[^/\s:@]+:)([^/\s@]+)(@)")
364
-
365
- # <password>s3cr3t</password> -> <password>[REDACTED]</password>, POM and
366
- # settings.xml convention. The backreference keeps the closing tag honest;
367
- # the value excludes `<`, so nested markup cannot be swallowed. Groups:
368
- # 1 opening tag, 2 tag name, 3 value.
369
- _XML_PASSWORD_RE = re.compile(
370
- r"(?i)(<(password|passwd)>)([^<]{4,200})(</\2>)")
371
-
372
- # password = "hunter2" / password: "hunter2" / password "hunter2" ->
373
- # the value becomes [REDACTED] inside its quotes. The separator accepts
374
- # `=`, `:` AND Gradle's bare-space credentials-block form -- which is the
375
- # issue's own headline case and which a `[=:]`-only sketch misses. Quoted
376
- # values only, minimum 4 chars: unquoted .properties-style values are out
377
- # of scope by decision, and the value excludes newlines so a quote further
378
- # down a file cannot close a match opened lines above. Groups: 1 keyword,
379
- # 2 separator, 3 opening quote, 4 value.
380
- _ASSIGNMENT_RE = re.compile(
381
- r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
382
- r"([\"'])([^\"'\n]{4,200})\3")
383
-
384
- # --- Batch 160, TECHNICAL_DEBT 52: three STRUCTURAL shapes, output only by
385
- # --- owner ruling. Each carries a credential with no vendor prefix, and
386
- # --- each anchors on structure a security paper does not produce by
387
- # --- accident, which is the false-positive argument the password shapes
388
- # --- above already make.
389
-
390
- # Authorization: Bearer abc... -> Authorization: Bearer [REDACTED]. Anchored
391
- # on the HEADER NAME, never on the scheme word: "Bearer tokens expire" is
392
- # prose and must survive, and that is the keyword-widening the comment above
393
- # refuses. The separator admits a header line, a JSON or dict key (`"..":`)
394
- # and `=`, so a curl line, a captured request and a logged dict all match.
395
- # The value's alphabet has no `$ < > { } [`, so `Bearer $TOKEN`,
396
- # `Bearer <token>` and an already-redacted value never match, and
397
- # `Digest username=...`, whose secret is not one value, is left alone. A
398
- # value with NO scheme word must also be 16 characters and carry a digit,
399
- # because API documentation writes "Authorization: required" and a bare
400
- # 8-character floor redacted the word.
401
- # Groups: 1 header name, 2 separator, 3 scheme (or None).
402
- _AUTH_HEADER_RE = re.compile(
403
- r"(?i)\b((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
404
- r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
405
- r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
406
-
407
- # A JSON Web Token, whole: three base64url segments, the first two of
408
- # which open with `eyJ` because both are JSON objects (`{"`). A bearer
409
- # credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
410
- # reach when the token is pasted without its header.
411
- _JWT_RE = re.compile(
412
- r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
413
-
414
- # The END of a PEM key block whose BEGIN is not in this string: the footer
415
- # and the run of base64 lines above it. The vendor pattern takes a block
416
- # from its header; this is the other half, for text a cut started in the
417
- # middle of a block -- a page, a session's tail as its start sweeps through
418
- # the output. Found as a literal and walked BACKWARDS in code, because a
419
- # regex that has to start at the first body line is tried at every line
420
- # start of a long base64 blob and is quadratic on one with no footer.
421
- _PEM_END_RE = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
422
- _PEM_BODY_CHARS = frozenset(
423
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
424
-
425
-
426
- def _before_pem_break(text: str, i: int):
427
- """The index before the line break(s) ending at *i*, or None.
428
-
429
- `_PEM_SEP` read backwards: spaces, then one or more breaks (real, or
430
- the JSON escape), each with its own spaces."""
431
- k = i
432
- while k and text[k - 1] in " \t":
433
- k -= 1
434
- seen = False
435
- while True:
436
- if k and text[k - 1] == "\n":
437
- k -= 1
438
- if k and text[k - 1] == "\r":
439
- k -= 1
440
- elif k >= 2 and text[k - 2:k] == "\\n":
441
- k -= 2
442
- if k >= 2 and text[k - 2:k] == "\\r":
443
- k -= 2
444
- else:
445
- break
446
- seen = True
447
- while k and text[k - 1] in " \t":
448
- k -= 1
449
- return k if seen else None
450
-
451
-
452
- def _redact_pem_tails(text: str) -> str:
453
- """Replace each orphan footer, and the base64 lines above it, whole.
454
-
455
- Linear: each walk stops at the first character outside the body
456
- alphabet, and a footer's own `-` is one, so no two walks share text."""
457
- pieces, last = [], 0
458
- for m in _PEM_END_RE.finditer(text):
459
- start = i = m.start()
460
- while True:
461
- k = _before_pem_break(text, i)
462
- if k is None:
463
- break
464
- r = k
465
- while r > last and text[r - 1] in _PEM_BODY_CHARS:
466
- r -= 1
467
- if r == k:
468
- break
469
- start = i = r
470
- pieces.append(text[last:start])
471
- pieces.append(REDACTION_MARKER)
472
- last = m.end()
473
- pieces.append(text[last:])
474
- return "".join(pieces)
475
-
476
-
477
- def _redact_credential_shapes(text: str) -> str:
478
- """Apply the context and structural shapes, preserving surrounding
479
- structure.
480
-
481
- Each replacer checks the captured VALUE against the placeholder shapes
482
- before substituting -- a template reference is exactly what docs and
483
- build files are full of, and redacting it destroys the example while
484
- protecting nothing."""
485
- text = _URL_USERINFO_RE.sub(rf"\1{REDACTION_MARKER}\3", text)
486
-
487
- def _xml(m):
488
- if _TEMPLATE_VALUE_RE.match(m.group(3)):
489
- return m.group(0)
490
- return f"{m.group(1)}{REDACTION_MARKER}{m.group(4)}"
491
-
492
- text = _XML_PASSWORD_RE.sub(_xml, text)
493
-
494
- def _assign(m):
495
- if _TEMPLATE_VALUE_RE.match(m.group(4)):
496
- return m.group(0)
497
- return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
498
- f"{REDACTION_MARKER}{m.group(3)}")
499
-
500
- text = _ASSIGNMENT_RE.sub(_assign, text)
501
-
502
- text = _AUTH_HEADER_RE.sub(
503
- lambda m: (f"{m.group(1)}{m.group(2)}{m.group(3) or ''}"
504
- f"{REDACTION_MARKER}"), text)
505
- text = _JWT_RE.sub(REDACTION_MARKER, text)
506
- return _redact_pem_tails(text)
507
-
508
-
509
- def redaction_enabled() -> bool:
510
- """Whether pattern-based redaction of tool output is active.
511
-
512
- Two switches, one question. config.REDACT_TOOL_OUTPUTS is the
513
- permanent answer; VENASTINE_REDACT_OFF weakens it for one run --
514
- a debugging session that needs the model to see real values --
515
- without editing anything persistent. The environment can only ever
516
- turn redaction OFF, never back on past the constant: an off-by-default
517
- protection an environment variable could silently re-enable would not
518
- be a switch but a coin flip.
519
-
520
- §40 (UN1): BOTH switches are now read from the frozen posture rather
521
- than live, and the environment half is the reason that mattered here.
522
- `config` binds at import so mutating the environment could never move
523
- the constant -- but `os.environ.get(...)` at call time followed a
524
- mutation, so this one function was a live in-process switch for
525
- redaction. The distinction above survives as two posture fields; only
526
- the moment of reading moved.
527
-
528
- What this does NOT govern, whichever way it reads: input refusals
529
- (_input_leaf -- a denial is legible, not destructive), the depth-cap
530
- substitution (fail-closed structural bound), and `redact_diagnostic_text`
531
- -- the harness's own diagnostics, which guard app.log, the transcript,
532
- `MessageLog.error` and a pipeline record regardless of this answer. That
533
- last clause used to name `logging_setup` alone and to be true only about
534
- the moment of reading: the formatter did run regardless, with a strictly
535
- SMALLER vocabulary than this function's, which is the gap batch 144
536
- closed. The two now differ in one respect and it is the one named here.
537
- """
538
- active = posture.current()
539
- if not active.redact_tool_outputs:
540
- return False
541
- return not active.redact_off_env
542
-
543
-
544
- def redact_diagnostic_text(text: str) -> str:
545
- """The one redaction path the HARNESS'S OWN DIAGNOSTICS take.
546
-
547
- THE SAME VOCABULARY AS TOOL OUTPUT, which it was not until batch 144
548
- (batch 144). `redact_secrets` is the seven vendor-token patterns and
549
- nothing else. `_redact_credential_shapes` is the only thing in this
550
- project that matches a credential with NO vendor prefix -- a URL's
551
- userinfo, a `<password>` element, a quoted `password =` assignment --
552
- and it had exactly one caller, `redact_output_text`. So one string took
553
- two different vocabularies depending on which sink it reached. Driven
554
- through the real `configure_logging` with nothing monkeypatched, on
555
- issue #167's own seven cases:
556
-
557
- case app.log transcript tool output
558
- anthropic key redacted redacted redacted
559
- openai key redacted redacted redacted
560
- github pat redacted redacted redacted
561
- aws access key id redacted redacted redacted
562
- gradle password LEAKED LEAKED redacted
563
- maven server password LEAKED LEAKED redacted
564
- gem source url LEAKED LEAKED redacted
565
-
566
- The four vendor rows are the control: they prove the formatter was
567
- attached and firing, so the three LEAKED rows were the vocabulary and
568
- not a probe that had stopped working. It needed no contrivance to
569
- reach: `fetch_url` logs the URL it failed on BY DESIGN and says so in
570
- a comment naming this redaction as the guard, and `core/loop.py` puts
571
- the same description in `MessageLog.error`, where it persists in
572
- `app.db` -- which `docs/PRIVACY.md` says is deleted by wiping the
573
- database rather than one thread.
574
-
575
- FIVE CALLERS, and the one-caller shape is what made the gap
576
- survivable: `logging_setup`'s formatter, `tui/app.py`'s
577
- `TranscriptLogHandler.emit` and its tool-error line, `core/loop.py`'s
578
- `mark_turn_failed`, and the orchestrator's pipeline event. Adding a
579
- pattern or a shape now reaches all five, and
580
- `tests/test_logging_setup.py`'s parity test is what fails if a sixth
581
- sink composes its own.
582
-
583
- UNCONDITIONAL, all three passes, and that is not an extension of the
584
- switch's reach but the removal of a difference. `redaction_enabled()`
585
- governs what the MODEL is shown, which a user may reasonably choose to
586
- see raw for one run; it has never governed what `app.log` keeps across
587
- runs, and the switch's own docstring already said so.
588
-
589
- LIVE VALUES FIRST, which is the finding rather than a preference
590
- (batch 144). `redact_live_values` matches a stored value by exact
591
- substring, so a pattern pass that has already rewritten part of one
592
- leaves the remainder standing. Measured, with a stored value that
593
- properly contains a vendor token: the log's old order returned
594
- `PREFIX-[REDACTED]-SUFFIX` and this one returns `[REDACTED]`. The repo
595
- has locked the general rule three times -- `fetch_url` redacts before
596
- it cuts, `param_digest` redacts before truncating, the diff block runs
597
- on whole texts -- and this is the same rule where the destroyer is a
598
- substitution rather than a cut.
599
-
600
- The cost is measured and is per log RECORD: 6.7 us -> 9.4 us on Linux,
601
- +40%, for three more regex passes over a line that already takes
602
- seven. Invisible at INFO, and under a millisecond per thousand records
603
- at DEBUG.
604
- """
605
- return _redact_credential_shapes(
606
- redact_secrets(harness_secrets.redact_live_values(text)))
607
-
608
-
609
- def redact_output_text(text: str) -> str:
610
- """The one redaction path tool OUTPUT takes, under one switch.
611
-
612
- Vendor tokens first, then credential shapes on what survives. Both are
613
- substitutions of content the user chose to see raw when they flip the
614
- switch off -- so this is where VENASTINE_REDACT_OFF bites. The depth
615
- cap is NOT routed through here: it is structure, not content judgment,
616
- and stays fail-closed unconditionally.
617
-
618
- PUBLIC since batch 41 (X4). It was private while `param_digest` was
619
- its only caller outside this module's own dispatch path, and the §41
620
- diff is the second: a `write` renders the content the tool put on
621
- disk, which is tool output by any reading, and it replaces the very
622
- digest line that goes through here. `redact_secrets` alone would have
623
- redacted LESS than the line it replaced -- the vendor tokens but not
624
- the credential shapes -- which is a regression dressed as a feature.
625
- The docstring above already called this "the one path"; it now is one
626
- for both consumers.
627
-
628
- AND IT IS NOW `redact_diagnostic_text` UNDER THE SWITCH (batch 144),
629
- rather than its own copy of the same three passes. The two had drifted
630
- into different vocabularies AND different orders while reading as
631
- siblings, which is why the difference that is deliberate is now the
632
- only one there is."""
633
- # SS61: THE HARNESS'S OWN SECRETS COME OUT FIRST, AND UNCONDITIONALLY.
634
- # Above the switch on purpose, and still first inside the shared
635
- # composition below. `redact_tool_outputs` governs pattern GUESSES
636
- # about other people's credentials, and a user who turns it off has
637
- # chosen to see those raw. A value this harness put into a subprocess,
638
- # coming back out of one, is a different thing: passing it to the model
639
- # is the harness leaking its OWN secret, and no switch asks for that.
640
- # It sits beside the depth cap, which this module already keeps
641
- # fail-closed for the same shape of reason -- it is structure and
642
- # self-inflicted leakage, not content judgment.
643
- #
644
- # The VALUES never come here: security/secrets.py does the substitution
645
- # itself, so exactly one module holds plaintext and the redactor is not
646
- # it (fix at the producer).
647
- if not redaction_enabled():
648
- return harness_secrets.redact_live_values(text)
649
- return redact_diagnostic_text(text)
650
-
651
-
652
- #: What a credential is spelled with: base64 and base64url, hex, and the
653
- #: separators vendor tokens use. A run of these that a cut splits is the
654
- #: fragment the cut drops, because the redactor runs after the cut and
655
- #: cannot match half a token. Punctuation that no token contains -- quotes,
656
- #: `:`, `@`, braces, commas -- ends a run, so a cut through minified JSON
657
- #: costs one value and not the whole line. Two cuts take it: a shell
658
- #: session's dropped middle (A72) and an MCP result's bound (A79).
659
- TOKEN_CHARS = frozenset(
660
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-+/=.~")
661
-
662
-
663
- def fragment_end(head: str) -> int:
664
- """Where the token-character run that ends *head* begins."""
665
- i = len(head)
666
- while i and head[i - 1] in TOKEN_CHARS:
667
- i -= 1
668
- return i
669
-
670
-
671
- def fragment_start(tail: str) -> int:
672
- """Where the token-character run that starts *tail* ends."""
673
- i = 0
674
- while i < len(tail) and tail[i] in TOKEN_CHARS:
675
- i += 1
676
- return i
677
-
678
-
679
-
680
- # How much of one tool parameter reaches a digest, and how much of the whole
681
- # digest reaches a consumer. A sidebar is 22 columns and a transcript line
682
- # wraps; the point is to say WHICH url or WHICH query, not to reproduce the
683
- # call.
684
- _DIGEST_VALUE_CHARS = 60
685
- _DIGEST_CHARS = 140
686
-
687
- # A VALUE CARRYING A URL GETS ITS OWN, WIDER CAP, and the digest a wider
688
- # total to hold one (batch 96).
689
- #
690
- # Batch 65 made a truncated URL clickable by carrying the whole one
691
- # alongside the line, so the click has always opened the right page. What
692
- # it could not fix is the TERMINAL: `https://host/a/b…` minus the ellipsis
693
- # is itself a syntactically valid URL, so a terminal's own link detector
694
- # sees a shorter address than the harness resolved and ctrl+click opens
695
- # THAT -- measured, and reported as the click going somewhere wrong.
696
- #
697
- # The honest fix is to stop producing the mismatch: draw the whole URL, so
698
- # the visible text is the target again and the two layers cannot disagree.
699
- # `markdown.link_spans`' elision stays for anything past this cap, where
700
- # the invariant is still "the visible text tells you the origin".
701
- #
702
- # BOTH CAPS MOVE OR NEITHER DOES. A 200-character value cut at a
703
- # 140-character TOTAL is the same mismatch one line further down, which is
704
- # why the total is raised only for a digest that actually holds a URL --
705
- # every other digest keeps the width it has always had.
706
- _DIGEST_URL_CHARS = 200
707
- _DIGEST_URL_TOTAL = 240
708
-
709
-
710
- def param_digest(params, omit=()) -> str:
711
- """A short, REDACTED description of what a tool was called with.
712
-
713
- REDACTED BEFORE TRUNCATION, not after. Truncating first can cut a
714
- credential in half and leave the surviving half unmatched by the pattern
715
- that would have caught it -- the redactor works on whole strings, so it
716
- has to see one. That ordering is the whole reason this function is
717
- shared rather than copied: §26 established it for the research view, and
718
- §27's replay renderer is its second caller. A second copy is how the
719
- ordering silently regresses in one of them, which is this project's
720
- canonical bug shape (fix at the producer, not the consumer).
721
-
722
- It lives HERE, beside redact_secrets, rather than in the orchestrator
723
- that first needed it: the rule it enforces is a content-policy rule,
724
- and this module is where post-call content policy lives.
725
-
726
- Tool arguments are the path that genuinely was not scanned: §25 R5 added
727
- check_input_policy(), but that REFUSES a call, it does not redact one,
728
- and dispatch()'s check_output_policy only ever saw results. A fetch_url
729
- whose url carries an API key would otherwise be printed verbatim in two
730
- shells and, since §27, replayed out of the archive into a third place.
731
-
732
- Values only, no keys: for the tools a pass actually reaches for -- one
733
- url, one query, one command -- the key is noise, and per-value
734
- truncation is what keeps a `write` call from pasting a file into the
735
- transcript.
736
-
737
- `omit` NAMES KEYS TO SKIP (§42, RA3), and it is a parameter rather
738
- than a rule because this module must not learn about the registry:
739
- it imports `config` and `security.posture` and nothing else, and
740
- which tool declares which param is the registry's knowledge.
741
- `registry.call_digest()` is the one caller that does the lookup.
742
-
743
- It exists for `shell`'s `rationale`, which is prose written for a
744
- person to read and would otherwise take sixty of the digest's one
745
- hundred and forty characters. Key ORDER is the model's, so it
746
- lands first about half the time: measured, `ls -la dist/` became
747
- `I need to check whether the build artifacts directory still… ls
748
- -la dist/` -- the command no longer the first thing a reader sees.
749
- """
750
- if not isinstance(params, dict) or not params:
751
- return ""
752
- parts = []
753
- widened = False
754
- for key, value in params.items():
755
- if key in omit:
756
- continue
757
- text = value if isinstance(value, str) else json.dumps(value, default=str)
758
- # The same redaction tool output takes -- shapes included (#167),
759
- # so a fetch_url whose url carries userinfo does not print its
760
- # password into the transcript, the CLI and (since §27) the
761
- # replayed archive. Under VENASTINE_REDACT_OFF this shows raw,
762
- # like every other pattern substitution; it stays display-only
763
- # and never logs, whatever the switch reads.
764
- text = redact_output_text(text)
765
- # ONE LINE, and the collapse happens BEFORE the truncation so that
766
- # sixty characters means sixty VISIBLE ones (batch 96). A digest is
767
- # drawn as a single row -- `▸ shell <digest>` -- and nothing
768
- # downstream re-flows it, so a multi-line command (a heredoc, an
769
- # inline `python3 - <<EOF`) drew its remaining lines at column 0
770
- # underneath, outside the indent and outside the tool's line. The
771
- # transcript, `core/replay.py` and the research pass trace all read
772
- # this one producer, so all three drew it.
773
- #
774
- # AFTER the redaction, which is the ordering this function exists
775
- # to hold: the redactor works on whole strings, and collapsing
776
- # first would not change what it matches, but doing it after keeps
777
- # the rule "redact, THEN shorten" true of every step below it.
778
- text = " ".join(text.split())
779
- # `http` is a FILTER, not a grammar: which substrings are URLs is
780
- # `tui/markdown.py`'s question, asked there against one definition.
781
- # This only declines to widen a value that cannot contain one --
782
- # the same test, and the same reasoning, as `redacted_values`.
783
- has_url = "http" in text
784
- widened = widened or has_url
785
- cap = _DIGEST_URL_CHARS if has_url else _DIGEST_VALUE_CHARS
786
- if len(text) > cap:
787
- text = text[:cap - 1] + "…"
788
- parts.append(text)
789
- digest = " ".join(p for p in parts if p)
790
- total = _DIGEST_URL_TOTAL if widened else _DIGEST_CHARS
791
- if len(digest) > total:
792
- digest = digest[:total - 1] + "…"
793
- return digest
794
-
795
-
796
- def redacted_values(params, omit=()) -> tuple:
797
- """The same values `param_digest` summarises, UNTRUNCATED (batch 65).
798
-
799
- The transcript arms a URL in a tool line for ctrl+click, and the
800
- line it draws is a digest: the 60-character value cap above turns
801
- most real documentation URLs into `https://…/pa…`, which cannot be
802
- clicked because it is not the URL. These are the candidates a
803
- truncated span resolves against.
804
-
805
- IT LIVES HERE, beside param_digest, and it is the redaction that
806
- puts it here rather than tidiness. A click hands its target to the
807
- platform's browser, so a target taken from the RAW params would
808
- send the `?api_key=…` this module deliberately kept out of the
809
- transcript -- #167's protection defeated at the one place it was
810
- designed to hold, by a feature whose whole subject is the same
811
- string.
812
-
813
- A VALUE THE REDACTOR CHANGED IS DROPPED ENTIRELY. Redacting and
814
- then offering the result would arm a link to somewhere that does
815
- not exist -- `https://user:[REDACTED]@host/…` is a real URL, it is
816
- simply not the one that was fetched. Refusing the value is the
817
- honest answer: nothing was hidden from the reader, and nothing
818
- invented is offered to the browser. (`markdown.clickable` refuses
819
- that particular shape a second time, at the point of action, for
820
- the reason every check in this project is asked twice.)
821
-
822
- Values only and `omit` skipped, both matching param_digest exactly:
823
- the two functions describe ONE call, and a candidate the line does
824
- not draw is a target nothing can resolve to.
825
-
826
- The `http` test is a filter and not a grammar. Which substrings are
827
- URLs is `tui/markdown.py`'s question and is asked there against one
828
- definition; this only declines to carry values that cannot possibly
829
- contain one, so a `write` call's file content never reaches the
830
- renderer.
831
- """
832
- if not isinstance(params, dict) or not params:
833
- return ()
834
- values = []
835
- for key, value in params.items():
836
- if key in omit:
837
- continue
838
- text = value if isinstance(value, str) \
839
- else json.dumps(value, default=str)
840
- if "http" not in text:
841
- continue
842
- redacted = redact_output_text(text)
843
- if redacted != text:
844
- continue
845
- values.append(redacted)
846
- return tuple(dict.fromkeys(values))
847
-
848
-
849
- # ---------------------------------------------------------------------------
850
- # ---- Shared scan traversal ------------------------------------------------
851
- # ---------------------------------------------------------------------------
852
-
853
- # THERE IS NO KEY ALLOWLIST ANY MORE (#47). There used to be:
854
- #
855
- # _SCANNED_KEYS = ("content", "result", "stdout", "stderr", "error")
856
- #
857
- # and `check_output_policy` tested exact membership. Both search tools
858
- # return their payload under `results` -- plural -- which is not `result`,
859
- # so the output of the two tools whose content is ENTIRELY third-party
860
- # text was the output nothing redacted. Verified end to end: a planted
861
- # key survived into the persisted archive and into what the model was
862
- # sent the next turn.
863
- #
864
- # The allowlist grew correctly each time someone noticed a gap ("error"
865
- # was added because an MCP server reports a failed call in band, often
866
- # quoting the credential that was rejected), and that is the problem: it
867
- # re-created the defect for every tool added after it, and it did so
868
- # SILENTLY. mcp_client/client.py's _normalize was written to satisfy this
869
- # set deliberately, treating membership as a guarantee -- so the contract
870
- # was real and known, and the two oldest built-ins never satisfied it.
871
- #
872
- # Scanning every key is the producer-side fix this project's own
873
- # convention names ("a per-case branch added to a normalizer usually
874
- # means the fix belongs one layer down"). A tool that returns text now
875
- # has that text scanned because it is text, not because someone
876
- # remembered to name its key.
877
-
878
-
879
- # How deep to descend into nested values. Bounded so a pathological or
880
- # hostile structure can't turn a scan into a stack overflow -- this walks
881
- # output from code the user didn't write, and arguments chosen by a model
882
- # that has been reading attacker-controlled web pages.
883
- _MAX_SCAN_DEPTH = 12
884
-
885
-
886
- class _PolicyRefusal(Exception):
887
- """Internal control flow: a leaf handler rejecting what it was given.
888
-
889
- Never escapes this module -- check_input_policy() converts it to a
890
- reason string, and the redaction path's leaf handler never raises.
891
- """
892
-
893
-
894
- def _walk(value, leaf, on_cap, depth: int = 0, scan_keys: bool = False):
895
- """Depth-bounded traversal of a JSON-shaped value.
896
-
897
- ONE traversal, two callers. `leaf` maps a str to its replacement (and
898
- may raise `_PolicyRefusal`); `on_cap` decides what happens when the
899
- depth bound is reached. Redaction and argument scanning differ ONLY in
900
- those two behaviours, and writing the walk twice is how the two drift
901
- -- the §17 hole existed because `structured_content` was nested and
902
- the scan was not, and a second copy would reintroduce exactly that
903
- class of gap on the input side.
904
-
905
- ADDED IN §17 (as `_redact_value`), and it closed a real hole rather
906
- than generalising for its own sake. The original scanned only
907
- TOP-LEVEL strings, which was sufficient while every tool returned
908
- `{"result": "<some string>"}`. MCP breaks that assumption:
909
- `structured_content` is arbitrary JSON from a third-party server, and
910
- it arrives under `result` as a dict -- so the output most likely to
911
- contain a leaked credential was the one output that was never
912
- scanned. Caught by an end-to-end test, not by reading the code.
913
-
914
- Fixed here rather than in mcp_client/, because the producer of the
915
- unscanned shape isn't MCP -- it's any tool returning nested data under
916
- a scanned key, which `discrete_math` and `probability_stats` already
917
- can. A special case at the MCP boundary would have left those.
918
-
919
- scan_keys is False for redaction and True for argument scanning. The
920
- asymmetry is deliberate: rewriting a result's dict KEYS would change
921
- the shape a tool's consumer sees, while a secret sitting in an
922
- argument key (`{"sk-ant-...": "x"}`) is a plausible way to smuggle one
923
- past a values-only scan.
924
- """
925
- if isinstance(value, str):
926
- return leaf(value)
927
- if depth >= _MAX_SCAN_DEPTH:
928
- # Fail CLOSED. Returning the container untouched made the depth
929
- # bound a deterministic bypass: a server placing a credential 13
930
- # levels down under a scanned key passed straight through, and
931
- # third-party structured_content is precisely the threat this
932
- # function's docstring names. The cap exists to stop a hostile
933
- # structure exhausting the stack, so refusing to descend must
934
- # mean refusing to emit, not emitting unchecked.
935
- return on_cap()
936
- if isinstance(value, dict):
937
- return {
938
- (leaf(k) if scan_keys and isinstance(k, str) else k):
939
- _walk(v, leaf, on_cap, depth + 1, scan_keys)
940
- for k, v in value.items()
941
- }
942
- if isinstance(value, (list, tuple)):
943
- return type(value)(
944
- _walk(v, leaf, on_cap, depth + 1, scan_keys) for v in value
945
- )
946
- return value
947
-
948
-
949
- # ---------------------------------------------------------------------------
950
- # ---- Output policy check --------------------------------------------------
951
- # ---------------------------------------------------------------------------
952
-
953
- def check_output_policy(tool_name: str, result: dict) -> dict:
954
- """Apply content-level policy to a tool's output.
955
-
956
- Called once, centrally, by registry.dispatch() after every tool
957
- call — not something individual tool files call themselves.
958
- Mutates *result* in place and returns it.
959
-
960
- EVERY value is scanned, not an allowlist of keys (#47) — see the
961
- note above `_walk`. `scan_keys=False` still, so a result's dict KEYS
962
- are left alone: rewriting them would change the shape a tool's
963
- consumer sees, and a tool result's keys are harness- or
964
- server-declared names rather than free text. That asymmetry with
965
- `check_input_policy` is deliberate and predates this change.
966
-
967
- NOTHING THIS FUNCTION DOES IS SILENT ANY MORE (#49). Every
968
- alteration — a redacted value, a depth-cap substitution — produces
969
- exactly ONE warning naming the tool, what was altered and how many
970
- values were touched (and WHICH top-level key sat over a capped
971
- container). The matched content is never echoed: this function
972
- exists because something credential-shaped was found, so quoting it
973
- to say so would leak it into app.log, the TUI transcript and every
974
- place a WARNING travels — the same reasoning _input_leaf's refusal
975
- already states. Before #49 the depth cap replaced a whole nested
976
- container with a sentence and logged nothing, leaving truncated
977
- output indistinguishable from complete output; `registry.schemas`
978
- states the rule this was violating ("no quiet invisibility").
979
- """
980
- redacted_values = 0
981
- capped_keys = []
982
-
983
- def _leaf(text: str) -> str:
984
- nonlocal redacted_values
985
- replacement = redact_output_text(text)
986
- if replacement != text:
987
- redacted_values += 1
988
- return replacement
989
-
990
- for key in list(result):
991
- hit_cap = []
992
- result[key] = _walk(
993
- result[key],
994
- _leaf,
995
- lambda: (hit_cap.append(key),
996
- "[REDACTED: nested beyond scan depth]")[1],
997
- )
998
- if hit_cap:
999
- capped_keys.append(key)
1000
-
1001
- if redacted_values or capped_keys:
1002
- parts = []
1003
- if redacted_values:
1004
- parts.append(f"{redacted_values} value(s) had credentials "
1005
- f"replaced")
1006
- if capped_keys:
1007
- parts.append("nested content beyond scan depth was substituted "
1008
- "under key(s): " + ", ".join(map(repr, capped_keys)))
1009
- logger.warning("Output policy altered %s result: %s.",
1010
- tool_name, "; ".join(parts))
1011
- return result
1012
-
1013
-
1014
- # ---------------------------------------------------------------------------
1015
- # ---- Input policy check ---------------------------------------------------
1016
- # ---------------------------------------------------------------------------
1017
- #
1018
- # ROADMAP_v2 §25 (R5). The symmetric half of check_output_policy, and it
1019
- # closes two holes that predate the research-pipeline work:
1020
- #
1021
- # 1. Results were scanned for secrets; ARGUMENTS never were. A tool the
1022
- # user approved can be called with text drawn from context, so the
1023
- # argument list is an egress path and nothing was watching it.
1024
- # 2. BLOCKED_DOMAINS was opt-in PER TOOL -- only web_search.py and
1025
- # fetch_url.py import is_domain_blocked(). Any other tool taking a
1026
- # URL (every MCP tool that fetches anything) bypassed the list
1027
- # entirely, which is the `fetch_url` defect shape again: a control
1028
- # that exists, is documented, and does not cover what it claims to.
1029
- #
1030
- # Enforced at dispatch() so it covers every tool in every mode, rather
1031
- # than at the pipeline boundary that prompted it.
1032
-
1033
- # A URL is recognised by its scheme, not by looking like a hostname. A
1034
- # BARE domain is deliberately NOT refused here: this harness researches
1035
- # security topics, so "exploit-db.com" appears in legitimate queries,
1036
- # claims and reports constantly, and refusing every mention would make the
1037
- # blocklist unusable while blocking nothing an attacker couldn't rephrase.
1038
- # fetch_url.py keeps its own bare-domain check for the value it is given.
1039
- _URL_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s\"'<>]+")
1040
-
1041
-
1042
- def _input_leaf(text: str) -> str:
1043
- """Refuse a string carrying a credential or a blocked URL."""
1044
- for pattern in _SECRET_PATTERNS:
1045
- if pattern.search(text):
1046
- # The matched text is NOT echoed. A refusal reason travels
1047
- # into model context, the TUI transcript and the persisted
1048
- # MessageLog -- quoting the credential to explain that a
1049
- # credential leaked would leak it three more places.
1050
- raise _PolicyRefusal(
1051
- "an argument contains what looks like a credential")
1052
- for url in _URL_RE.findall(text):
1053
- if is_domain_blocked(url):
1054
- raise _PolicyRefusal(f"an argument targets a blocked domain: {url}")
1055
- return text
1056
-
1057
-
1058
- def check_input_policy(tool_name: str, params: dict) -> str | None:
1059
- """Content-level policy on a tool's ARGUMENTS. Returns a refusal
1060
- reason, or None to proceed.
1061
-
1062
- Called by registry.dispatch() before the handler runs. REFUSES rather
1063
- than redacting: silently rewriting an argument changes what the call
1064
- does, so a tool would run with parameters neither the model nor the
1065
- user chose and report success. A denial is legible to both -- the same
1066
- reasoning D24 settled for uncallable tools.
1067
- """
1068
- try:
1069
- _walk(
1070
- params,
1071
- _input_leaf,
1072
- lambda: _raise_refusal("arguments nested beyond the scan depth"),
1073
- scan_keys=True,
1074
- )
1075
- except _PolicyRefusal as e:
1076
- return f"{tool_name} call refused by policy: {e}"
1077
- return None
1078
-
1079
-
1080
- def _raise_refusal(reason: str):
1081
- """`lambda: raise` is not valid syntax; this is the on_cap callback."""
1082
- raise _PolicyRefusal(reason)
1
+ """
2
+ safety/policy_enforcement.py
3
+
4
+ Content-level policy, applied AFTER a tool runs, to its output — NOT
5
+ access control (that's security/permissions.py's job, applied BEFORE a
6
+ tool runs, deciding whether it runs at all). registry.py's dispatch()
7
+ calls check_output_policy() on every tool result, regardless of which
8
+ tool produced it.
9
+
10
+ Three responsibilities:
11
+ 1. Centralized URL policy — is_url_permitted() answers "may this URL be
12
+ requested at all", composing the blocked-domain list with a
13
+ non-public-address guard. web_search.py and fetch_url.py both call
14
+ it, so a URL cannot be permitted for one and refused by the other.
15
+ is_domain_blocked() remains the domain half on its own.
16
+ 2. Secret redaction — defense-in-depth: if a tool's output accidentally
17
+ contains something that looks like a leaked API key, it is replaced
18
+ with [REDACTED] before the content reaches storage or the model.
19
+ 3. Argument policy (§25) — the symmetric half of (2), applied BEFORE a
20
+ tool runs, to what it was asked to do rather than to what it returned.
21
+ Refuses rather than redacts, and refuses on the blocked-domain list
22
+ too, which until then only the two tools that opted in enforced.
23
+
24
+ (1) and (3) overlap deliberately: (3) is the choke point every tool
25
+ passes through, (1) is what a tool checks about a value only it can
26
+ interpret. Neither makes the other redundant.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import ipaddress
32
+ import json
33
+ import logging
34
+ import re
35
+ import socket
36
+ from urllib.parse import urlparse
37
+
38
+ # safety -> security, and never the reverse. `security.posture` imports
39
+ # `config` and the stdlib only, so this adds no cycle -- which is exactly
40
+ # why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
41
+ # this module, and the edge only stays acyclic while it runs one way.
42
+ from security import posture
43
+ from security import secrets as harness_secrets
44
+
45
+ logger = logging.getLogger(__name__)
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # ---- Blocked domains ------------------------------------------------------
49
+ # ---------------------------------------------------------------------------
50
+
51
+ # Centralized blocked-domain list. web_search.py and fetch_url.py import
52
+ # is_domain_blocked() to enforce this. Add domains hosting malware,
53
+ # phishing, or other harmful content based on your threat model.
54
+ #
55
+ # MAINTENANCE: domains change ownership, get taken down, or are repurposed.
56
+ # Review this list periodically. Remove domains that are no longer harmful;
57
+ # add new ones as threats emerge. An empty set disables domain filtering.
58
+ BLOCKED_DOMAINS: set[str] = {
59
+ # Well-known harmful domains — examples only, maintain as needed.
60
+ "malware-traffic-analysis.net", # malware sample distribution
61
+ "exploit-db.com", # exploit code repository
62
+ "packetstormsecurity.com", # exploit/tool archive
63
+ }
64
+
65
+
66
+ def _hostname(url: str) -> str:
67
+ """The host of a full URL or a bare domain, normalised for comparison.
68
+
69
+ ONE normaliser, because the two branches below had drifted apart and
70
+ the difference was a bypass (#48). `.hostname` already lowercases,
71
+ strips the port and strips userinfo; the bare-domain branch did none
72
+ of that except by accident, so `exploit-db.com:443` passed it while
73
+ the URL form of the same host was caught.
74
+
75
+ The trailing dot defeated BOTH branches. `exploit-db.com.` is a legal,
76
+ resolvable FQDN naming exactly the same host, `urlparse` keeps the
77
+ dot, and no amount of lowercasing makes the strings equal. Stripped
78
+ here rather than at each call site, since every caller compares.
79
+ """
80
+ if "://" in url:
81
+ try:
82
+ host = urlparse(url).hostname or ""
83
+ except Exception:
84
+ host = url
85
+ else:
86
+ # Strip path, then userinfo, then port -- the three things
87
+ # .hostname does for the URL branch and this one used not to.
88
+ host = url.split("/")[0]
89
+ host = host.rpartition("@")[2]
90
+ if host.startswith("["): # [::1]:8080
91
+ host = host.partition("]")[0].lstrip("[")
92
+ else:
93
+ host = host.partition(":")[0]
94
+ return host.lower().rstrip(".")
95
+
96
+
97
+ def is_domain_blocked(url: str) -> bool:
98
+ """Return True if the URL's host is in, or under, BLOCKED_DOMAINS.
99
+
100
+ Handles full URLs (https://example.com/path) and bare domains
101
+ (example.com), through one normaliser -- see `_hostname`.
102
+
103
+ Subdomains ARE matched: blocking ``evil.com`` also blocks
104
+ ``sub.evil.com``. This reverses a documented carve-out, and the
105
+ reversal is the point. A blocklist an attacker leaves by prepending
106
+ one label is close to decorative, and the old docstring's advice
107
+ ("block ``evil.com`` explicitly if you also want ``sub.evil.com``")
108
+ asked a maintainer to enumerate an infinite set. Matching a suffix
109
+ can only ever block MORE than the list's author wrote down, and this
110
+ list's whole purpose is refusal, so over-blocking is the safe
111
+ direction to be wrong in.
112
+
113
+ The match is label-wise (`host == blocked` or `host` ends with
114
+ `"." + blocked`), never a bare string suffix: `notevil.com` must not
115
+ match `evil.com`.
116
+ """
117
+ if not BLOCKED_DOMAINS:
118
+ return False
119
+ host = _hostname(url)
120
+ return any(host == blocked or host.endswith("." + blocked)
121
+ for blocked in (b.lower().rstrip(".") for b in BLOCKED_DOMAINS))
122
+
123
+
124
+ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
125
+ """Refusal reason for a URL, or None if it may be requested.
126
+
127
+ ONE checker, two callers -- `fetch_url` before every hop it issues,
128
+ and `web_search` over the results it is about to hand the model. They
129
+ were asking different questions about the same policy, which is how
130
+ #53 and #54 stayed open while #48 looked like the whole of it.
131
+
132
+ Three gates, cheapest first:
133
+
134
+ 1. Scheme. http/https only. Moved here from fetch_url so the answer
135
+ does not depend on which caller asked.
136
+ 2. `is_domain_blocked`.
137
+ 3. Address. Rejects anything that is not a GLOBAL address --
138
+ loopback, private, link-local, reserved and multicast in one
139
+ predicate. `169.254.169.254` is the one that matters most: on a
140
+ cloud instance with IMDSv1 it returns role credentials, and the
141
+ route to it is a fetched page saying "for full details see
142
+ <that URL>" to a grounding pass that obliges.
143
+
144
+ `resolve=False` answers from the literal text only -- an IP literal
145
+ is still checked, in every spelling the platform's resolver reads as
146
+ a number (#207: `2130706433` and `0x7f.0.0.1` too, not only dotted
147
+ quads), and a NAME is taken on trust. That is for `web_search`,
148
+ which filters results rather than fetching them: a DNS lookup per
149
+ search hit is real latency for a check whose failure mode is "the
150
+ model is shown a URL that fetch_url will refuse anyway". The caller
151
+ that actually opens a socket always resolves.
152
+
153
+ KNOWN LIMITATION -- DNS rebinding. When we resolve, httpx then
154
+ resolves again independently, so a host whose records change between
155
+ this check and the connect can still be reached. Closing it means
156
+ connecting to the pinned IP and passing the original host in a
157
+ `Host:` header, which on HTTPS fights SNI and certificate
158
+ verification -- a subtly wrong implementation there is a worse hole
159
+ than this one. Recorded rather than silently accepted; the attacker
160
+ needs control of DNS for a host the model was already induced to
161
+ fetch.
162
+ """
163
+ if not (url.startswith("http://") or url.startswith("https://")):
164
+ return "URL must start with http:// or https://"
165
+
166
+ if is_domain_blocked(url):
167
+ return f"Domain is blocked by policy: {url}"
168
+
169
+ host = _hostname(url)
170
+ if not host:
171
+ return f"URL has no host: {url}"
172
+
173
+ try:
174
+ literal = ipaddress.ip_address(host)
175
+ except ValueError:
176
+ literal = None
177
+
178
+ if literal is None and not resolve:
179
+ # #207, batch 213. `ipaddress` reads dotted quads and IPv6 only, so
180
+ # `2130706433`, `0x7f.0.0.1` and `127.1` -- all 127.0.0.1 to the
181
+ # platform's resolver -- were taken on trust as NAMES here, and
182
+ # only fetch_url refused them. AI_NUMERICHOST asks that resolver to
183
+ # read the text as a number and never to look anything up, so it
184
+ # costs no network, which was the whole reason for resolve=False.
185
+ # Measured on Windows and in WSL: the same spellings parse, and a
186
+ # name fails with gaierror.
187
+ try:
188
+ literal = ipaddress.ip_address(socket.getaddrinfo(
189
+ host, None, flags=socket.AI_NUMERICHOST)[0][4][0])
190
+ except (socket.gaierror, ValueError, IndexError):
191
+ literal = None
192
+
193
+ if literal is not None:
194
+ addresses = [literal]
195
+ elif resolve:
196
+ try:
197
+ addresses = [
198
+ ipaddress.ip_address(info[4][0])
199
+ for info in socket.getaddrinfo(host, None)
200
+ ]
201
+ except (socket.gaierror, ValueError) as e:
202
+ # Fail CLOSED. A name we cannot resolve is one we cannot
203
+ # clear, and httpx would only fail on it a moment later --
204
+ # refusing here costs nothing and keeps "unchecked" from
205
+ # ever meaning "allowed".
206
+ return f"Could not resolve host for policy check: {host} ({e})"
207
+ else:
208
+ addresses = []
209
+
210
+ # ANY, not all. A name resolving to one public and one private
211
+ # address is the rebinding shape, and taking the first answer or
212
+ # requiring unanimity would both let it through.
213
+ for address in addresses:
214
+ if not address.is_global:
215
+ return (f"URL resolves to a non-public address, which this "
216
+ f"harness will not fetch: {host} -> {address}")
217
+
218
+ return None
219
+
220
+
221
+ # ---------------------------------------------------------------------------
222
+ # ---- Secret redaction -----------------------------------------------------
223
+ # ---------------------------------------------------------------------------
224
+
225
+ # A PEM body's line break: a real one, or the two-character escape a JSON
226
+ # string carries -- a cloud service-account key file is exactly that shape.
227
+ # Unambiguous on purpose: every repetition holds a mandatory break, so the
228
+ # spaces between two breaks belong to exactly one repetition and a body
229
+ # that fails to close backtracks in linear time, not exponential.
230
+ _PEM_BREAK = r"(?:[ \t]*(?:\r?\n|\\r\\n|\\n))"
231
+ _PEM_SEP = _PEM_BREAK + r"+[ \t]*"
232
+ # One body line: base64, at least 16 characters, so a word of prose after a
233
+ # quoted header is not one; or the two RFC 1421 headers a legacy encrypted
234
+ # key carries before its base64.
235
+ _PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
236
+
237
+ _SECRET_PATTERNS = [
238
+ # OpenAI-style keys
239
+ re.compile(r"sk-[a-zA-Z0-9]{20,}"),
240
+ # Anthropic-style keys
241
+ re.compile(r"sk-ant-[a-zA-Z0-9\-]{20,}"),
242
+ # GitHub personal access tokens
243
+ re.compile(r"ghp_[a-zA-Z0-9]{36}"),
244
+ # AWS access key IDs
245
+ re.compile(r"AKIA[0-9A-Z]{16}"),
246
+ # Google API keys
247
+ re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
248
+ # Slack tokens
249
+ re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
250
+ # PEM private keys: the header, and the WHOLE BLOCK when it is there
251
+ # (TECHNICAL_DEBT 52, batch 160). This used to be the header line alone,
252
+ # so the base64 body -- the key material -- survived verbatim under a
253
+ # `[REDACTED]` that read as the key having been removed. The body is
254
+ # OPTIONAL in the pattern on purpose: `_input_leaf` uses `.search()`,
255
+ # and anything carrying the header still matches, so input refusals are
256
+ # exactly what they were. The body is LINES, each after a line break:
257
+ # a block with its END line goes whole, and one a cut ended goes as far
258
+ # as its lines do. Lines rather than "anything up to END" because prose
259
+ # quotes both markers -- "starts with -----BEGIN ... and ends with
260
+ # -----END ..." -- and a lazy match took the sentence between them.
261
+ # Measured: that sentence now loses its two markers and keeps its words.
262
+ re.compile(
263
+ r"-----BEGIN [A-Z ]*PRIVATE KEY-----"
264
+ r"(?:(?:" + _PEM_SEP + _PEM_UNIT + r")*"
265
+ r"(?:" + _PEM_SEP + r"[A-Za-z0-9+/=]{1,15})?"
266
+ r"(?:" + _PEM_BREAK + r"*[ \t]*)-----END [A-Z ]*PRIVATE KEY-----"
267
+ r"|(?:" + _PEM_SEP + _PEM_UNIT + r")*)"),
268
+ # --- Batch 160, TECHNICAL_DEBT 52: the CURRENT siblings of the vendor
269
+ # --- formats above, each of which the older pattern beside it misses
270
+ # --- by a character class -- a `_` or a second `-` the 2024 shapes
271
+ # --- never had. Measured through the real `redact_output_text` before
272
+ # --- this: every one survived. They refuse inputs too, as `ghp_` and
273
+ # --- `sk-` always have; one vendor list, one vocabulary.
274
+ # GitHub fine-grained PATs, the format `ghp_` is being retired for
275
+ re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
276
+ # GitHub OAuth, user-to-server, server-to-server and refresh tokens
277
+ re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
278
+ # OpenAI project, service-account and admin keys: `sk-proj-...`
279
+ re.compile(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{20,}"),
280
+ # Stripe live secret and restricted keys
281
+ re.compile(r"[sr]k_live_[A-Za-z0-9]{16,}"),
282
+ # npm access tokens
283
+ re.compile(r"npm_[A-Za-z0-9]{36}"),
284
+ # --- ROADMAP_v3 §79 (HD5): the three credentials `/connect` hands the
285
+ # --- harness that no pattern above can see. A sign-in puts them in a
286
+ # --- session the model shares, so they are redacted and refused like
287
+ # --- the rest of the list.
288
+ # Hugging Face user access tokens and OAuth access tokens. An
289
+ # identifier like `hf_hub_download` breaks the run of word characters
290
+ # at its second underscore and is not matched.
291
+ re.compile(r"(?<![A-Za-z0-9])hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
292
+ # OpenRouter keys, which `sk-` above misses at the second hyphen
293
+ re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
294
+ # ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
295
+ # bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
296
+ # that merely ENDS a word like `items-<uuid>`.
297
+ re.compile(r"(?<![A-Za-z0-9])ms-[0-9a-f]{8}-[0-9a-f]{4}"
298
+ r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
299
+ ]
300
+
301
+
302
+ #: What this module puts where a secret was. Named in batch 65 rather
303
+ #: than written out at each of the four substitutions, because a second
304
+ #: module now has to RECOGNISE it: `tui/markdown.py` refuses to arm a
305
+ #: URL wearing this mark for a click, since a redacted address is a real
306
+ #: one and simply not the one that was fetched. That module is pure and
307
+ #: may not import policy, so it keeps its own copy and
308
+ #: `tests/test_markdown_render.py` holds the two against each other --
309
+ #: one string, two readers, and a check rather than a hope.
310
+ REDACTION_MARKER = "[REDACTED]"
311
+
312
+
313
+ def redact_secrets(text: str) -> str:
314
+ """Replace strings matching known secret patterns with [REDACTED]."""
315
+ for pattern in _SECRET_PATTERNS:
316
+ text = pattern.sub(REDACTION_MARKER, text)
317
+ return text
318
+
319
+
320
+ # ---------------------------------------------------------------------------
321
+ # ---- Credential shapes (#167, batch 20) ------------------------------------
322
+ # ---------------------------------------------------------------------------
323
+ #
324
+ # The patterns above are VENDOR TOKENS -- every one has a prefix some
325
+ # company mints, so a match is high-confidence and a bare `[REDACTED]` loses
326
+ # nothing structural. A credential that is just a password in a field has no
327
+ # prefix, and nothing matched it (#167): batch 14 added five build manifests
328
+ # to read_project_doc's allowlist (#94), whose credential conventions are
329
+ # exactly the three unscrubbed shapes below.
330
+ #
331
+ # These are CONTEXT shapes, not tokens: each anchors on a credential FIELD
332
+ # NAME (or URL userinfo), so an auth error code, a stack frame or an ARN
333
+ # does not match, and the replacement is SURGICAL -- tag names, field names,
334
+ # usernames and hosts survive, only the value goes. That structure is why
335
+ # the false-positive cost is acceptable: documentation quoting an example
336
+ # keeps its shape and loses only its example value.
337
+ #
338
+ # OUTPUT REDACTION ONLY. These never feed check_input_policy: refusing a
339
+ # call because its arguments mention `password = "..."` would break every
340
+ # legitimate call that carries build-file content the user asked for --
341
+ # _SECRET_PATTERNS stay the input side's whole vocabulary. And the keyword
342
+ # set is password|passwd ONLY, deliberately not token/secret/key: those
343
+ # appear constantly in ordinary prose ("token bucket", JWT discussion) and
344
+ # would redact aggressively.
345
+
346
+ # Template placeholders survive untouched: `${PASSWORD}`, `$DB_PASS`,
347
+ # `{{ secrets.DB_PW }}`, `<your-password-here>` name where a credential
348
+ # WOULD go; they are what docs and templates contain, and redacting them
349
+ # destroys the example while protecting nothing.
350
+ _TEMPLATE_VALUE_RE = re.compile(
351
+ r"^(?:"
352
+ r"\$\{[^}]*\}" # ${PASSWORD}, ${secrets.DB}
353
+ r"|\$[A-Za-z_][A-Za-z0-9_]*" # $PASSWORD
354
+ r"|\{\{[^}]*\}\}" # {{ secrets.DB_PW }}
355
+ r"|<[^<>]+>" # <your-password-here>
356
+ r")$")
357
+
358
+ # scheme://user:secret@host -> scheme://user:[REDACTED]@host. Credentials
359
+ # in a URL's userinfo are never anything else, which makes this the cheapest
360
+ # of the three; anchoring on `//` keeps bare `user:pass@host` fragments in
361
+ # prose alone, and excluding `/ \s : @` from both sides means a host:port
362
+ # path (`https://example.com:8080/x`) cannot masquerade as userinfo.
363
+ _URL_USERINFO_RE = re.compile(r"(//[^/\s:@]+:)([^/\s@]+)(@)")
364
+
365
+ # <password>s3cr3t</password> -> <password>[REDACTED]</password>, POM and
366
+ # settings.xml convention. The backreference keeps the closing tag honest;
367
+ # the value excludes `<`, so nested markup cannot be swallowed. Groups:
368
+ # 1 opening tag, 2 tag name, 3 value.
369
+ _XML_PASSWORD_RE = re.compile(
370
+ r"(?i)(<(password|passwd)>)([^<]{4,200})(</\2>)")
371
+
372
+ # password = "hunter2" / password: "hunter2" / password "hunter2" ->
373
+ # the value becomes [REDACTED] inside its quotes. The separator accepts
374
+ # `=`, `:` AND Gradle's bare-space credentials-block form -- which is the
375
+ # issue's own headline case and which a `[=:]`-only sketch misses. Quoted
376
+ # values only, minimum 4 chars: unquoted .properties-style values are out
377
+ # of scope by decision, and the value excludes newlines so a quote further
378
+ # down a file cannot close a match opened lines above. Groups: 1 keyword,
379
+ # 2 separator, 3 opening quote, 4 value.
380
+ _ASSIGNMENT_RE = re.compile(
381
+ r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
382
+ r"([\"'])([^\"'\n]{4,200})\3")
383
+
384
+ # --- Batch 160, TECHNICAL_DEBT 52: three STRUCTURAL shapes, output only by
385
+ # --- owner ruling. Each carries a credential with no vendor prefix, and
386
+ # --- each anchors on structure a security paper does not produce by
387
+ # --- accident, which is the false-positive argument the password shapes
388
+ # --- above already make.
389
+
390
+ # Authorization: Bearer abc... -> Authorization: Bearer [REDACTED]. Anchored
391
+ # on the HEADER NAME, never on the scheme word: "Bearer tokens expire" is
392
+ # prose and must survive, and that is the keyword-widening the comment above
393
+ # refuses. The separator admits a header line, a JSON or dict key (`"..":`)
394
+ # and `=`, so a curl line, a captured request and a logged dict all match.
395
+ # The value's alphabet has no `$ < > { } [`, so `Bearer $TOKEN`,
396
+ # `Bearer <token>` and an already-redacted value never match, and
397
+ # `Digest username=...`, whose secret is not one value, is left alone. A
398
+ # value with NO scheme word must also be 16 characters and carry a digit,
399
+ # because API documentation writes "Authorization: required" and a bare
400
+ # 8-character floor redacted the word.
401
+ # Groups: 1 header name, 2 separator, 3 scheme (or None).
402
+ _AUTH_HEADER_RE = re.compile(
403
+ r"(?i)\b((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
404
+ r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
405
+ r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
406
+
407
+ # A JSON Web Token, whole: three base64url segments, the first two of
408
+ # which open with `eyJ` because both are JSON objects (`{"`). A bearer
409
+ # credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
410
+ # reach when the token is pasted without its header.
411
+ _JWT_RE = re.compile(
412
+ r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
413
+
414
+ # The END of a PEM key block whose BEGIN is not in this string: the footer
415
+ # and the run of base64 lines above it. The vendor pattern takes a block
416
+ # from its header; this is the other half, for text a cut started in the
417
+ # middle of a block -- a page, a session's tail as its start sweeps through
418
+ # the output. Found as a literal and walked BACKWARDS in code, because a
419
+ # regex that has to start at the first body line is tried at every line
420
+ # start of a long base64 blob and is quadratic on one with no footer.
421
+ _PEM_END_RE = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
422
+ _PEM_BODY_CHARS = frozenset(
423
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
424
+
425
+
426
+ def _before_pem_break(text: str, i: int):
427
+ """The index before the line break(s) ending at *i*, or None.
428
+
429
+ `_PEM_SEP` read backwards: spaces, then one or more breaks (real, or
430
+ the JSON escape), each with its own spaces."""
431
+ k = i
432
+ while k and text[k - 1] in " \t":
433
+ k -= 1
434
+ seen = False
435
+ while True:
436
+ if k and text[k - 1] == "\n":
437
+ k -= 1
438
+ if k and text[k - 1] == "\r":
439
+ k -= 1
440
+ elif k >= 2 and text[k - 2:k] == "\\n":
441
+ k -= 2
442
+ if k >= 2 and text[k - 2:k] == "\\r":
443
+ k -= 2
444
+ else:
445
+ break
446
+ seen = True
447
+ while k and text[k - 1] in " \t":
448
+ k -= 1
449
+ return k if seen else None
450
+
451
+
452
+ def _redact_pem_tails(text: str) -> str:
453
+ """Replace each orphan footer, and the base64 lines above it, whole.
454
+
455
+ Linear: each walk stops at the first character outside the body
456
+ alphabet, and a footer's own `-` is one, so no two walks share text."""
457
+ pieces, last = [], 0
458
+ for m in _PEM_END_RE.finditer(text):
459
+ start = i = m.start()
460
+ while True:
461
+ k = _before_pem_break(text, i)
462
+ if k is None:
463
+ break
464
+ r = k
465
+ while r > last and text[r - 1] in _PEM_BODY_CHARS:
466
+ r -= 1
467
+ if r == k:
468
+ break
469
+ start = i = r
470
+ pieces.append(text[last:start])
471
+ pieces.append(REDACTION_MARKER)
472
+ last = m.end()
473
+ pieces.append(text[last:])
474
+ return "".join(pieces)
475
+
476
+
477
+ def _redact_credential_shapes(text: str) -> str:
478
+ """Apply the context and structural shapes, preserving surrounding
479
+ structure.
480
+
481
+ Each replacer checks the captured VALUE against the placeholder shapes
482
+ before substituting -- a template reference is exactly what docs and
483
+ build files are full of, and redacting it destroys the example while
484
+ protecting nothing."""
485
+ text = _URL_USERINFO_RE.sub(rf"\1{REDACTION_MARKER}\3", text)
486
+
487
+ def _xml(m):
488
+ if _TEMPLATE_VALUE_RE.match(m.group(3)):
489
+ return m.group(0)
490
+ return f"{m.group(1)}{REDACTION_MARKER}{m.group(4)}"
491
+
492
+ text = _XML_PASSWORD_RE.sub(_xml, text)
493
+
494
+ def _assign(m):
495
+ if _TEMPLATE_VALUE_RE.match(m.group(4)):
496
+ return m.group(0)
497
+ return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
498
+ f"{REDACTION_MARKER}{m.group(3)}")
499
+
500
+ text = _ASSIGNMENT_RE.sub(_assign, text)
501
+
502
+ text = _AUTH_HEADER_RE.sub(
503
+ lambda m: (f"{m.group(1)}{m.group(2)}{m.group(3) or ''}"
504
+ f"{REDACTION_MARKER}"), text)
505
+ text = _JWT_RE.sub(REDACTION_MARKER, text)
506
+ return _redact_pem_tails(text)
507
+
508
+
509
+ def redaction_enabled() -> bool:
510
+ """Whether pattern-based redaction of tool output is active.
511
+
512
+ Two switches, one question. config.REDACT_TOOL_OUTPUTS is the
513
+ permanent answer; VENASTINE_REDACT_OFF weakens it for one run --
514
+ a debugging session that needs the model to see real values --
515
+ without editing anything persistent. The environment can only ever
516
+ turn redaction OFF, never back on past the constant: an off-by-default
517
+ protection an environment variable could silently re-enable would not
518
+ be a switch but a coin flip.
519
+
520
+ §40 (UN1): BOTH switches are now read from the frozen posture rather
521
+ than live, and the environment half is the reason that mattered here.
522
+ `config` binds at import so mutating the environment could never move
523
+ the constant -- but `os.environ.get(...)` at call time followed a
524
+ mutation, so this one function was a live in-process switch for
525
+ redaction. The distinction above survives as two posture fields; only
526
+ the moment of reading moved.
527
+
528
+ What this does NOT govern, whichever way it reads: input refusals
529
+ (_input_leaf -- a denial is legible, not destructive), the depth-cap
530
+ substitution (fail-closed structural bound), and `redact_diagnostic_text`
531
+ -- the harness's own diagnostics, which guard app.log, the transcript,
532
+ `MessageLog.error` and a pipeline record regardless of this answer. That
533
+ last clause used to name `logging_setup` alone and to be true only about
534
+ the moment of reading: the formatter did run regardless, with a strictly
535
+ SMALLER vocabulary than this function's, which is the gap batch 144
536
+ closed. The two now differ in one respect and it is the one named here.
537
+ """
538
+ active = posture.current()
539
+ if not active.redact_tool_outputs:
540
+ return False
541
+ return not active.redact_off_env
542
+
543
+
544
+ def redact_diagnostic_text(text: str) -> str:
545
+ """The one redaction path the HARNESS'S OWN DIAGNOSTICS take.
546
+
547
+ THE SAME VOCABULARY AS TOOL OUTPUT, which it was not until batch 144
548
+ (batch 144). `redact_secrets` is the seven vendor-token patterns and
549
+ nothing else. `_redact_credential_shapes` is the only thing in this
550
+ project that matches a credential with NO vendor prefix -- a URL's
551
+ userinfo, a `<password>` element, a quoted `password =` assignment --
552
+ and it had exactly one caller, `redact_output_text`. So one string took
553
+ two different vocabularies depending on which sink it reached. Driven
554
+ through the real `configure_logging` with nothing monkeypatched, on
555
+ issue #167's own seven cases:
556
+
557
+ case app.log transcript tool output
558
+ anthropic key redacted redacted redacted
559
+ openai key redacted redacted redacted
560
+ github pat redacted redacted redacted
561
+ aws access key id redacted redacted redacted
562
+ gradle password LEAKED LEAKED redacted
563
+ maven server password LEAKED LEAKED redacted
564
+ gem source url LEAKED LEAKED redacted
565
+
566
+ The four vendor rows are the control: they prove the formatter was
567
+ attached and firing, so the three LEAKED rows were the vocabulary and
568
+ not a probe that had stopped working. It needed no contrivance to
569
+ reach: `fetch_url` logs the URL it failed on BY DESIGN and says so in
570
+ a comment naming this redaction as the guard, and `core/loop.py` puts
571
+ the same description in `MessageLog.error`, where it persists in
572
+ `app.db` -- which `docs/PRIVACY.md` says is deleted by wiping the
573
+ database rather than one thread.
574
+
575
+ FIVE CALLERS, and the one-caller shape is what made the gap
576
+ survivable: `logging_setup`'s formatter, `tui/app.py`'s
577
+ `TranscriptLogHandler.emit` and its tool-error line, `core/loop.py`'s
578
+ `mark_turn_failed`, and the orchestrator's pipeline event. Adding a
579
+ pattern or a shape now reaches all five, and
580
+ `tests/test_logging_setup.py`'s parity test is what fails if a sixth
581
+ sink composes its own.
582
+
583
+ UNCONDITIONAL, all three passes, and that is not an extension of the
584
+ switch's reach but the removal of a difference. `redaction_enabled()`
585
+ governs what the MODEL is shown, which a user may reasonably choose to
586
+ see raw for one run; it has never governed what `app.log` keeps across
587
+ runs, and the switch's own docstring already said so.
588
+
589
+ LIVE VALUES FIRST, which is the finding rather than a preference
590
+ (batch 144). `redact_live_values` matches a stored value by exact
591
+ substring, so a pattern pass that has already rewritten part of one
592
+ leaves the remainder standing. Measured, with a stored value that
593
+ properly contains a vendor token: the log's old order returned
594
+ `PREFIX-[REDACTED]-SUFFIX` and this one returns `[REDACTED]`. The repo
595
+ has locked the general rule three times -- `fetch_url` redacts before
596
+ it cuts, `param_digest` redacts before truncating, the diff block runs
597
+ on whole texts -- and this is the same rule where the destroyer is a
598
+ substitution rather than a cut.
599
+
600
+ The cost is measured and is per log RECORD: 6.7 us -> 9.4 us on Linux,
601
+ +40%, for three more regex passes over a line that already takes
602
+ seven. Invisible at INFO, and under a millisecond per thousand records
603
+ at DEBUG.
604
+ """
605
+ return _redact_credential_shapes(
606
+ redact_secrets(harness_secrets.redact_live_values(text)))
607
+
608
+
609
+ def redact_output_text(text: str) -> str:
610
+ """The one redaction path tool OUTPUT takes, under one switch.
611
+
612
+ Vendor tokens first, then credential shapes on what survives. Both are
613
+ substitutions of content the user chose to see raw when they flip the
614
+ switch off -- so this is where VENASTINE_REDACT_OFF bites. The depth
615
+ cap is NOT routed through here: it is structure, not content judgment,
616
+ and stays fail-closed unconditionally.
617
+
618
+ PUBLIC since batch 41 (X4). It was private while `param_digest` was
619
+ its only caller outside this module's own dispatch path, and the §41
620
+ diff is the second: a `write` renders the content the tool put on
621
+ disk, which is tool output by any reading, and it replaces the very
622
+ digest line that goes through here. `redact_secrets` alone would have
623
+ redacted LESS than the line it replaced -- the vendor tokens but not
624
+ the credential shapes -- which is a regression dressed as a feature.
625
+ The docstring above already called this "the one path"; it now is one
626
+ for both consumers.
627
+
628
+ AND IT IS NOW `redact_diagnostic_text` UNDER THE SWITCH (batch 144),
629
+ rather than its own copy of the same three passes. The two had drifted
630
+ into different vocabularies AND different orders while reading as
631
+ siblings, which is why the difference that is deliberate is now the
632
+ only one there is."""
633
+ # SS61: THE HARNESS'S OWN SECRETS COME OUT FIRST, AND UNCONDITIONALLY.
634
+ # Above the switch on purpose, and still first inside the shared
635
+ # composition below. `redact_tool_outputs` governs pattern GUESSES
636
+ # about other people's credentials, and a user who turns it off has
637
+ # chosen to see those raw. A value this harness put into a subprocess,
638
+ # coming back out of one, is a different thing: passing it to the model
639
+ # is the harness leaking its OWN secret, and no switch asks for that.
640
+ # It sits beside the depth cap, which this module already keeps
641
+ # fail-closed for the same shape of reason -- it is structure and
642
+ # self-inflicted leakage, not content judgment.
643
+ #
644
+ # The VALUES never come here: security/secrets.py does the substitution
645
+ # itself, so exactly one module holds plaintext and the redactor is not
646
+ # it (fix at the producer).
647
+ if not redaction_enabled():
648
+ return harness_secrets.redact_live_values(text)
649
+ return redact_diagnostic_text(text)
650
+
651
+
652
+ #: What a credential is spelled with: base64 and base64url, hex, and the
653
+ #: separators vendor tokens use. A run of these that a cut splits is the
654
+ #: fragment the cut drops, because the redactor runs after the cut and
655
+ #: cannot match half a token. Punctuation that no token contains -- quotes,
656
+ #: `:`, `@`, braces, commas -- ends a run, so a cut through minified JSON
657
+ #: costs one value and not the whole line. Two cuts take it: a shell
658
+ #: session's dropped middle (A72) and an MCP result's bound (A79).
659
+ TOKEN_CHARS = frozenset(
660
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-+/=.~")
661
+
662
+
663
+ def fragment_end(head: str) -> int:
664
+ """Where the token-character run that ends *head* begins."""
665
+ i = len(head)
666
+ while i and head[i - 1] in TOKEN_CHARS:
667
+ i -= 1
668
+ return i
669
+
670
+
671
+ def fragment_start(tail: str) -> int:
672
+ """Where the token-character run that starts *tail* ends."""
673
+ i = 0
674
+ while i < len(tail) and tail[i] in TOKEN_CHARS:
675
+ i += 1
676
+ return i
677
+
678
+
679
+
680
+ # How much of one tool parameter reaches a digest, and how much of the whole
681
+ # digest reaches a consumer. A sidebar is 22 columns and a transcript line
682
+ # wraps; the point is to say WHICH url or WHICH query, not to reproduce the
683
+ # call.
684
+ _DIGEST_VALUE_CHARS = 60
685
+ _DIGEST_CHARS = 140
686
+
687
+ # A VALUE CARRYING A URL GETS ITS OWN, WIDER CAP, and the digest a wider
688
+ # total to hold one (batch 96).
689
+ #
690
+ # Batch 65 made a truncated URL clickable by carrying the whole one
691
+ # alongside the line, so the click has always opened the right page. What
692
+ # it could not fix is the TERMINAL: `https://host/a/b…` minus the ellipsis
693
+ # is itself a syntactically valid URL, so a terminal's own link detector
694
+ # sees a shorter address than the harness resolved and ctrl+click opens
695
+ # THAT -- measured, and reported as the click going somewhere wrong.
696
+ #
697
+ # The honest fix is to stop producing the mismatch: draw the whole URL, so
698
+ # the visible text is the target again and the two layers cannot disagree.
699
+ # `markdown.link_spans`' elision stays for anything past this cap, where
700
+ # the invariant is still "the visible text tells you the origin".
701
+ #
702
+ # BOTH CAPS MOVE OR NEITHER DOES. A 200-character value cut at a
703
+ # 140-character TOTAL is the same mismatch one line further down, which is
704
+ # why the total is raised only for a digest that actually holds a URL --
705
+ # every other digest keeps the width it has always had.
706
+ _DIGEST_URL_CHARS = 200
707
+ _DIGEST_URL_TOTAL = 240
708
+
709
+
710
+ def param_digest(params, omit=()) -> str:
711
+ """A short, REDACTED description of what a tool was called with.
712
+
713
+ REDACTED BEFORE TRUNCATION, not after. Truncating first can cut a
714
+ credential in half and leave the surviving half unmatched by the pattern
715
+ that would have caught it -- the redactor works on whole strings, so it
716
+ has to see one. That ordering is the whole reason this function is
717
+ shared rather than copied: §26 established it for the research view, and
718
+ §27's replay renderer is its second caller. A second copy is how the
719
+ ordering silently regresses in one of them, which is this project's
720
+ canonical bug shape (fix at the producer, not the consumer).
721
+
722
+ It lives HERE, beside redact_secrets, rather than in the orchestrator
723
+ that first needed it: the rule it enforces is a content-policy rule,
724
+ and this module is where post-call content policy lives.
725
+
726
+ Tool arguments are the path that genuinely was not scanned: §25 R5 added
727
+ check_input_policy(), but that REFUSES a call, it does not redact one,
728
+ and dispatch()'s check_output_policy only ever saw results. A fetch_url
729
+ whose url carries an API key would otherwise be printed verbatim in two
730
+ shells and, since §27, replayed out of the archive into a third place.
731
+
732
+ Values only, no keys: for the tools a pass actually reaches for -- one
733
+ url, one query, one command -- the key is noise, and per-value
734
+ truncation is what keeps a `write` call from pasting a file into the
735
+ transcript.
736
+
737
+ `omit` NAMES KEYS TO SKIP (§42, RA3), and it is a parameter rather
738
+ than a rule because this module must not learn about the registry:
739
+ it imports `config` and `security.posture` and nothing else, and
740
+ which tool declares which param is the registry's knowledge.
741
+ `registry.call_digest()` is the one caller that does the lookup.
742
+
743
+ It exists for `shell`'s `rationale`, which is prose written for a
744
+ person to read and would otherwise take sixty of the digest's one
745
+ hundred and forty characters. Key ORDER is the model's, so it
746
+ lands first about half the time: measured, `ls -la dist/` became
747
+ `I need to check whether the build artifacts directory still… ls
748
+ -la dist/` -- the command no longer the first thing a reader sees.
749
+ """
750
+ if not isinstance(params, dict) or not params:
751
+ return ""
752
+ parts = []
753
+ widened = False
754
+ for key, value in params.items():
755
+ if key in omit:
756
+ continue
757
+ text = value if isinstance(value, str) else json.dumps(value, default=str)
758
+ # The same redaction tool output takes -- shapes included (#167),
759
+ # so a fetch_url whose url carries userinfo does not print its
760
+ # password into the transcript, the CLI and (since §27) the
761
+ # replayed archive. Under VENASTINE_REDACT_OFF this shows raw,
762
+ # like every other pattern substitution; it stays display-only
763
+ # and never logs, whatever the switch reads.
764
+ text = redact_output_text(text)
765
+ # ONE LINE, and the collapse happens BEFORE the truncation so that
766
+ # sixty characters means sixty VISIBLE ones (batch 96). A digest is
767
+ # drawn as a single row -- `▸ shell <digest>` -- and nothing
768
+ # downstream re-flows it, so a multi-line command (a heredoc, an
769
+ # inline `python3 - <<EOF`) drew its remaining lines at column 0
770
+ # underneath, outside the indent and outside the tool's line. The
771
+ # transcript, `core/replay.py` and the research pass trace all read
772
+ # this one producer, so all three drew it.
773
+ #
774
+ # AFTER the redaction, which is the ordering this function exists
775
+ # to hold: the redactor works on whole strings, and collapsing
776
+ # first would not change what it matches, but doing it after keeps
777
+ # the rule "redact, THEN shorten" true of every step below it.
778
+ text = " ".join(text.split())
779
+ # `http` is a FILTER, not a grammar: which substrings are URLs is
780
+ # `tui/markdown.py`'s question, asked there against one definition.
781
+ # This only declines to widen a value that cannot contain one --
782
+ # the same test, and the same reasoning, as `redacted_values`.
783
+ has_url = "http" in text
784
+ widened = widened or has_url
785
+ cap = _DIGEST_URL_CHARS if has_url else _DIGEST_VALUE_CHARS
786
+ if len(text) > cap:
787
+ text = text[:cap - 1] + "…"
788
+ parts.append(text)
789
+ digest = " ".join(p for p in parts if p)
790
+ total = _DIGEST_URL_TOTAL if widened else _DIGEST_CHARS
791
+ if len(digest) > total:
792
+ digest = digest[:total - 1] + "…"
793
+ return digest
794
+
795
+
796
+ def redacted_values(params, omit=()) -> tuple:
797
+ """The same values `param_digest` summarises, UNTRUNCATED (batch 65).
798
+
799
+ The transcript arms a URL in a tool line for ctrl+click, and the
800
+ line it draws is a digest: the 60-character value cap above turns
801
+ most real documentation URLs into `https://…/pa…`, which cannot be
802
+ clicked because it is not the URL. These are the candidates a
803
+ truncated span resolves against.
804
+
805
+ IT LIVES HERE, beside param_digest, and it is the redaction that
806
+ puts it here rather than tidiness. A click hands its target to the
807
+ platform's browser, so a target taken from the RAW params would
808
+ send the `?api_key=…` this module deliberately kept out of the
809
+ transcript -- #167's protection defeated at the one place it was
810
+ designed to hold, by a feature whose whole subject is the same
811
+ string.
812
+
813
+ A VALUE THE REDACTOR CHANGED IS DROPPED ENTIRELY. Redacting and
814
+ then offering the result would arm a link to somewhere that does
815
+ not exist -- `https://user:[REDACTED]@host/…` is a real URL, it is
816
+ simply not the one that was fetched. Refusing the value is the
817
+ honest answer: nothing was hidden from the reader, and nothing
818
+ invented is offered to the browser. (`markdown.clickable` refuses
819
+ that particular shape a second time, at the point of action, for
820
+ the reason every check in this project is asked twice.)
821
+
822
+ Values only and `omit` skipped, both matching param_digest exactly:
823
+ the two functions describe ONE call, and a candidate the line does
824
+ not draw is a target nothing can resolve to.
825
+
826
+ The `http` test is a filter and not a grammar. Which substrings are
827
+ URLs is `tui/markdown.py`'s question and is asked there against one
828
+ definition; this only declines to carry values that cannot possibly
829
+ contain one, so a `write` call's file content never reaches the
830
+ renderer.
831
+ """
832
+ if not isinstance(params, dict) or not params:
833
+ return ()
834
+ values = []
835
+ for key, value in params.items():
836
+ if key in omit:
837
+ continue
838
+ text = value if isinstance(value, str) \
839
+ else json.dumps(value, default=str)
840
+ if "http" not in text:
841
+ continue
842
+ redacted = redact_output_text(text)
843
+ if redacted != text:
844
+ continue
845
+ values.append(redacted)
846
+ return tuple(dict.fromkeys(values))
847
+
848
+
849
+ # ---------------------------------------------------------------------------
850
+ # ---- Shared scan traversal ------------------------------------------------
851
+ # ---------------------------------------------------------------------------
852
+
853
+ # THERE IS NO KEY ALLOWLIST ANY MORE (#47). There used to be:
854
+ #
855
+ # _SCANNED_KEYS = ("content", "result", "stdout", "stderr", "error")
856
+ #
857
+ # and `check_output_policy` tested exact membership. Both search tools
858
+ # return their payload under `results` -- plural -- which is not `result`,
859
+ # so the output of the two tools whose content is ENTIRELY third-party
860
+ # text was the output nothing redacted. Verified end to end: a planted
861
+ # key survived into the persisted archive and into what the model was
862
+ # sent the next turn.
863
+ #
864
+ # The allowlist grew correctly each time someone noticed a gap ("error"
865
+ # was added because an MCP server reports a failed call in band, often
866
+ # quoting the credential that was rejected), and that is the problem: it
867
+ # re-created the defect for every tool added after it, and it did so
868
+ # SILENTLY. mcp_client/client.py's _normalize was written to satisfy this
869
+ # set deliberately, treating membership as a guarantee -- so the contract
870
+ # was real and known, and the two oldest built-ins never satisfied it.
871
+ #
872
+ # Scanning every key is the producer-side fix this project's own
873
+ # convention names ("a per-case branch added to a normalizer usually
874
+ # means the fix belongs one layer down"). A tool that returns text now
875
+ # has that text scanned because it is text, not because someone
876
+ # remembered to name its key.
877
+
878
+
879
+ # How deep to descend into nested values. Bounded so a pathological or
880
+ # hostile structure can't turn a scan into a stack overflow -- this walks
881
+ # output from code the user didn't write, and arguments chosen by a model
882
+ # that has been reading attacker-controlled web pages.
883
+ _MAX_SCAN_DEPTH = 12
884
+
885
+
886
+ class _PolicyRefusal(Exception):
887
+ """Internal control flow: a leaf handler rejecting what it was given.
888
+
889
+ Never escapes this module -- check_input_policy() converts it to a
890
+ reason string, and the redaction path's leaf handler never raises.
891
+ """
892
+
893
+
894
+ def _walk(value, leaf, on_cap, depth: int = 0, scan_keys: bool = False):
895
+ """Depth-bounded traversal of a JSON-shaped value.
896
+
897
+ ONE traversal, two callers. `leaf` maps a str to its replacement (and
898
+ may raise `_PolicyRefusal`); `on_cap` decides what happens when the
899
+ depth bound is reached. Redaction and argument scanning differ ONLY in
900
+ those two behaviours, and writing the walk twice is how the two drift
901
+ -- the §17 hole existed because `structured_content` was nested and
902
+ the scan was not, and a second copy would reintroduce exactly that
903
+ class of gap on the input side.
904
+
905
+ ADDED IN §17 (as `_redact_value`), and it closed a real hole rather
906
+ than generalising for its own sake. The original scanned only
907
+ TOP-LEVEL strings, which was sufficient while every tool returned
908
+ `{"result": "<some string>"}`. MCP breaks that assumption:
909
+ `structured_content` is arbitrary JSON from a third-party server, and
910
+ it arrives under `result` as a dict -- so the output most likely to
911
+ contain a leaked credential was the one output that was never
912
+ scanned. Caught by an end-to-end test, not by reading the code.
913
+
914
+ Fixed here rather than in mcp_client/, because the producer of the
915
+ unscanned shape isn't MCP -- it's any tool returning nested data under
916
+ a scanned key, which `discrete_math` and `probability_stats` already
917
+ can. A special case at the MCP boundary would have left those.
918
+
919
+ scan_keys is False for redaction and True for argument scanning. The
920
+ asymmetry is deliberate: rewriting a result's dict KEYS would change
921
+ the shape a tool's consumer sees, while a secret sitting in an
922
+ argument key (`{"sk-ant-...": "x"}`) is a plausible way to smuggle one
923
+ past a values-only scan.
924
+ """
925
+ if isinstance(value, str):
926
+ return leaf(value)
927
+ if depth >= _MAX_SCAN_DEPTH:
928
+ # Fail CLOSED. Returning the container untouched made the depth
929
+ # bound a deterministic bypass: a server placing a credential 13
930
+ # levels down under a scanned key passed straight through, and
931
+ # third-party structured_content is precisely the threat this
932
+ # function's docstring names. The cap exists to stop a hostile
933
+ # structure exhausting the stack, so refusing to descend must
934
+ # mean refusing to emit, not emitting unchecked.
935
+ return on_cap()
936
+ if isinstance(value, dict):
937
+ return {
938
+ (leaf(k) if scan_keys and isinstance(k, str) else k):
939
+ _walk(v, leaf, on_cap, depth + 1, scan_keys)
940
+ for k, v in value.items()
941
+ }
942
+ if isinstance(value, (list, tuple)):
943
+ return type(value)(
944
+ _walk(v, leaf, on_cap, depth + 1, scan_keys) for v in value
945
+ )
946
+ return value
947
+
948
+
949
+ # ---------------------------------------------------------------------------
950
+ # ---- Output policy check --------------------------------------------------
951
+ # ---------------------------------------------------------------------------
952
+
953
+ def check_output_policy(tool_name: str, result: dict) -> dict:
954
+ """Apply content-level policy to a tool's output.
955
+
956
+ Called once, centrally, by registry.dispatch() after every tool
957
+ call — not something individual tool files call themselves.
958
+ Mutates *result* in place and returns it.
959
+
960
+ EVERY value is scanned, not an allowlist of keys (#47) — see the
961
+ note above `_walk`. `scan_keys=False` still, so a result's dict KEYS
962
+ are left alone: rewriting them would change the shape a tool's
963
+ consumer sees, and a tool result's keys are harness- or
964
+ server-declared names rather than free text. That asymmetry with
965
+ `check_input_policy` is deliberate and predates this change.
966
+
967
+ NOTHING THIS FUNCTION DOES IS SILENT ANY MORE (#49). Every
968
+ alteration — a redacted value, a depth-cap substitution — produces
969
+ exactly ONE warning naming the tool, what was altered and how many
970
+ values were touched (and WHICH top-level key sat over a capped
971
+ container). The matched content is never echoed: this function
972
+ exists because something credential-shaped was found, so quoting it
973
+ to say so would leak it into app.log, the TUI transcript and every
974
+ place a WARNING travels — the same reasoning _input_leaf's refusal
975
+ already states. Before #49 the depth cap replaced a whole nested
976
+ container with a sentence and logged nothing, leaving truncated
977
+ output indistinguishable from complete output; `registry.schemas`
978
+ states the rule this was violating ("no quiet invisibility").
979
+ """
980
+ redacted_values = 0
981
+ capped_keys = []
982
+
983
+ def _leaf(text: str) -> str:
984
+ nonlocal redacted_values
985
+ replacement = redact_output_text(text)
986
+ if replacement != text:
987
+ redacted_values += 1
988
+ return replacement
989
+
990
+ for key in list(result):
991
+ hit_cap = []
992
+ result[key] = _walk(
993
+ result[key],
994
+ _leaf,
995
+ lambda: (hit_cap.append(key),
996
+ "[REDACTED: nested beyond scan depth]")[1],
997
+ )
998
+ if hit_cap:
999
+ capped_keys.append(key)
1000
+
1001
+ if redacted_values or capped_keys:
1002
+ parts = []
1003
+ if redacted_values:
1004
+ parts.append(f"{redacted_values} value(s) had credentials "
1005
+ f"replaced")
1006
+ if capped_keys:
1007
+ parts.append("nested content beyond scan depth was substituted "
1008
+ "under key(s): " + ", ".join(map(repr, capped_keys)))
1009
+ logger.warning("Output policy altered %s result: %s.",
1010
+ tool_name, "; ".join(parts))
1011
+ return result
1012
+
1013
+
1014
+ # ---------------------------------------------------------------------------
1015
+ # ---- Input policy check ---------------------------------------------------
1016
+ # ---------------------------------------------------------------------------
1017
+ #
1018
+ # ROADMAP_v2 §25 (R5). The symmetric half of check_output_policy, and it
1019
+ # closes two holes that predate the research-pipeline work:
1020
+ #
1021
+ # 1. Results were scanned for secrets; ARGUMENTS never were. A tool the
1022
+ # user approved can be called with text drawn from context, so the
1023
+ # argument list is an egress path and nothing was watching it.
1024
+ # 2. BLOCKED_DOMAINS was opt-in PER TOOL -- only web_search.py and
1025
+ # fetch_url.py import is_domain_blocked(). Any other tool taking a
1026
+ # URL (every MCP tool that fetches anything) bypassed the list
1027
+ # entirely, which is the `fetch_url` defect shape again: a control
1028
+ # that exists, is documented, and does not cover what it claims to.
1029
+ #
1030
+ # Enforced at dispatch() so it covers every tool in every mode, rather
1031
+ # than at the pipeline boundary that prompted it.
1032
+
1033
+ # A URL is recognised by its scheme, not by looking like a hostname. A
1034
+ # BARE domain is deliberately NOT refused here: this harness researches
1035
+ # security topics, so "exploit-db.com" appears in legitimate queries,
1036
+ # claims and reports constantly, and refusing every mention would make the
1037
+ # blocklist unusable while blocking nothing an attacker couldn't rephrase.
1038
+ # fetch_url.py keeps its own bare-domain check for the value it is given.
1039
+ _URL_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s\"'<>]+")
1040
+
1041
+
1042
+ def _input_leaf(text: str) -> str:
1043
+ """Refuse a string carrying a credential or a blocked URL."""
1044
+ for pattern in _SECRET_PATTERNS:
1045
+ if pattern.search(text):
1046
+ # The matched text is NOT echoed. A refusal reason travels
1047
+ # into model context, the TUI transcript and the persisted
1048
+ # MessageLog -- quoting the credential to explain that a
1049
+ # credential leaked would leak it three more places.
1050
+ raise _PolicyRefusal(
1051
+ "an argument contains what looks like a credential")
1052
+ for url in _URL_RE.findall(text):
1053
+ if is_domain_blocked(url):
1054
+ raise _PolicyRefusal(f"an argument targets a blocked domain: {url}")
1055
+ return text
1056
+
1057
+
1058
+ def check_input_policy(tool_name: str, params: dict) -> str | None:
1059
+ """Content-level policy on a tool's ARGUMENTS. Returns a refusal
1060
+ reason, or None to proceed.
1061
+
1062
+ Called by registry.dispatch() before the handler runs. REFUSES rather
1063
+ than redacting: silently rewriting an argument changes what the call
1064
+ does, so a tool would run with parameters neither the model nor the
1065
+ user chose and report success. A denial is legible to both -- the same
1066
+ reasoning D24 settled for uncallable tools.
1067
+ """
1068
+ try:
1069
+ _walk(
1070
+ params,
1071
+ _input_leaf,
1072
+ lambda: _raise_refusal("arguments nested beyond the scan depth"),
1073
+ scan_keys=True,
1074
+ )
1075
+ except _PolicyRefusal as e:
1076
+ return f"{tool_name} call refused by policy: {e}"
1077
+ return None
1078
+
1079
+
1080
+ def _raise_refusal(reason: str):
1081
+ """`lambda: raise` is not valid syntax; this is the on_cap callback."""
1082
+ raise _PolicyRefusal(reason)