@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,682 @@
1
+ """
2
+ safety/policy_enforcement.py
3
+
4
+ Content-level policy, applied AFTER a tool runs, to its output — NOT
5
+ access control (that's security/permissions.py's job, applied BEFORE a
6
+ tool runs, deciding whether it runs at all). registry.py's dispatch()
7
+ calls check_output_policy() on every tool result, regardless of which
8
+ tool produced it.
9
+
10
+ Three responsibilities:
11
+ 1. Centralized URL policy — is_url_permitted() answers "may this URL be
12
+ requested at all", composing the blocked-domain list with a
13
+ non-public-address guard. web_search.py and fetch_url.py both call
14
+ it, so a URL cannot be permitted for one and refused by the other.
15
+ is_domain_blocked() remains the domain half on its own.
16
+ 2. Secret redaction — defense-in-depth: if a tool's output accidentally
17
+ contains something that looks like a leaked API key, it is replaced
18
+ with [REDACTED] before the content reaches storage or the model.
19
+ 3. Argument policy (§25) — the symmetric half of (2), applied BEFORE a
20
+ tool runs, to what it was asked to do rather than to what it returned.
21
+ Refuses rather than redacts, and refuses on the blocked-domain list
22
+ too, which until then only the two tools that opted in enforced.
23
+
24
+ (1) and (3) overlap deliberately: (3) is the choke point every tool
25
+ passes through, (1) is what a tool checks about a value only it can
26
+ interpret. Neither makes the other redundant.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import ipaddress
32
+ import json
33
+ import logging
34
+ import os
35
+ import re
36
+ import socket
37
+ from urllib.parse import urlparse
38
+
39
+ import config
40
+ # safety -> security, and never the reverse. `security.posture` imports
41
+ # `config` and the stdlib only, so this adds no cycle -- which is exactly
42
+ # why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
43
+ # this module, and the edge only stays acyclic while it runs one way.
44
+ from security import posture
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+ # ---------------------------------------------------------------------------
49
+ # ---- Blocked domains ------------------------------------------------------
50
+ # ---------------------------------------------------------------------------
51
+
52
+ # Centralized blocked-domain list. web_search.py and fetch_url.py import
53
+ # is_domain_blocked() to enforce this. Add domains hosting malware,
54
+ # phishing, or other harmful content based on your threat model.
55
+ #
56
+ # MAINTENANCE: domains change ownership, get taken down, or are repurposed.
57
+ # Review this list periodically. Remove domains that are no longer harmful;
58
+ # add new ones as threats emerge. An empty set disables domain filtering.
59
+ BLOCKED_DOMAINS: set[str] = {
60
+ # Well-known harmful domains — examples only, maintain as needed.
61
+ "malware-traffic-analysis.net", # malware sample distribution
62
+ "exploit-db.com", # exploit code repository
63
+ "packetstormsecurity.com", # exploit/tool archive
64
+ }
65
+
66
+
67
+ def _hostname(url: str) -> str:
68
+ """The host of a full URL or a bare domain, normalised for comparison.
69
+
70
+ ONE normaliser, because the two branches below had drifted apart and
71
+ the difference was a bypass (#48). `.hostname` already lowercases,
72
+ strips the port and strips userinfo; the bare-domain branch did none
73
+ of that except by accident, so `exploit-db.com:443` passed it while
74
+ the URL form of the same host was caught.
75
+
76
+ The trailing dot defeated BOTH branches. `exploit-db.com.` is a legal,
77
+ resolvable FQDN naming exactly the same host, `urlparse` keeps the
78
+ dot, and no amount of lowercasing makes the strings equal. Stripped
79
+ here rather than at each call site, since every caller compares.
80
+ """
81
+ if "://" in url:
82
+ try:
83
+ host = urlparse(url).hostname or ""
84
+ except Exception:
85
+ host = url
86
+ else:
87
+ # Strip path, then userinfo, then port -- the three things
88
+ # .hostname does for the URL branch and this one used not to.
89
+ host = url.split("/")[0]
90
+ host = host.rpartition("@")[2]
91
+ if host.startswith("["): # [::1]:8080
92
+ host = host.partition("]")[0].lstrip("[")
93
+ else:
94
+ host = host.partition(":")[0]
95
+ return host.lower().rstrip(".")
96
+
97
+
98
+ def is_domain_blocked(url: str) -> bool:
99
+ """Return True if the URL's host is in, or under, BLOCKED_DOMAINS.
100
+
101
+ Handles full URLs (https://example.com/path) and bare domains
102
+ (example.com), through one normaliser -- see `_hostname`.
103
+
104
+ Subdomains ARE matched: blocking ``evil.com`` also blocks
105
+ ``sub.evil.com``. This reverses a documented carve-out, and the
106
+ reversal is the point. A blocklist an attacker leaves by prepending
107
+ one label is close to decorative, and the old docstring's advice
108
+ ("block ``evil.com`` explicitly if you also want ``sub.evil.com``")
109
+ asked a maintainer to enumerate an infinite set. Matching a suffix
110
+ can only ever block MORE than the list's author wrote down, and this
111
+ list's whole purpose is refusal, so over-blocking is the safe
112
+ direction to be wrong in.
113
+
114
+ The match is label-wise (`host == blocked` or `host` ends with
115
+ `"." + blocked`), never a bare string suffix: `notevil.com` must not
116
+ match `evil.com`.
117
+ """
118
+ if not BLOCKED_DOMAINS:
119
+ return False
120
+ host = _hostname(url)
121
+ return any(host == blocked or host.endswith("." + blocked)
122
+ for blocked in (b.lower().rstrip(".") for b in BLOCKED_DOMAINS))
123
+
124
+
125
+ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
126
+ """Refusal reason for a URL, or None if it may be requested.
127
+
128
+ ONE checker, two callers -- `fetch_url` before every hop it issues,
129
+ and `web_search` over the results it is about to hand the model. They
130
+ were asking different questions about the same policy, which is how
131
+ #53 and #54 stayed open while #48 looked like the whole of it.
132
+
133
+ Three gates, cheapest first:
134
+
135
+ 1. Scheme. http/https only. Moved here from fetch_url so the answer
136
+ does not depend on which caller asked.
137
+ 2. `is_domain_blocked`.
138
+ 3. Address. Rejects anything that is not a GLOBAL address --
139
+ loopback, private, link-local, reserved and multicast in one
140
+ predicate. `169.254.169.254` is the one that matters most: on a
141
+ cloud instance with IMDSv1 it returns role credentials, and the
142
+ route to it is a fetched page saying "for full details see
143
+ <that URL>" to a grounding pass that obliges.
144
+
145
+ `resolve=False` answers from the literal text only -- an IP literal
146
+ is still checked, a NAME is taken on trust. That is for `web_search`,
147
+ which filters results rather than fetching them: a DNS lookup per
148
+ search hit is real latency for a check whose failure mode is "the
149
+ model is shown a URL that fetch_url will refuse anyway". The caller
150
+ that actually opens a socket always resolves.
151
+
152
+ KNOWN LIMITATION -- DNS rebinding. When we resolve, httpx then
153
+ resolves again independently, so a host whose records change between
154
+ this check and the connect can still be reached. Closing it means
155
+ connecting to the pinned IP and passing the original host in a
156
+ `Host:` header, which on HTTPS fights SNI and certificate
157
+ verification -- a subtly wrong implementation there is a worse hole
158
+ than this one. Recorded rather than silently accepted; the attacker
159
+ needs control of DNS for a host the model was already induced to
160
+ fetch.
161
+ """
162
+ if not (url.startswith("http://") or url.startswith("https://")):
163
+ return "URL must start with http:// or https://"
164
+
165
+ if is_domain_blocked(url):
166
+ return f"Domain is blocked by policy: {url}"
167
+
168
+ host = _hostname(url)
169
+ if not host:
170
+ return f"URL has no host: {url}"
171
+
172
+ try:
173
+ literal = ipaddress.ip_address(host)
174
+ except ValueError:
175
+ literal = None
176
+
177
+ if literal is not None:
178
+ addresses = [literal]
179
+ elif resolve:
180
+ try:
181
+ addresses = [
182
+ ipaddress.ip_address(info[4][0])
183
+ for info in socket.getaddrinfo(host, None)
184
+ ]
185
+ except (socket.gaierror, ValueError) as e:
186
+ # Fail CLOSED. A name we cannot resolve is one we cannot
187
+ # clear, and httpx would only fail on it a moment later --
188
+ # refusing here costs nothing and keeps "unchecked" from
189
+ # ever meaning "allowed".
190
+ return f"Could not resolve host for policy check: {host} ({e})"
191
+ else:
192
+ addresses = []
193
+
194
+ # ANY, not all. A name resolving to one public and one private
195
+ # address is the rebinding shape, and taking the first answer or
196
+ # requiring unanimity would both let it through.
197
+ for address in addresses:
198
+ if not address.is_global:
199
+ return (f"URL resolves to a non-public address, which this "
200
+ f"harness will not fetch: {host} -> {address}")
201
+
202
+ return None
203
+
204
+
205
+ # ---------------------------------------------------------------------------
206
+ # ---- Secret redaction -----------------------------------------------------
207
+ # ---------------------------------------------------------------------------
208
+
209
+ _SECRET_PATTERNS = [
210
+ # OpenAI-style keys
211
+ re.compile(r"sk-[a-zA-Z0-9]{20,}"),
212
+ # Anthropic-style keys
213
+ re.compile(r"sk-ant-[a-zA-Z0-9\-]{20,}"),
214
+ # GitHub personal access tokens
215
+ re.compile(r"ghp_[a-zA-Z0-9]{36}"),
216
+ # AWS access key IDs
217
+ re.compile(r"AKIA[0-9A-Z]{16}"),
218
+ # Google API keys
219
+ re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
220
+ # Slack tokens
221
+ re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
222
+ # PEM private key headers
223
+ re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
224
+ ]
225
+
226
+
227
+ def redact_secrets(text: str) -> str:
228
+ """Replace strings matching known secret patterns with [REDACTED]."""
229
+ for pattern in _SECRET_PATTERNS:
230
+ text = pattern.sub("[REDACTED]", text)
231
+ return text
232
+
233
+
234
+ # ---------------------------------------------------------------------------
235
+ # ---- Credential shapes (#167, batch 20) ------------------------------------
236
+ # ---------------------------------------------------------------------------
237
+ #
238
+ # The seven patterns above are VENDOR TOKENS -- every one has a prefix some
239
+ # company mints, so a match is high-confidence and a bare `[REDACTED]` loses
240
+ # nothing structural. A credential that is just a password in a field has no
241
+ # prefix, and nothing matched it (#167): batch 14 added five build manifests
242
+ # to read_project_doc's allowlist (#94), whose credential conventions are
243
+ # exactly the three unscrubbed shapes below.
244
+ #
245
+ # These are CONTEXT shapes, not tokens: each anchors on a credential FIELD
246
+ # NAME (or URL userinfo), so an auth error code, a stack frame or an ARN
247
+ # does not match, and the replacement is SURGICAL -- tag names, field names,
248
+ # usernames and hosts survive, only the value goes. That structure is why
249
+ # the false-positive cost is acceptable: documentation quoting an example
250
+ # keeps its shape and loses only its example value.
251
+ #
252
+ # OUTPUT REDACTION ONLY. These never feed check_input_policy: refusing a
253
+ # call because its arguments mention `password = "..."` would break every
254
+ # legitimate call that carries build-file content the user asked for --
255
+ # _SECRET_PATTERNS stay the input side's whole vocabulary. And the keyword
256
+ # set is password|passwd ONLY, deliberately not token/secret/key: those
257
+ # appear constantly in ordinary prose ("token bucket", JWT discussion) and
258
+ # would redact aggressively.
259
+
260
+ # Template placeholders survive untouched: `${PASSWORD}`, `$DB_PASS`,
261
+ # `{{ secrets.DB_PW }}`, `<your-password-here>` name where a credential
262
+ # WOULD go; they are what docs and templates contain, and redacting them
263
+ # destroys the example while protecting nothing.
264
+ _TEMPLATE_VALUE_RE = re.compile(
265
+ r"^(?:"
266
+ r"\$\{[^}]*\}" # ${PASSWORD}, ${secrets.DB}
267
+ r"|\$[A-Za-z_][A-Za-z0-9_]*" # $PASSWORD
268
+ r"|\{\{[^}]*\}\}" # {{ secrets.DB_PW }}
269
+ r"|<[^<>]+>" # <your-password-here>
270
+ r")$")
271
+
272
+ # scheme://user:secret@host -> scheme://user:[REDACTED]@host. Credentials
273
+ # in a URL's userinfo are never anything else, which makes this the cheapest
274
+ # of the three; anchoring on `//` keeps bare `user:pass@host` fragments in
275
+ # prose alone, and excluding `/ \s : @` from both sides means a host:port
276
+ # path (`https://example.com:8080/x`) cannot masquerade as userinfo.
277
+ _URL_USERINFO_RE = re.compile(r"(//[^/\s:@]+:)([^/\s@]+)(@)")
278
+
279
+ # <password>s3cr3t</password> -> <password>[REDACTED]</password>, POM and
280
+ # settings.xml convention. The backreference keeps the closing tag honest;
281
+ # the value excludes `<`, so nested markup cannot be swallowed. Groups:
282
+ # 1 opening tag, 2 tag name, 3 value.
283
+ _XML_PASSWORD_RE = re.compile(
284
+ r"(?i)(<(password|passwd)>)([^<]{4,200})(</\2>)")
285
+
286
+ # password = "hunter2" / password: "hunter2" / password "hunter2" ->
287
+ # the value becomes [REDACTED] inside its quotes. The separator accepts
288
+ # `=`, `:` AND Gradle's bare-space credentials-block form -- which is the
289
+ # issue's own headline case and which a `[=:]`-only sketch misses. Quoted
290
+ # values only, minimum 4 chars: unquoted .properties-style values are out
291
+ # of scope by decision, and the value excludes newlines so a quote further
292
+ # down a file cannot close a match opened lines above. Groups: 1 keyword,
293
+ # 2 separator, 3 opening quote, 4 value.
294
+ _ASSIGNMENT_RE = re.compile(
295
+ r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
296
+ r"([\"'])([^\"'\n]{4,200})\3")
297
+
298
+
299
+ def _redact_credential_shapes(text: str) -> str:
300
+ """Apply the three context shapes, preserving surrounding structure.
301
+
302
+ Each replacer checks the captured VALUE against the placeholder shapes
303
+ before substituting -- a template reference is exactly what docs and
304
+ build files are full of, and redacting it destroys the example while
305
+ protecting nothing."""
306
+ text = _URL_USERINFO_RE.sub(r"\1[REDACTED]\3", text)
307
+
308
+ def _xml(m):
309
+ if _TEMPLATE_VALUE_RE.match(m.group(3)):
310
+ return m.group(0)
311
+ return f"{m.group(1)}[REDACTED]{m.group(4)}"
312
+
313
+ text = _XML_PASSWORD_RE.sub(_xml, text)
314
+
315
+ def _assign(m):
316
+ if _TEMPLATE_VALUE_RE.match(m.group(4)):
317
+ return m.group(0)
318
+ return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
319
+ f"[REDACTED]{m.group(3)}")
320
+
321
+ return _ASSIGNMENT_RE.sub(_assign, text)
322
+
323
+
324
+ def redaction_enabled() -> bool:
325
+ """Whether pattern-based redaction of tool output is active.
326
+
327
+ Two switches, one question. config.REDACT_TOOL_OUTPUTS is the
328
+ permanent answer; VENASTINE_REDACT_OFF weakens it for one run --
329
+ a debugging session that needs the model to see real values --
330
+ without editing anything persistent. The environment can only ever
331
+ turn redaction OFF, never back on past the constant: an off-by-default
332
+ protection an environment variable could silently re-enable would not
333
+ be a switch but a coin flip.
334
+
335
+ §40 (UN1): BOTH switches are now read from the frozen posture rather
336
+ than live, and the environment half is the reason that mattered here.
337
+ `config` binds at import so mutating the environment could never move
338
+ the constant -- but `os.environ.get(...)` at call time followed a
339
+ mutation, so this one function was a live in-process switch for
340
+ redaction. The distinction above survives as two posture fields; only
341
+ the moment of reading moved.
342
+
343
+ What this does NOT govern, whichever way it reads: input refusals
344
+ (_input_leaf -- a denial is legible, not destructive), the depth-cap
345
+ substitution (fail-closed structural bound), and logging_setup's
346
+ formatter redaction (the second sink guards app.log regardless).
347
+ """
348
+ active = posture.current()
349
+ if not active.redact_tool_outputs:
350
+ return False
351
+ return not active.redact_off_env
352
+
353
+
354
+ def redact_output_text(text: str) -> str:
355
+ """The one redaction path tool OUTPUT takes, under one switch.
356
+
357
+ Vendor tokens first, then credential shapes on what survives. Both are
358
+ substitutions of content the user chose to see raw when they flip the
359
+ switch off -- so this is where VENASTINE_REDACT_OFF bites. The depth
360
+ cap is NOT routed through here: it is structure, not content judgment,
361
+ and stays fail-closed unconditionally.
362
+
363
+ PUBLIC since batch 41 (X4). It was private while `param_digest` was
364
+ its only caller outside this module's own dispatch path, and the §41
365
+ diff is the second: a `write` renders the content the tool put on
366
+ disk, which is tool output by any reading, and it replaces the very
367
+ digest line that goes through here. `redact_secrets` alone would have
368
+ redacted LESS than the line it replaced -- the vendor tokens but not
369
+ the credential shapes -- which is a regression dressed as a feature.
370
+ The docstring above already called this "the one path"; it now is one
371
+ for both consumers."""
372
+ if not redaction_enabled():
373
+ return text
374
+ return _redact_credential_shapes(redact_secrets(text))
375
+
376
+
377
+
378
+ # How much of one tool parameter reaches a digest, and how much of the whole
379
+ # digest reaches a consumer. A sidebar is 22 columns and a transcript line
380
+ # wraps; the point is to say WHICH url or WHICH query, not to reproduce the
381
+ # call.
382
+ _DIGEST_VALUE_CHARS = 60
383
+ _DIGEST_CHARS = 140
384
+
385
+
386
+ def param_digest(params, omit=()) -> str:
387
+ """A short, REDACTED description of what a tool was called with.
388
+
389
+ REDACTED BEFORE TRUNCATION, not after. Truncating first can cut a
390
+ credential in half and leave the surviving half unmatched by the pattern
391
+ that would have caught it -- the redactor works on whole strings, so it
392
+ has to see one. That ordering is the whole reason this function is
393
+ shared rather than copied: §26 established it for the research view, and
394
+ §27's replay renderer is its second caller. A second copy is how the
395
+ ordering silently regresses in one of them, which is this project's
396
+ canonical bug shape (fix at the producer, not the consumer).
397
+
398
+ It lives HERE, beside redact_secrets, rather than in the orchestrator
399
+ that first needed it: the rule it enforces is a content-policy rule,
400
+ and this module is where post-call content policy lives.
401
+
402
+ Tool arguments are the path that genuinely was not scanned: §25 R5 added
403
+ check_input_policy(), but that REFUSES a call, it does not redact one,
404
+ and dispatch()'s check_output_policy only ever saw results. A fetch_url
405
+ whose url carries an API key would otherwise be printed verbatim in two
406
+ shells and, since §27, replayed out of the archive into a third place.
407
+
408
+ Values only, no keys: for the tools a pass actually reaches for -- one
409
+ url, one query, one command -- the key is noise, and per-value
410
+ truncation is what keeps a `write` call from pasting a file into the
411
+ transcript.
412
+
413
+ `omit` NAMES KEYS TO SKIP (§42, RA3), and it is a parameter rather
414
+ than a rule because this module must not learn about the registry:
415
+ it imports `config` and `security.posture` and nothing else, and
416
+ which tool declares which param is the registry's knowledge.
417
+ `registry.call_digest()` is the one caller that does the lookup.
418
+
419
+ It exists for `shell`'s `rationale`, which is prose written for a
420
+ person to read and would otherwise take sixty of the digest's one
421
+ hundred and forty characters. Key ORDER is the model's, so it
422
+ lands first about half the time: measured, `ls -la dist/` became
423
+ `I need to check whether the build artifacts directory still… ls
424
+ -la dist/` -- the command no longer the first thing a reader sees.
425
+ """
426
+ if not isinstance(params, dict) or not params:
427
+ return ""
428
+ parts = []
429
+ for key, value in params.items():
430
+ if key in omit:
431
+ continue
432
+ text = value if isinstance(value, str) else json.dumps(value, default=str)
433
+ # The same redaction tool output takes -- shapes included (#167),
434
+ # so a fetch_url whose url carries userinfo does not print its
435
+ # password into the transcript, the CLI and (since §27) the
436
+ # replayed archive. Under VENASTINE_REDACT_OFF this shows raw,
437
+ # like every other pattern substitution; it stays display-only
438
+ # and never logs, whatever the switch reads.
439
+ text = redact_output_text(text)
440
+ if len(text) > _DIGEST_VALUE_CHARS:
441
+ text = text[:_DIGEST_VALUE_CHARS - 1] + "…"
442
+ parts.append(text)
443
+ digest = " ".join(p for p in parts if p)
444
+ if len(digest) > _DIGEST_CHARS:
445
+ digest = digest[:_DIGEST_CHARS - 1] + "…"
446
+ return digest
447
+
448
+
449
+ # ---------------------------------------------------------------------------
450
+ # ---- Shared scan traversal ------------------------------------------------
451
+ # ---------------------------------------------------------------------------
452
+
453
+ # THERE IS NO KEY ALLOWLIST ANY MORE (#47). There used to be:
454
+ #
455
+ # _SCANNED_KEYS = ("content", "result", "stdout", "stderr", "error")
456
+ #
457
+ # and `check_output_policy` tested exact membership. Both search tools
458
+ # return their payload under `results` -- plural -- which is not `result`,
459
+ # so the output of the two tools whose content is ENTIRELY third-party
460
+ # text was the output nothing redacted. Verified end to end: a planted
461
+ # key survived into the persisted archive and into what the model was
462
+ # sent the next turn.
463
+ #
464
+ # The allowlist grew correctly each time someone noticed a gap ("error"
465
+ # was added because an MCP server reports a failed call in band, often
466
+ # quoting the credential that was rejected), and that is the problem: it
467
+ # re-created the defect for every tool added after it, and it did so
468
+ # SILENTLY. mcp_client/client.py's _normalize was written to satisfy this
469
+ # set deliberately, treating membership as a guarantee -- so the contract
470
+ # was real and known, and the two oldest built-ins never satisfied it.
471
+ #
472
+ # Scanning every key is the producer-side fix this project's own
473
+ # convention names ("a per-case branch added to a normalizer usually
474
+ # means the fix belongs one layer down"). A tool that returns text now
475
+ # has that text scanned because it is text, not because someone
476
+ # remembered to name its key.
477
+
478
+
479
+ # How deep to descend into nested values. Bounded so a pathological or
480
+ # hostile structure can't turn a scan into a stack overflow -- this walks
481
+ # output from code the user didn't write, and arguments chosen by a model
482
+ # that has been reading attacker-controlled web pages.
483
+ _MAX_SCAN_DEPTH = 12
484
+
485
+
486
+ class _PolicyRefusal(Exception):
487
+ """Internal control flow: a leaf handler rejecting what it was given.
488
+
489
+ Never escapes this module -- check_input_policy() converts it to a
490
+ reason string, and the redaction path's leaf handler never raises.
491
+ """
492
+
493
+
494
+ def _walk(value, leaf, on_cap, depth: int = 0, scan_keys: bool = False):
495
+ """Depth-bounded traversal of a JSON-shaped value.
496
+
497
+ ONE traversal, two callers. `leaf` maps a str to its replacement (and
498
+ may raise `_PolicyRefusal`); `on_cap` decides what happens when the
499
+ depth bound is reached. Redaction and argument scanning differ ONLY in
500
+ those two behaviours, and writing the walk twice is how the two drift
501
+ -- the §17 hole existed because `structured_content` was nested and
502
+ the scan was not, and a second copy would reintroduce exactly that
503
+ class of gap on the input side.
504
+
505
+ ADDED IN §17 (as `_redact_value`), and it closed a real hole rather
506
+ than generalising for its own sake. The original scanned only
507
+ TOP-LEVEL strings, which was sufficient while every tool returned
508
+ `{"result": "<some string>"}`. MCP breaks that assumption:
509
+ `structured_content` is arbitrary JSON from a third-party server, and
510
+ it arrives under `result` as a dict -- so the output most likely to
511
+ contain a leaked credential was the one output that was never
512
+ scanned. Caught by an end-to-end test, not by reading the code.
513
+
514
+ Fixed here rather than in mcp_client/, because the producer of the
515
+ unscanned shape isn't MCP -- it's any tool returning nested data under
516
+ a scanned key, which `discrete_math` and `probability_stats` already
517
+ can. A special case at the MCP boundary would have left those.
518
+
519
+ scan_keys is False for redaction and True for argument scanning. The
520
+ asymmetry is deliberate: rewriting a result's dict KEYS would change
521
+ the shape a tool's consumer sees, while a secret sitting in an
522
+ argument key (`{"sk-ant-...": "x"}`) is a plausible way to smuggle one
523
+ past a values-only scan.
524
+ """
525
+ if isinstance(value, str):
526
+ return leaf(value)
527
+ if depth >= _MAX_SCAN_DEPTH:
528
+ # Fail CLOSED. Returning the container untouched made the depth
529
+ # bound a deterministic bypass: a server placing a credential 13
530
+ # levels down under a scanned key passed straight through, and
531
+ # third-party structured_content is precisely the threat this
532
+ # function's docstring names. The cap exists to stop a hostile
533
+ # structure exhausting the stack, so refusing to descend must
534
+ # mean refusing to emit, not emitting unchecked.
535
+ return on_cap()
536
+ if isinstance(value, dict):
537
+ return {
538
+ (leaf(k) if scan_keys and isinstance(k, str) else k):
539
+ _walk(v, leaf, on_cap, depth + 1, scan_keys)
540
+ for k, v in value.items()
541
+ }
542
+ if isinstance(value, (list, tuple)):
543
+ return type(value)(
544
+ _walk(v, leaf, on_cap, depth + 1, scan_keys) for v in value
545
+ )
546
+ return value
547
+
548
+
549
+ # ---------------------------------------------------------------------------
550
+ # ---- Output policy check --------------------------------------------------
551
+ # ---------------------------------------------------------------------------
552
+
553
+ def check_output_policy(tool_name: str, result: dict) -> dict:
554
+ """Apply content-level policy to a tool's output.
555
+
556
+ Called once, centrally, by registry.dispatch() after every tool
557
+ call — not something individual tool files call themselves.
558
+ Mutates *result* in place and returns it.
559
+
560
+ EVERY value is scanned, not an allowlist of keys (#47) — see the
561
+ note above `_walk`. `scan_keys=False` still, so a result's dict KEYS
562
+ are left alone: rewriting them would change the shape a tool's
563
+ consumer sees, and a tool result's keys are harness- or
564
+ server-declared names rather than free text. That asymmetry with
565
+ `check_input_policy` is deliberate and predates this change.
566
+
567
+ NOTHING THIS FUNCTION DOES IS SILENT ANY MORE (#49). Every
568
+ alteration — a redacted value, a depth-cap substitution — produces
569
+ exactly ONE warning naming the tool, what was altered and how many
570
+ values were touched (and WHICH top-level key sat over a capped
571
+ container). The matched content is never echoed: this function
572
+ exists because something credential-shaped was found, so quoting it
573
+ to say so would leak it into app.log, the TUI transcript and every
574
+ place a WARNING travels — the same reasoning _input_leaf's refusal
575
+ already states. Before #49 the depth cap replaced a whole nested
576
+ container with a sentence and logged nothing, leaving truncated
577
+ output indistinguishable from complete output; `registry.schemas`
578
+ states the rule this was violating ("no quiet invisibility").
579
+ """
580
+ redacted_values = 0
581
+ capped_keys = []
582
+
583
+ def _leaf(text: str) -> str:
584
+ nonlocal redacted_values
585
+ replacement = redact_output_text(text)
586
+ if replacement != text:
587
+ redacted_values += 1
588
+ return replacement
589
+
590
+ for key in list(result):
591
+ hit_cap = []
592
+ result[key] = _walk(
593
+ result[key],
594
+ _leaf,
595
+ lambda: (hit_cap.append(key),
596
+ "[REDACTED: nested beyond scan depth]")[1],
597
+ )
598
+ if hit_cap:
599
+ capped_keys.append(key)
600
+
601
+ if redacted_values or capped_keys:
602
+ parts = []
603
+ if redacted_values:
604
+ parts.append(f"{redacted_values} value(s) had credentials "
605
+ f"replaced")
606
+ if capped_keys:
607
+ parts.append("nested content beyond scan depth was substituted "
608
+ "under key(s): " + ", ".join(map(repr, capped_keys)))
609
+ logger.warning("Output policy altered %s result: %s.",
610
+ tool_name, "; ".join(parts))
611
+ return result
612
+
613
+
614
+ # ---------------------------------------------------------------------------
615
+ # ---- Input policy check ---------------------------------------------------
616
+ # ---------------------------------------------------------------------------
617
+ #
618
+ # ROADMAP_v2 §25 (R5). The symmetric half of check_output_policy, and it
619
+ # closes two holes that predate the research-pipeline work:
620
+ #
621
+ # 1. Results were scanned for secrets; ARGUMENTS never were. A tool the
622
+ # user approved can be called with text drawn from context, so the
623
+ # argument list is an egress path and nothing was watching it.
624
+ # 2. BLOCKED_DOMAINS was opt-in PER TOOL -- only web_search.py and
625
+ # fetch_url.py import is_domain_blocked(). Any other tool taking a
626
+ # URL (every MCP tool that fetches anything) bypassed the list
627
+ # entirely, which is the `fetch_url` defect shape again: a control
628
+ # that exists, is documented, and does not cover what it claims to.
629
+ #
630
+ # Enforced at dispatch() so it covers every tool in every mode, rather
631
+ # than at the pipeline boundary that prompted it.
632
+
633
+ # A URL is recognised by its scheme, not by looking like a hostname. A
634
+ # BARE domain is deliberately NOT refused here: this harness researches
635
+ # security topics, so "exploit-db.com" appears in legitimate queries,
636
+ # claims and reports constantly, and refusing every mention would make the
637
+ # blocklist unusable while blocking nothing an attacker couldn't rephrase.
638
+ # fetch_url.py keeps its own bare-domain check for the value it is given.
639
+ _URL_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s\"'<>]+")
640
+
641
+
642
+ def _input_leaf(text: str) -> str:
643
+ """Refuse a string carrying a credential or a blocked URL."""
644
+ for pattern in _SECRET_PATTERNS:
645
+ if pattern.search(text):
646
+ # The matched text is NOT echoed. A refusal reason travels
647
+ # into model context, the TUI transcript and the persisted
648
+ # MessageLog -- quoting the credential to explain that a
649
+ # credential leaked would leak it three more places.
650
+ raise _PolicyRefusal(
651
+ "an argument contains what looks like a credential")
652
+ for url in _URL_RE.findall(text):
653
+ if is_domain_blocked(url):
654
+ raise _PolicyRefusal(f"an argument targets a blocked domain: {url}")
655
+ return text
656
+
657
+
658
+ def check_input_policy(tool_name: str, params: dict) -> str | None:
659
+ """Content-level policy on a tool's ARGUMENTS. Returns a refusal
660
+ reason, or None to proceed.
661
+
662
+ Called by registry.dispatch() before the handler runs. REFUSES rather
663
+ than redacting: silently rewriting an argument changes what the call
664
+ does, so a tool would run with parameters neither the model nor the
665
+ user chose and report success. A denial is legible to both -- the same
666
+ reasoning D24 settled for uncallable tools.
667
+ """
668
+ try:
669
+ _walk(
670
+ params,
671
+ _input_leaf,
672
+ lambda: _raise_refusal("arguments nested beyond the scan depth"),
673
+ scan_keys=True,
674
+ )
675
+ except _PolicyRefusal as e:
676
+ return f"{tool_name} call refused by policy: {e}"
677
+ return None
678
+
679
+
680
+ def _raise_refusal(reason: str):
681
+ """`lambda: raise` is not valid syntax; this is the on_cap callback."""
682
+ raise _PolicyRefusal(reason)
File without changes