@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -1,682 +1,1082 @@
|
|
|
1
|
-
"""
|
|
2
|
-
safety/policy_enforcement.py
|
|
3
|
-
|
|
4
|
-
Content-level policy, applied AFTER a tool runs, to its output — NOT
|
|
5
|
-
access control (that's security/permissions.py's job, applied BEFORE a
|
|
6
|
-
tool runs, deciding whether it runs at all). registry.py's dispatch()
|
|
7
|
-
calls check_output_policy() on every tool result, regardless of which
|
|
8
|
-
tool produced it.
|
|
9
|
-
|
|
10
|
-
Three responsibilities:
|
|
11
|
-
1. Centralized URL policy — is_url_permitted() answers "may this URL be
|
|
12
|
-
requested at all", composing the blocked-domain list with a
|
|
13
|
-
non-public-address guard. web_search.py and fetch_url.py both call
|
|
14
|
-
it, so a URL cannot be permitted for one and refused by the other.
|
|
15
|
-
is_domain_blocked() remains the domain half on its own.
|
|
16
|
-
2. Secret redaction — defense-in-depth: if a tool's output accidentally
|
|
17
|
-
contains something that looks like a leaked API key, it is replaced
|
|
18
|
-
with [REDACTED] before the content reaches storage or the model.
|
|
19
|
-
3. Argument policy (§25) — the symmetric half of (2), applied BEFORE a
|
|
20
|
-
tool runs, to what it was asked to do rather than to what it returned.
|
|
21
|
-
Refuses rather than redacts, and refuses on the blocked-domain list
|
|
22
|
-
too, which until then only the two tools that opted in enforced.
|
|
23
|
-
|
|
24
|
-
(1) and (3) overlap deliberately: (3) is the choke point every tool
|
|
25
|
-
passes through, (1) is what a tool checks about a value only it can
|
|
26
|
-
interpret. Neither makes the other redundant.
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
from __future__ import annotations
|
|
30
|
-
|
|
31
|
-
import ipaddress
|
|
32
|
-
import json
|
|
33
|
-
import logging
|
|
34
|
-
import
|
|
35
|
-
import
|
|
36
|
-
import
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
"
|
|
62
|
-
"
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
the
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
""
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
#
|
|
88
|
-
|
|
89
|
-
host =
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
host
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
#
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
#
|
|
211
|
-
|
|
212
|
-
#
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
#
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
#
|
|
239
|
-
|
|
240
|
-
#
|
|
241
|
-
|
|
242
|
-
#
|
|
243
|
-
|
|
244
|
-
#
|
|
245
|
-
|
|
246
|
-
#
|
|
247
|
-
|
|
248
|
-
#
|
|
249
|
-
|
|
250
|
-
#
|
|
251
|
-
#
|
|
252
|
-
#
|
|
253
|
-
#
|
|
254
|
-
#
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
|
|
260
|
-
#
|
|
261
|
-
#
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
#
|
|
273
|
-
#
|
|
274
|
-
#
|
|
275
|
-
|
|
276
|
-
#
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
#
|
|
281
|
-
|
|
282
|
-
#
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
#
|
|
287
|
-
# the
|
|
288
|
-
#
|
|
289
|
-
#
|
|
290
|
-
#
|
|
291
|
-
|
|
292
|
-
#
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
""
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
#
|
|
379
|
-
#
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
def
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
#
|
|
634
|
-
#
|
|
635
|
-
#
|
|
636
|
-
#
|
|
637
|
-
#
|
|
638
|
-
#
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
1
|
+
"""
|
|
2
|
+
safety/policy_enforcement.py
|
|
3
|
+
|
|
4
|
+
Content-level policy, applied AFTER a tool runs, to its output — NOT
|
|
5
|
+
access control (that's security/permissions.py's job, applied BEFORE a
|
|
6
|
+
tool runs, deciding whether it runs at all). registry.py's dispatch()
|
|
7
|
+
calls check_output_policy() on every tool result, regardless of which
|
|
8
|
+
tool produced it.
|
|
9
|
+
|
|
10
|
+
Three responsibilities:
|
|
11
|
+
1. Centralized URL policy — is_url_permitted() answers "may this URL be
|
|
12
|
+
requested at all", composing the blocked-domain list with a
|
|
13
|
+
non-public-address guard. web_search.py and fetch_url.py both call
|
|
14
|
+
it, so a URL cannot be permitted for one and refused by the other.
|
|
15
|
+
is_domain_blocked() remains the domain half on its own.
|
|
16
|
+
2. Secret redaction — defense-in-depth: if a tool's output accidentally
|
|
17
|
+
contains something that looks like a leaked API key, it is replaced
|
|
18
|
+
with [REDACTED] before the content reaches storage or the model.
|
|
19
|
+
3. Argument policy (§25) — the symmetric half of (2), applied BEFORE a
|
|
20
|
+
tool runs, to what it was asked to do rather than to what it returned.
|
|
21
|
+
Refuses rather than redacts, and refuses on the blocked-domain list
|
|
22
|
+
too, which until then only the two tools that opted in enforced.
|
|
23
|
+
|
|
24
|
+
(1) and (3) overlap deliberately: (3) is the choke point every tool
|
|
25
|
+
passes through, (1) is what a tool checks about a value only it can
|
|
26
|
+
interpret. Neither makes the other redundant.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import ipaddress
|
|
32
|
+
import json
|
|
33
|
+
import logging
|
|
34
|
+
import re
|
|
35
|
+
import socket
|
|
36
|
+
from urllib.parse import urlparse
|
|
37
|
+
|
|
38
|
+
# safety -> security, and never the reverse. `security.posture` imports
|
|
39
|
+
# `config` and the stdlib only, so this adds no cycle -- which is exactly
|
|
40
|
+
# why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
|
|
41
|
+
# this module, and the edge only stays acyclic while it runs one way.
|
|
42
|
+
from security import posture
|
|
43
|
+
from security import secrets as harness_secrets
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
# ---------------------------------------------------------------------------
|
|
48
|
+
# ---- Blocked domains ------------------------------------------------------
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
# Centralized blocked-domain list. web_search.py and fetch_url.py import
|
|
52
|
+
# is_domain_blocked() to enforce this. Add domains hosting malware,
|
|
53
|
+
# phishing, or other harmful content based on your threat model.
|
|
54
|
+
#
|
|
55
|
+
# MAINTENANCE: domains change ownership, get taken down, or are repurposed.
|
|
56
|
+
# Review this list periodically. Remove domains that are no longer harmful;
|
|
57
|
+
# add new ones as threats emerge. An empty set disables domain filtering.
|
|
58
|
+
BLOCKED_DOMAINS: set[str] = {
|
|
59
|
+
# Well-known harmful domains — examples only, maintain as needed.
|
|
60
|
+
"malware-traffic-analysis.net", # malware sample distribution
|
|
61
|
+
"exploit-db.com", # exploit code repository
|
|
62
|
+
"packetstormsecurity.com", # exploit/tool archive
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _hostname(url: str) -> str:
|
|
67
|
+
"""The host of a full URL or a bare domain, normalised for comparison.
|
|
68
|
+
|
|
69
|
+
ONE normaliser, because the two branches below had drifted apart and
|
|
70
|
+
the difference was a bypass (#48). `.hostname` already lowercases,
|
|
71
|
+
strips the port and strips userinfo; the bare-domain branch did none
|
|
72
|
+
of that except by accident, so `exploit-db.com:443` passed it while
|
|
73
|
+
the URL form of the same host was caught.
|
|
74
|
+
|
|
75
|
+
The trailing dot defeated BOTH branches. `exploit-db.com.` is a legal,
|
|
76
|
+
resolvable FQDN naming exactly the same host, `urlparse` keeps the
|
|
77
|
+
dot, and no amount of lowercasing makes the strings equal. Stripped
|
|
78
|
+
here rather than at each call site, since every caller compares.
|
|
79
|
+
"""
|
|
80
|
+
if "://" in url:
|
|
81
|
+
try:
|
|
82
|
+
host = urlparse(url).hostname or ""
|
|
83
|
+
except Exception:
|
|
84
|
+
host = url
|
|
85
|
+
else:
|
|
86
|
+
# Strip path, then userinfo, then port -- the three things
|
|
87
|
+
# .hostname does for the URL branch and this one used not to.
|
|
88
|
+
host = url.split("/")[0]
|
|
89
|
+
host = host.rpartition("@")[2]
|
|
90
|
+
if host.startswith("["): # [::1]:8080
|
|
91
|
+
host = host.partition("]")[0].lstrip("[")
|
|
92
|
+
else:
|
|
93
|
+
host = host.partition(":")[0]
|
|
94
|
+
return host.lower().rstrip(".")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def is_domain_blocked(url: str) -> bool:
|
|
98
|
+
"""Return True if the URL's host is in, or under, BLOCKED_DOMAINS.
|
|
99
|
+
|
|
100
|
+
Handles full URLs (https://example.com/path) and bare domains
|
|
101
|
+
(example.com), through one normaliser -- see `_hostname`.
|
|
102
|
+
|
|
103
|
+
Subdomains ARE matched: blocking ``evil.com`` also blocks
|
|
104
|
+
``sub.evil.com``. This reverses a documented carve-out, and the
|
|
105
|
+
reversal is the point. A blocklist an attacker leaves by prepending
|
|
106
|
+
one label is close to decorative, and the old docstring's advice
|
|
107
|
+
("block ``evil.com`` explicitly if you also want ``sub.evil.com``")
|
|
108
|
+
asked a maintainer to enumerate an infinite set. Matching a suffix
|
|
109
|
+
can only ever block MORE than the list's author wrote down, and this
|
|
110
|
+
list's whole purpose is refusal, so over-blocking is the safe
|
|
111
|
+
direction to be wrong in.
|
|
112
|
+
|
|
113
|
+
The match is label-wise (`host == blocked` or `host` ends with
|
|
114
|
+
`"." + blocked`), never a bare string suffix: `notevil.com` must not
|
|
115
|
+
match `evil.com`.
|
|
116
|
+
"""
|
|
117
|
+
if not BLOCKED_DOMAINS:
|
|
118
|
+
return False
|
|
119
|
+
host = _hostname(url)
|
|
120
|
+
return any(host == blocked or host.endswith("." + blocked)
|
|
121
|
+
for blocked in (b.lower().rstrip(".") for b in BLOCKED_DOMAINS))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def is_url_permitted(url: str, resolve: bool = True) -> str | None:
|
|
125
|
+
"""Refusal reason for a URL, or None if it may be requested.
|
|
126
|
+
|
|
127
|
+
ONE checker, two callers -- `fetch_url` before every hop it issues,
|
|
128
|
+
and `web_search` over the results it is about to hand the model. They
|
|
129
|
+
were asking different questions about the same policy, which is how
|
|
130
|
+
#53 and #54 stayed open while #48 looked like the whole of it.
|
|
131
|
+
|
|
132
|
+
Three gates, cheapest first:
|
|
133
|
+
|
|
134
|
+
1. Scheme. http/https only. Moved here from fetch_url so the answer
|
|
135
|
+
does not depend on which caller asked.
|
|
136
|
+
2. `is_domain_blocked`.
|
|
137
|
+
3. Address. Rejects anything that is not a GLOBAL address --
|
|
138
|
+
loopback, private, link-local, reserved and multicast in one
|
|
139
|
+
predicate. `169.254.169.254` is the one that matters most: on a
|
|
140
|
+
cloud instance with IMDSv1 it returns role credentials, and the
|
|
141
|
+
route to it is a fetched page saying "for full details see
|
|
142
|
+
<that URL>" to a grounding pass that obliges.
|
|
143
|
+
|
|
144
|
+
`resolve=False` answers from the literal text only -- an IP literal
|
|
145
|
+
is still checked, in every spelling the platform's resolver reads as
|
|
146
|
+
a number (#207: `2130706433` and `0x7f.0.0.1` too, not only dotted
|
|
147
|
+
quads), and a NAME is taken on trust. That is for `web_search`,
|
|
148
|
+
which filters results rather than fetching them: a DNS lookup per
|
|
149
|
+
search hit is real latency for a check whose failure mode is "the
|
|
150
|
+
model is shown a URL that fetch_url will refuse anyway". The caller
|
|
151
|
+
that actually opens a socket always resolves.
|
|
152
|
+
|
|
153
|
+
KNOWN LIMITATION -- DNS rebinding. When we resolve, httpx then
|
|
154
|
+
resolves again independently, so a host whose records change between
|
|
155
|
+
this check and the connect can still be reached. Closing it means
|
|
156
|
+
connecting to the pinned IP and passing the original host in a
|
|
157
|
+
`Host:` header, which on HTTPS fights SNI and certificate
|
|
158
|
+
verification -- a subtly wrong implementation there is a worse hole
|
|
159
|
+
than this one. Recorded rather than silently accepted; the attacker
|
|
160
|
+
needs control of DNS for a host the model was already induced to
|
|
161
|
+
fetch.
|
|
162
|
+
"""
|
|
163
|
+
if not (url.startswith("http://") or url.startswith("https://")):
|
|
164
|
+
return "URL must start with http:// or https://"
|
|
165
|
+
|
|
166
|
+
if is_domain_blocked(url):
|
|
167
|
+
return f"Domain is blocked by policy: {url}"
|
|
168
|
+
|
|
169
|
+
host = _hostname(url)
|
|
170
|
+
if not host:
|
|
171
|
+
return f"URL has no host: {url}"
|
|
172
|
+
|
|
173
|
+
try:
|
|
174
|
+
literal = ipaddress.ip_address(host)
|
|
175
|
+
except ValueError:
|
|
176
|
+
literal = None
|
|
177
|
+
|
|
178
|
+
if literal is None and not resolve:
|
|
179
|
+
# #207, batch 213. `ipaddress` reads dotted quads and IPv6 only, so
|
|
180
|
+
# `2130706433`, `0x7f.0.0.1` and `127.1` -- all 127.0.0.1 to the
|
|
181
|
+
# platform's resolver -- were taken on trust as NAMES here, and
|
|
182
|
+
# only fetch_url refused them. AI_NUMERICHOST asks that resolver to
|
|
183
|
+
# read the text as a number and never to look anything up, so it
|
|
184
|
+
# costs no network, which was the whole reason for resolve=False.
|
|
185
|
+
# Measured on Windows and in WSL: the same spellings parse, and a
|
|
186
|
+
# name fails with gaierror.
|
|
187
|
+
try:
|
|
188
|
+
literal = ipaddress.ip_address(socket.getaddrinfo(
|
|
189
|
+
host, None, flags=socket.AI_NUMERICHOST)[0][4][0])
|
|
190
|
+
except (socket.gaierror, ValueError, IndexError):
|
|
191
|
+
literal = None
|
|
192
|
+
|
|
193
|
+
if literal is not None:
|
|
194
|
+
addresses = [literal]
|
|
195
|
+
elif resolve:
|
|
196
|
+
try:
|
|
197
|
+
addresses = [
|
|
198
|
+
ipaddress.ip_address(info[4][0])
|
|
199
|
+
for info in socket.getaddrinfo(host, None)
|
|
200
|
+
]
|
|
201
|
+
except (socket.gaierror, ValueError) as e:
|
|
202
|
+
# Fail CLOSED. A name we cannot resolve is one we cannot
|
|
203
|
+
# clear, and httpx would only fail on it a moment later --
|
|
204
|
+
# refusing here costs nothing and keeps "unchecked" from
|
|
205
|
+
# ever meaning "allowed".
|
|
206
|
+
return f"Could not resolve host for policy check: {host} ({e})"
|
|
207
|
+
else:
|
|
208
|
+
addresses = []
|
|
209
|
+
|
|
210
|
+
# ANY, not all. A name resolving to one public and one private
|
|
211
|
+
# address is the rebinding shape, and taking the first answer or
|
|
212
|
+
# requiring unanimity would both let it through.
|
|
213
|
+
for address in addresses:
|
|
214
|
+
if not address.is_global:
|
|
215
|
+
return (f"URL resolves to a non-public address, which this "
|
|
216
|
+
f"harness will not fetch: {host} -> {address}")
|
|
217
|
+
|
|
218
|
+
return None
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# ---------------------------------------------------------------------------
|
|
222
|
+
# ---- Secret redaction -----------------------------------------------------
|
|
223
|
+
# ---------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
# A PEM body's line break: a real one, or the two-character escape a JSON
|
|
226
|
+
# string carries -- a cloud service-account key file is exactly that shape.
|
|
227
|
+
# Unambiguous on purpose: every repetition holds a mandatory break, so the
|
|
228
|
+
# spaces between two breaks belong to exactly one repetition and a body
|
|
229
|
+
# that fails to close backtracks in linear time, not exponential.
|
|
230
|
+
_PEM_BREAK = r"(?:[ \t]*(?:\r?\n|\\r\\n|\\n))"
|
|
231
|
+
_PEM_SEP = _PEM_BREAK + r"+[ \t]*"
|
|
232
|
+
# One body line: base64, at least 16 characters, so a word of prose after a
|
|
233
|
+
# quoted header is not one; or the two RFC 1421 headers a legacy encrypted
|
|
234
|
+
# key carries before its base64.
|
|
235
|
+
_PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
|
|
236
|
+
|
|
237
|
+
_SECRET_PATTERNS = [
|
|
238
|
+
# OpenAI-style keys
|
|
239
|
+
re.compile(r"sk-[a-zA-Z0-9]{20,}"),
|
|
240
|
+
# Anthropic-style keys
|
|
241
|
+
re.compile(r"sk-ant-[a-zA-Z0-9\-]{20,}"),
|
|
242
|
+
# GitHub personal access tokens
|
|
243
|
+
re.compile(r"ghp_[a-zA-Z0-9]{36}"),
|
|
244
|
+
# AWS access key IDs
|
|
245
|
+
re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
246
|
+
# Google API keys
|
|
247
|
+
re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
|
|
248
|
+
# Slack tokens
|
|
249
|
+
re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
|
|
250
|
+
# PEM private keys: the header, and the WHOLE BLOCK when it is there
|
|
251
|
+
# (TECHNICAL_DEBT 52, batch 160). This used to be the header line alone,
|
|
252
|
+
# so the base64 body -- the key material -- survived verbatim under a
|
|
253
|
+
# `[REDACTED]` that read as the key having been removed. The body is
|
|
254
|
+
# OPTIONAL in the pattern on purpose: `_input_leaf` uses `.search()`,
|
|
255
|
+
# and anything carrying the header still matches, so input refusals are
|
|
256
|
+
# exactly what they were. The body is LINES, each after a line break:
|
|
257
|
+
# a block with its END line goes whole, and one a cut ended goes as far
|
|
258
|
+
# as its lines do. Lines rather than "anything up to END" because prose
|
|
259
|
+
# quotes both markers -- "starts with -----BEGIN ... and ends with
|
|
260
|
+
# -----END ..." -- and a lazy match took the sentence between them.
|
|
261
|
+
# Measured: that sentence now loses its two markers and keeps its words.
|
|
262
|
+
re.compile(
|
|
263
|
+
r"-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
|
264
|
+
r"(?:(?:" + _PEM_SEP + _PEM_UNIT + r")*"
|
|
265
|
+
r"(?:" + _PEM_SEP + r"[A-Za-z0-9+/=]{1,15})?"
|
|
266
|
+
r"(?:" + _PEM_BREAK + r"*[ \t]*)-----END [A-Z ]*PRIVATE KEY-----"
|
|
267
|
+
r"|(?:" + _PEM_SEP + _PEM_UNIT + r")*)"),
|
|
268
|
+
# --- Batch 160, TECHNICAL_DEBT 52: the CURRENT siblings of the vendor
|
|
269
|
+
# --- formats above, each of which the older pattern beside it misses
|
|
270
|
+
# --- by a character class -- a `_` or a second `-` the 2024 shapes
|
|
271
|
+
# --- never had. Measured through the real `redact_output_text` before
|
|
272
|
+
# --- this: every one survived. They refuse inputs too, as `ghp_` and
|
|
273
|
+
# --- `sk-` always have; one vendor list, one vocabulary.
|
|
274
|
+
# GitHub fine-grained PATs, the format `ghp_` is being retired for
|
|
275
|
+
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
276
|
+
# GitHub OAuth, user-to-server, server-to-server and refresh tokens
|
|
277
|
+
re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
|
|
278
|
+
# OpenAI project, service-account and admin keys: `sk-proj-...`
|
|
279
|
+
re.compile(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{20,}"),
|
|
280
|
+
# Stripe live secret and restricted keys
|
|
281
|
+
re.compile(r"[sr]k_live_[A-Za-z0-9]{16,}"),
|
|
282
|
+
# npm access tokens
|
|
283
|
+
re.compile(r"npm_[A-Za-z0-9]{36}"),
|
|
284
|
+
# --- ROADMAP_v3 §79 (HD5): the three credentials `/connect` hands the
|
|
285
|
+
# --- harness that no pattern above can see. A sign-in puts them in a
|
|
286
|
+
# --- session the model shares, so they are redacted and refused like
|
|
287
|
+
# --- the rest of the list.
|
|
288
|
+
# Hugging Face user access tokens and OAuth access tokens. An
|
|
289
|
+
# identifier like `hf_hub_download` breaks the run of word characters
|
|
290
|
+
# at its second underscore and is not matched.
|
|
291
|
+
re.compile(r"(?<![A-Za-z0-9])hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
|
|
292
|
+
# OpenRouter keys, which `sk-` above misses at the second hyphen
|
|
293
|
+
re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
|
|
294
|
+
# ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
|
|
295
|
+
# bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
|
|
296
|
+
# that merely ENDS a word like `items-<uuid>`.
|
|
297
|
+
re.compile(r"(?<![A-Za-z0-9])ms-[0-9a-f]{8}-[0-9a-f]{4}"
|
|
298
|
+
r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
|
|
299
|
+
]
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
#: What this module puts where a secret was. Named in batch 65 rather
|
|
303
|
+
#: than written out at each of the four substitutions, because a second
|
|
304
|
+
#: module now has to RECOGNISE it: `tui/markdown.py` refuses to arm a
|
|
305
|
+
#: URL wearing this mark for a click, since a redacted address is a real
|
|
306
|
+
#: one and simply not the one that was fetched. That module is pure and
|
|
307
|
+
#: may not import policy, so it keeps its own copy and
|
|
308
|
+
#: `tests/test_markdown_render.py` holds the two against each other --
|
|
309
|
+
#: one string, two readers, and a check rather than a hope.
|
|
310
|
+
REDACTION_MARKER = "[REDACTED]"
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def redact_secrets(text: str) -> str:
|
|
314
|
+
"""Replace strings matching known secret patterns with [REDACTED]."""
|
|
315
|
+
for pattern in _SECRET_PATTERNS:
|
|
316
|
+
text = pattern.sub(REDACTION_MARKER, text)
|
|
317
|
+
return text
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
# ---------------------------------------------------------------------------
|
|
321
|
+
# ---- Credential shapes (#167, batch 20) ------------------------------------
|
|
322
|
+
# ---------------------------------------------------------------------------
|
|
323
|
+
#
|
|
324
|
+
# The patterns above are VENDOR TOKENS -- every one has a prefix some
|
|
325
|
+
# company mints, so a match is high-confidence and a bare `[REDACTED]` loses
|
|
326
|
+
# nothing structural. A credential that is just a password in a field has no
|
|
327
|
+
# prefix, and nothing matched it (#167): batch 14 added five build manifests
|
|
328
|
+
# to read_project_doc's allowlist (#94), whose credential conventions are
|
|
329
|
+
# exactly the three unscrubbed shapes below.
|
|
330
|
+
#
|
|
331
|
+
# These are CONTEXT shapes, not tokens: each anchors on a credential FIELD
|
|
332
|
+
# NAME (or URL userinfo), so an auth error code, a stack frame or an ARN
|
|
333
|
+
# does not match, and the replacement is SURGICAL -- tag names, field names,
|
|
334
|
+
# usernames and hosts survive, only the value goes. That structure is why
|
|
335
|
+
# the false-positive cost is acceptable: documentation quoting an example
|
|
336
|
+
# keeps its shape and loses only its example value.
|
|
337
|
+
#
|
|
338
|
+
# OUTPUT REDACTION ONLY. These never feed check_input_policy: refusing a
|
|
339
|
+
# call because its arguments mention `password = "..."` would break every
|
|
340
|
+
# legitimate call that carries build-file content the user asked for --
|
|
341
|
+
# _SECRET_PATTERNS stay the input side's whole vocabulary. And the keyword
|
|
342
|
+
# set is password|passwd ONLY, deliberately not token/secret/key: those
|
|
343
|
+
# appear constantly in ordinary prose ("token bucket", JWT discussion) and
|
|
344
|
+
# would redact aggressively.
|
|
345
|
+
|
|
346
|
+
# Template placeholders survive untouched: `${PASSWORD}`, `$DB_PASS`,
|
|
347
|
+
# `{{ secrets.DB_PW }}`, `<your-password-here>` name where a credential
|
|
348
|
+
# WOULD go; they are what docs and templates contain, and redacting them
|
|
349
|
+
# destroys the example while protecting nothing.
|
|
350
|
+
_TEMPLATE_VALUE_RE = re.compile(
|
|
351
|
+
r"^(?:"
|
|
352
|
+
r"\$\{[^}]*\}" # ${PASSWORD}, ${secrets.DB}
|
|
353
|
+
r"|\$[A-Za-z_][A-Za-z0-9_]*" # $PASSWORD
|
|
354
|
+
r"|\{\{[^}]*\}\}" # {{ secrets.DB_PW }}
|
|
355
|
+
r"|<[^<>]+>" # <your-password-here>
|
|
356
|
+
r")$")
|
|
357
|
+
|
|
358
|
+
# scheme://user:secret@host -> scheme://user:[REDACTED]@host. Credentials
|
|
359
|
+
# in a URL's userinfo are never anything else, which makes this the cheapest
|
|
360
|
+
# of the three; anchoring on `//` keeps bare `user:pass@host` fragments in
|
|
361
|
+
# prose alone, and excluding `/ \s : @` from both sides means a host:port
|
|
362
|
+
# path (`https://example.com:8080/x`) cannot masquerade as userinfo.
|
|
363
|
+
_URL_USERINFO_RE = re.compile(r"(//[^/\s:@]+:)([^/\s@]+)(@)")
|
|
364
|
+
|
|
365
|
+
# <password>s3cr3t</password> -> <password>[REDACTED]</password>, POM and
|
|
366
|
+
# settings.xml convention. The backreference keeps the closing tag honest;
|
|
367
|
+
# the value excludes `<`, so nested markup cannot be swallowed. Groups:
|
|
368
|
+
# 1 opening tag, 2 tag name, 3 value.
|
|
369
|
+
_XML_PASSWORD_RE = re.compile(
|
|
370
|
+
r"(?i)(<(password|passwd)>)([^<]{4,200})(</\2>)")
|
|
371
|
+
|
|
372
|
+
# password = "hunter2" / password: "hunter2" / password "hunter2" ->
|
|
373
|
+
# the value becomes [REDACTED] inside its quotes. The separator accepts
|
|
374
|
+
# `=`, `:` AND Gradle's bare-space credentials-block form -- which is the
|
|
375
|
+
# issue's own headline case and which a `[=:]`-only sketch misses. Quoted
|
|
376
|
+
# values only, minimum 4 chars: unquoted .properties-style values are out
|
|
377
|
+
# of scope by decision, and the value excludes newlines so a quote further
|
|
378
|
+
# down a file cannot close a match opened lines above. Groups: 1 keyword,
|
|
379
|
+
# 2 separator, 3 opening quote, 4 value.
|
|
380
|
+
_ASSIGNMENT_RE = re.compile(
|
|
381
|
+
r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
|
|
382
|
+
r"([\"'])([^\"'\n]{4,200})\3")
|
|
383
|
+
|
|
384
|
+
# --- Batch 160, TECHNICAL_DEBT 52: three STRUCTURAL shapes, output only by
|
|
385
|
+
# --- owner ruling. Each carries a credential with no vendor prefix, and
|
|
386
|
+
# --- each anchors on structure a security paper does not produce by
|
|
387
|
+
# --- accident, which is the false-positive argument the password shapes
|
|
388
|
+
# --- above already make.
|
|
389
|
+
|
|
390
|
+
# Authorization: Bearer abc... -> Authorization: Bearer [REDACTED]. Anchored
|
|
391
|
+
# on the HEADER NAME, never on the scheme word: "Bearer tokens expire" is
|
|
392
|
+
# prose and must survive, and that is the keyword-widening the comment above
|
|
393
|
+
# refuses. The separator admits a header line, a JSON or dict key (`"..":`)
|
|
394
|
+
# and `=`, so a curl line, a captured request and a logged dict all match.
|
|
395
|
+
# The value's alphabet has no `$ < > { } [`, so `Bearer $TOKEN`,
|
|
396
|
+
# `Bearer <token>` and an already-redacted value never match, and
|
|
397
|
+
# `Digest username=...`, whose secret is not one value, is left alone. A
|
|
398
|
+
# value with NO scheme word must also be 16 characters and carry a digit,
|
|
399
|
+
# because API documentation writes "Authorization: required" and a bare
|
|
400
|
+
# 8-character floor redacted the word.
|
|
401
|
+
# Groups: 1 header name, 2 separator, 3 scheme (or None).
|
|
402
|
+
_AUTH_HEADER_RE = re.compile(
|
|
403
|
+
r"(?i)\b((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
|
|
404
|
+
r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
|
|
405
|
+
r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
|
|
406
|
+
|
|
407
|
+
# A JSON Web Token, whole: three base64url segments, the first two of
|
|
408
|
+
# which open with `eyJ` because both are JSON objects (`{"`). A bearer
|
|
409
|
+
# credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
|
|
410
|
+
# reach when the token is pasted without its header.
|
|
411
|
+
_JWT_RE = re.compile(
|
|
412
|
+
r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
|
|
413
|
+
|
|
414
|
+
# The END of a PEM key block whose BEGIN is not in this string: the footer
|
|
415
|
+
# and the run of base64 lines above it. The vendor pattern takes a block
|
|
416
|
+
# from its header; this is the other half, for text a cut started in the
|
|
417
|
+
# middle of a block -- a page, a session's tail as its start sweeps through
|
|
418
|
+
# the output. Found as a literal and walked BACKWARDS in code, because a
|
|
419
|
+
# regex that has to start at the first body line is tried at every line
|
|
420
|
+
# start of a long base64 blob and is quadratic on one with no footer.
|
|
421
|
+
_PEM_END_RE = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
|
|
422
|
+
_PEM_BODY_CHARS = frozenset(
|
|
423
|
+
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _before_pem_break(text: str, i: int):
|
|
427
|
+
"""The index before the line break(s) ending at *i*, or None.
|
|
428
|
+
|
|
429
|
+
`_PEM_SEP` read backwards: spaces, then one or more breaks (real, or
|
|
430
|
+
the JSON escape), each with its own spaces."""
|
|
431
|
+
k = i
|
|
432
|
+
while k and text[k - 1] in " \t":
|
|
433
|
+
k -= 1
|
|
434
|
+
seen = False
|
|
435
|
+
while True:
|
|
436
|
+
if k and text[k - 1] == "\n":
|
|
437
|
+
k -= 1
|
|
438
|
+
if k and text[k - 1] == "\r":
|
|
439
|
+
k -= 1
|
|
440
|
+
elif k >= 2 and text[k - 2:k] == "\\n":
|
|
441
|
+
k -= 2
|
|
442
|
+
if k >= 2 and text[k - 2:k] == "\\r":
|
|
443
|
+
k -= 2
|
|
444
|
+
else:
|
|
445
|
+
break
|
|
446
|
+
seen = True
|
|
447
|
+
while k and text[k - 1] in " \t":
|
|
448
|
+
k -= 1
|
|
449
|
+
return k if seen else None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _redact_pem_tails(text: str) -> str:
|
|
453
|
+
"""Replace each orphan footer, and the base64 lines above it, whole.
|
|
454
|
+
|
|
455
|
+
Linear: each walk stops at the first character outside the body
|
|
456
|
+
alphabet, and a footer's own `-` is one, so no two walks share text."""
|
|
457
|
+
pieces, last = [], 0
|
|
458
|
+
for m in _PEM_END_RE.finditer(text):
|
|
459
|
+
start = i = m.start()
|
|
460
|
+
while True:
|
|
461
|
+
k = _before_pem_break(text, i)
|
|
462
|
+
if k is None:
|
|
463
|
+
break
|
|
464
|
+
r = k
|
|
465
|
+
while r > last and text[r - 1] in _PEM_BODY_CHARS:
|
|
466
|
+
r -= 1
|
|
467
|
+
if r == k:
|
|
468
|
+
break
|
|
469
|
+
start = i = r
|
|
470
|
+
pieces.append(text[last:start])
|
|
471
|
+
pieces.append(REDACTION_MARKER)
|
|
472
|
+
last = m.end()
|
|
473
|
+
pieces.append(text[last:])
|
|
474
|
+
return "".join(pieces)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _redact_credential_shapes(text: str) -> str:
|
|
478
|
+
"""Apply the context and structural shapes, preserving surrounding
|
|
479
|
+
structure.
|
|
480
|
+
|
|
481
|
+
Each replacer checks the captured VALUE against the placeholder shapes
|
|
482
|
+
before substituting -- a template reference is exactly what docs and
|
|
483
|
+
build files are full of, and redacting it destroys the example while
|
|
484
|
+
protecting nothing."""
|
|
485
|
+
text = _URL_USERINFO_RE.sub(rf"\1{REDACTION_MARKER}\3", text)
|
|
486
|
+
|
|
487
|
+
def _xml(m):
|
|
488
|
+
if _TEMPLATE_VALUE_RE.match(m.group(3)):
|
|
489
|
+
return m.group(0)
|
|
490
|
+
return f"{m.group(1)}{REDACTION_MARKER}{m.group(4)}"
|
|
491
|
+
|
|
492
|
+
text = _XML_PASSWORD_RE.sub(_xml, text)
|
|
493
|
+
|
|
494
|
+
def _assign(m):
|
|
495
|
+
if _TEMPLATE_VALUE_RE.match(m.group(4)):
|
|
496
|
+
return m.group(0)
|
|
497
|
+
return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
|
|
498
|
+
f"{REDACTION_MARKER}{m.group(3)}")
|
|
499
|
+
|
|
500
|
+
text = _ASSIGNMENT_RE.sub(_assign, text)
|
|
501
|
+
|
|
502
|
+
text = _AUTH_HEADER_RE.sub(
|
|
503
|
+
lambda m: (f"{m.group(1)}{m.group(2)}{m.group(3) or ''}"
|
|
504
|
+
f"{REDACTION_MARKER}"), text)
|
|
505
|
+
text = _JWT_RE.sub(REDACTION_MARKER, text)
|
|
506
|
+
return _redact_pem_tails(text)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def redaction_enabled() -> bool:
|
|
510
|
+
"""Whether pattern-based redaction of tool output is active.
|
|
511
|
+
|
|
512
|
+
Two switches, one question. config.REDACT_TOOL_OUTPUTS is the
|
|
513
|
+
permanent answer; VENASTINE_REDACT_OFF weakens it for one run --
|
|
514
|
+
a debugging session that needs the model to see real values --
|
|
515
|
+
without editing anything persistent. The environment can only ever
|
|
516
|
+
turn redaction OFF, never back on past the constant: an off-by-default
|
|
517
|
+
protection an environment variable could silently re-enable would not
|
|
518
|
+
be a switch but a coin flip.
|
|
519
|
+
|
|
520
|
+
§40 (UN1): BOTH switches are now read from the frozen posture rather
|
|
521
|
+
than live, and the environment half is the reason that mattered here.
|
|
522
|
+
`config` binds at import so mutating the environment could never move
|
|
523
|
+
the constant -- but `os.environ.get(...)` at call time followed a
|
|
524
|
+
mutation, so this one function was a live in-process switch for
|
|
525
|
+
redaction. The distinction above survives as two posture fields; only
|
|
526
|
+
the moment of reading moved.
|
|
527
|
+
|
|
528
|
+
What this does NOT govern, whichever way it reads: input refusals
|
|
529
|
+
(_input_leaf -- a denial is legible, not destructive), the depth-cap
|
|
530
|
+
substitution (fail-closed structural bound), and `redact_diagnostic_text`
|
|
531
|
+
-- the harness's own diagnostics, which guard app.log, the transcript,
|
|
532
|
+
`MessageLog.error` and a pipeline record regardless of this answer. That
|
|
533
|
+
last clause used to name `logging_setup` alone and to be true only about
|
|
534
|
+
the moment of reading: the formatter did run regardless, with a strictly
|
|
535
|
+
SMALLER vocabulary than this function's, which is the gap batch 144
|
|
536
|
+
closed. The two now differ in one respect and it is the one named here.
|
|
537
|
+
"""
|
|
538
|
+
active = posture.current()
|
|
539
|
+
if not active.redact_tool_outputs:
|
|
540
|
+
return False
|
|
541
|
+
return not active.redact_off_env
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def redact_diagnostic_text(text: str) -> str:
|
|
545
|
+
"""The one redaction path the HARNESS'S OWN DIAGNOSTICS take.
|
|
546
|
+
|
|
547
|
+
THE SAME VOCABULARY AS TOOL OUTPUT, which it was not until batch 144
|
|
548
|
+
(batch 144). `redact_secrets` is the seven vendor-token patterns and
|
|
549
|
+
nothing else. `_redact_credential_shapes` is the only thing in this
|
|
550
|
+
project that matches a credential with NO vendor prefix -- a URL's
|
|
551
|
+
userinfo, a `<password>` element, a quoted `password =` assignment --
|
|
552
|
+
and it had exactly one caller, `redact_output_text`. So one string took
|
|
553
|
+
two different vocabularies depending on which sink it reached. Driven
|
|
554
|
+
through the real `configure_logging` with nothing monkeypatched, on
|
|
555
|
+
issue #167's own seven cases:
|
|
556
|
+
|
|
557
|
+
case app.log transcript tool output
|
|
558
|
+
anthropic key redacted redacted redacted
|
|
559
|
+
openai key redacted redacted redacted
|
|
560
|
+
github pat redacted redacted redacted
|
|
561
|
+
aws access key id redacted redacted redacted
|
|
562
|
+
gradle password LEAKED LEAKED redacted
|
|
563
|
+
maven server password LEAKED LEAKED redacted
|
|
564
|
+
gem source url LEAKED LEAKED redacted
|
|
565
|
+
|
|
566
|
+
The four vendor rows are the control: they prove the formatter was
|
|
567
|
+
attached and firing, so the three LEAKED rows were the vocabulary and
|
|
568
|
+
not a probe that had stopped working. It needed no contrivance to
|
|
569
|
+
reach: `fetch_url` logs the URL it failed on BY DESIGN and says so in
|
|
570
|
+
a comment naming this redaction as the guard, and `core/loop.py` puts
|
|
571
|
+
the same description in `MessageLog.error`, where it persists in
|
|
572
|
+
`app.db` -- which `docs/PRIVACY.md` says is deleted by wiping the
|
|
573
|
+
database rather than one thread.
|
|
574
|
+
|
|
575
|
+
FIVE CALLERS, and the one-caller shape is what made the gap
|
|
576
|
+
survivable: `logging_setup`'s formatter, `tui/app.py`'s
|
|
577
|
+
`TranscriptLogHandler.emit` and its tool-error line, `core/loop.py`'s
|
|
578
|
+
`mark_turn_failed`, and the orchestrator's pipeline event. Adding a
|
|
579
|
+
pattern or a shape now reaches all five, and
|
|
580
|
+
`tests/test_logging_setup.py`'s parity test is what fails if a sixth
|
|
581
|
+
sink composes its own.
|
|
582
|
+
|
|
583
|
+
UNCONDITIONAL, all three passes, and that is not an extension of the
|
|
584
|
+
switch's reach but the removal of a difference. `redaction_enabled()`
|
|
585
|
+
governs what the MODEL is shown, which a user may reasonably choose to
|
|
586
|
+
see raw for one run; it has never governed what `app.log` keeps across
|
|
587
|
+
runs, and the switch's own docstring already said so.
|
|
588
|
+
|
|
589
|
+
LIVE VALUES FIRST, which is the finding rather than a preference
|
|
590
|
+
(batch 144). `redact_live_values` matches a stored value by exact
|
|
591
|
+
substring, so a pattern pass that has already rewritten part of one
|
|
592
|
+
leaves the remainder standing. Measured, with a stored value that
|
|
593
|
+
properly contains a vendor token: the log's old order returned
|
|
594
|
+
`PREFIX-[REDACTED]-SUFFIX` and this one returns `[REDACTED]`. The repo
|
|
595
|
+
has locked the general rule three times -- `fetch_url` redacts before
|
|
596
|
+
it cuts, `param_digest` redacts before truncating, the diff block runs
|
|
597
|
+
on whole texts -- and this is the same rule where the destroyer is a
|
|
598
|
+
substitution rather than a cut.
|
|
599
|
+
|
|
600
|
+
The cost is measured and is per log RECORD: 6.7 us -> 9.4 us on Linux,
|
|
601
|
+
+40%, for three more regex passes over a line that already takes
|
|
602
|
+
seven. Invisible at INFO, and under a millisecond per thousand records
|
|
603
|
+
at DEBUG.
|
|
604
|
+
"""
|
|
605
|
+
return _redact_credential_shapes(
|
|
606
|
+
redact_secrets(harness_secrets.redact_live_values(text)))
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def redact_output_text(text: str) -> str:
|
|
610
|
+
"""The one redaction path tool OUTPUT takes, under one switch.
|
|
611
|
+
|
|
612
|
+
Vendor tokens first, then credential shapes on what survives. Both are
|
|
613
|
+
substitutions of content the user chose to see raw when they flip the
|
|
614
|
+
switch off -- so this is where VENASTINE_REDACT_OFF bites. The depth
|
|
615
|
+
cap is NOT routed through here: it is structure, not content judgment,
|
|
616
|
+
and stays fail-closed unconditionally.
|
|
617
|
+
|
|
618
|
+
PUBLIC since batch 41 (X4). It was private while `param_digest` was
|
|
619
|
+
its only caller outside this module's own dispatch path, and the §41
|
|
620
|
+
diff is the second: a `write` renders the content the tool put on
|
|
621
|
+
disk, which is tool output by any reading, and it replaces the very
|
|
622
|
+
digest line that goes through here. `redact_secrets` alone would have
|
|
623
|
+
redacted LESS than the line it replaced -- the vendor tokens but not
|
|
624
|
+
the credential shapes -- which is a regression dressed as a feature.
|
|
625
|
+
The docstring above already called this "the one path"; it now is one
|
|
626
|
+
for both consumers.
|
|
627
|
+
|
|
628
|
+
AND IT IS NOW `redact_diagnostic_text` UNDER THE SWITCH (batch 144),
|
|
629
|
+
rather than its own copy of the same three passes. The two had drifted
|
|
630
|
+
into different vocabularies AND different orders while reading as
|
|
631
|
+
siblings, which is why the difference that is deliberate is now the
|
|
632
|
+
only one there is."""
|
|
633
|
+
# SS61: THE HARNESS'S OWN SECRETS COME OUT FIRST, AND UNCONDITIONALLY.
|
|
634
|
+
# Above the switch on purpose, and still first inside the shared
|
|
635
|
+
# composition below. `redact_tool_outputs` governs pattern GUESSES
|
|
636
|
+
# about other people's credentials, and a user who turns it off has
|
|
637
|
+
# chosen to see those raw. A value this harness put into a subprocess,
|
|
638
|
+
# coming back out of one, is a different thing: passing it to the model
|
|
639
|
+
# is the harness leaking its OWN secret, and no switch asks for that.
|
|
640
|
+
# It sits beside the depth cap, which this module already keeps
|
|
641
|
+
# fail-closed for the same shape of reason -- it is structure and
|
|
642
|
+
# self-inflicted leakage, not content judgment.
|
|
643
|
+
#
|
|
644
|
+
# The VALUES never come here: security/secrets.py does the substitution
|
|
645
|
+
# itself, so exactly one module holds plaintext and the redactor is not
|
|
646
|
+
# it (fix at the producer).
|
|
647
|
+
if not redaction_enabled():
|
|
648
|
+
return harness_secrets.redact_live_values(text)
|
|
649
|
+
return redact_diagnostic_text(text)
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
#: What a credential is spelled with: base64 and base64url, hex, and the
|
|
653
|
+
#: separators vendor tokens use. A run of these that a cut splits is the
|
|
654
|
+
#: fragment the cut drops, because the redactor runs after the cut and
|
|
655
|
+
#: cannot match half a token. Punctuation that no token contains -- quotes,
|
|
656
|
+
#: `:`, `@`, braces, commas -- ends a run, so a cut through minified JSON
|
|
657
|
+
#: costs one value and not the whole line. Two cuts take it: a shell
|
|
658
|
+
#: session's dropped middle (A72) and an MCP result's bound (A79).
|
|
659
|
+
TOKEN_CHARS = frozenset(
|
|
660
|
+
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-+/=.~")
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def fragment_end(head: str) -> int:
|
|
664
|
+
"""Where the token-character run that ends *head* begins."""
|
|
665
|
+
i = len(head)
|
|
666
|
+
while i and head[i - 1] in TOKEN_CHARS:
|
|
667
|
+
i -= 1
|
|
668
|
+
return i
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def fragment_start(tail: str) -> int:
|
|
672
|
+
"""Where the token-character run that starts *tail* ends."""
|
|
673
|
+
i = 0
|
|
674
|
+
while i < len(tail) and tail[i] in TOKEN_CHARS:
|
|
675
|
+
i += 1
|
|
676
|
+
return i
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# How much of one tool parameter reaches a digest, and how much of the whole
|
|
681
|
+
# digest reaches a consumer. A sidebar is 22 columns and a transcript line
|
|
682
|
+
# wraps; the point is to say WHICH url or WHICH query, not to reproduce the
|
|
683
|
+
# call.
|
|
684
|
+
_DIGEST_VALUE_CHARS = 60
|
|
685
|
+
_DIGEST_CHARS = 140
|
|
686
|
+
|
|
687
|
+
# A VALUE CARRYING A URL GETS ITS OWN, WIDER CAP, and the digest a wider
|
|
688
|
+
# total to hold one (batch 96).
|
|
689
|
+
#
|
|
690
|
+
# Batch 65 made a truncated URL clickable by carrying the whole one
|
|
691
|
+
# alongside the line, so the click has always opened the right page. What
|
|
692
|
+
# it could not fix is the TERMINAL: `https://host/a/b…` minus the ellipsis
|
|
693
|
+
# is itself a syntactically valid URL, so a terminal's own link detector
|
|
694
|
+
# sees a shorter address than the harness resolved and ctrl+click opens
|
|
695
|
+
# THAT -- measured, and reported as the click going somewhere wrong.
|
|
696
|
+
#
|
|
697
|
+
# The honest fix is to stop producing the mismatch: draw the whole URL, so
|
|
698
|
+
# the visible text is the target again and the two layers cannot disagree.
|
|
699
|
+
# `markdown.link_spans`' elision stays for anything past this cap, where
|
|
700
|
+
# the invariant is still "the visible text tells you the origin".
|
|
701
|
+
#
|
|
702
|
+
# BOTH CAPS MOVE OR NEITHER DOES. A 200-character value cut at a
|
|
703
|
+
# 140-character TOTAL is the same mismatch one line further down, which is
|
|
704
|
+
# why the total is raised only for a digest that actually holds a URL --
|
|
705
|
+
# every other digest keeps the width it has always had.
|
|
706
|
+
_DIGEST_URL_CHARS = 200
|
|
707
|
+
_DIGEST_URL_TOTAL = 240
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def param_digest(params, omit=()) -> str:
|
|
711
|
+
"""A short, REDACTED description of what a tool was called with.
|
|
712
|
+
|
|
713
|
+
REDACTED BEFORE TRUNCATION, not after. Truncating first can cut a
|
|
714
|
+
credential in half and leave the surviving half unmatched by the pattern
|
|
715
|
+
that would have caught it -- the redactor works on whole strings, so it
|
|
716
|
+
has to see one. That ordering is the whole reason this function is
|
|
717
|
+
shared rather than copied: §26 established it for the research view, and
|
|
718
|
+
§27's replay renderer is its second caller. A second copy is how the
|
|
719
|
+
ordering silently regresses in one of them, which is this project's
|
|
720
|
+
canonical bug shape (fix at the producer, not the consumer).
|
|
721
|
+
|
|
722
|
+
It lives HERE, beside redact_secrets, rather than in the orchestrator
|
|
723
|
+
that first needed it: the rule it enforces is a content-policy rule,
|
|
724
|
+
and this module is where post-call content policy lives.
|
|
725
|
+
|
|
726
|
+
Tool arguments are the path that genuinely was not scanned: §25 R5 added
|
|
727
|
+
check_input_policy(), but that REFUSES a call, it does not redact one,
|
|
728
|
+
and dispatch()'s check_output_policy only ever saw results. A fetch_url
|
|
729
|
+
whose url carries an API key would otherwise be printed verbatim in two
|
|
730
|
+
shells and, since §27, replayed out of the archive into a third place.
|
|
731
|
+
|
|
732
|
+
Values only, no keys: for the tools a pass actually reaches for -- one
|
|
733
|
+
url, one query, one command -- the key is noise, and per-value
|
|
734
|
+
truncation is what keeps a `write` call from pasting a file into the
|
|
735
|
+
transcript.
|
|
736
|
+
|
|
737
|
+
`omit` NAMES KEYS TO SKIP (§42, RA3), and it is a parameter rather
|
|
738
|
+
than a rule because this module must not learn about the registry:
|
|
739
|
+
it imports `config` and `security.posture` and nothing else, and
|
|
740
|
+
which tool declares which param is the registry's knowledge.
|
|
741
|
+
`registry.call_digest()` is the one caller that does the lookup.
|
|
742
|
+
|
|
743
|
+
It exists for `shell`'s `rationale`, which is prose written for a
|
|
744
|
+
person to read and would otherwise take sixty of the digest's one
|
|
745
|
+
hundred and forty characters. Key ORDER is the model's, so it
|
|
746
|
+
lands first about half the time: measured, `ls -la dist/` became
|
|
747
|
+
`I need to check whether the build artifacts directory still… ls
|
|
748
|
+
-la dist/` -- the command no longer the first thing a reader sees.
|
|
749
|
+
"""
|
|
750
|
+
if not isinstance(params, dict) or not params:
|
|
751
|
+
return ""
|
|
752
|
+
parts = []
|
|
753
|
+
widened = False
|
|
754
|
+
for key, value in params.items():
|
|
755
|
+
if key in omit:
|
|
756
|
+
continue
|
|
757
|
+
text = value if isinstance(value, str) else json.dumps(value, default=str)
|
|
758
|
+
# The same redaction tool output takes -- shapes included (#167),
|
|
759
|
+
# so a fetch_url whose url carries userinfo does not print its
|
|
760
|
+
# password into the transcript, the CLI and (since §27) the
|
|
761
|
+
# replayed archive. Under VENASTINE_REDACT_OFF this shows raw,
|
|
762
|
+
# like every other pattern substitution; it stays display-only
|
|
763
|
+
# and never logs, whatever the switch reads.
|
|
764
|
+
text = redact_output_text(text)
|
|
765
|
+
# ONE LINE, and the collapse happens BEFORE the truncation so that
|
|
766
|
+
# sixty characters means sixty VISIBLE ones (batch 96). A digest is
|
|
767
|
+
# drawn as a single row -- `▸ shell <digest>` -- and nothing
|
|
768
|
+
# downstream re-flows it, so a multi-line command (a heredoc, an
|
|
769
|
+
# inline `python3 - <<EOF`) drew its remaining lines at column 0
|
|
770
|
+
# underneath, outside the indent and outside the tool's line. The
|
|
771
|
+
# transcript, `core/replay.py` and the research pass trace all read
|
|
772
|
+
# this one producer, so all three drew it.
|
|
773
|
+
#
|
|
774
|
+
# AFTER the redaction, which is the ordering this function exists
|
|
775
|
+
# to hold: the redactor works on whole strings, and collapsing
|
|
776
|
+
# first would not change what it matches, but doing it after keeps
|
|
777
|
+
# the rule "redact, THEN shorten" true of every step below it.
|
|
778
|
+
text = " ".join(text.split())
|
|
779
|
+
# `http` is a FILTER, not a grammar: which substrings are URLs is
|
|
780
|
+
# `tui/markdown.py`'s question, asked there against one definition.
|
|
781
|
+
# This only declines to widen a value that cannot contain one --
|
|
782
|
+
# the same test, and the same reasoning, as `redacted_values`.
|
|
783
|
+
has_url = "http" in text
|
|
784
|
+
widened = widened or has_url
|
|
785
|
+
cap = _DIGEST_URL_CHARS if has_url else _DIGEST_VALUE_CHARS
|
|
786
|
+
if len(text) > cap:
|
|
787
|
+
text = text[:cap - 1] + "…"
|
|
788
|
+
parts.append(text)
|
|
789
|
+
digest = " ".join(p for p in parts if p)
|
|
790
|
+
total = _DIGEST_URL_TOTAL if widened else _DIGEST_CHARS
|
|
791
|
+
if len(digest) > total:
|
|
792
|
+
digest = digest[:total - 1] + "…"
|
|
793
|
+
return digest
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def redacted_values(params, omit=()) -> tuple:
|
|
797
|
+
"""The same values `param_digest` summarises, UNTRUNCATED (batch 65).
|
|
798
|
+
|
|
799
|
+
The transcript arms a URL in a tool line for ctrl+click, and the
|
|
800
|
+
line it draws is a digest: the 60-character value cap above turns
|
|
801
|
+
most real documentation URLs into `https://…/pa…`, which cannot be
|
|
802
|
+
clicked because it is not the URL. These are the candidates a
|
|
803
|
+
truncated span resolves against.
|
|
804
|
+
|
|
805
|
+
IT LIVES HERE, beside param_digest, and it is the redaction that
|
|
806
|
+
puts it here rather than tidiness. A click hands its target to the
|
|
807
|
+
platform's browser, so a target taken from the RAW params would
|
|
808
|
+
send the `?api_key=…` this module deliberately kept out of the
|
|
809
|
+
transcript -- #167's protection defeated at the one place it was
|
|
810
|
+
designed to hold, by a feature whose whole subject is the same
|
|
811
|
+
string.
|
|
812
|
+
|
|
813
|
+
A VALUE THE REDACTOR CHANGED IS DROPPED ENTIRELY. Redacting and
|
|
814
|
+
then offering the result would arm a link to somewhere that does
|
|
815
|
+
not exist -- `https://user:[REDACTED]@host/…` is a real URL, it is
|
|
816
|
+
simply not the one that was fetched. Refusing the value is the
|
|
817
|
+
honest answer: nothing was hidden from the reader, and nothing
|
|
818
|
+
invented is offered to the browser. (`markdown.clickable` refuses
|
|
819
|
+
that particular shape a second time, at the point of action, for
|
|
820
|
+
the reason every check in this project is asked twice.)
|
|
821
|
+
|
|
822
|
+
Values only and `omit` skipped, both matching param_digest exactly:
|
|
823
|
+
the two functions describe ONE call, and a candidate the line does
|
|
824
|
+
not draw is a target nothing can resolve to.
|
|
825
|
+
|
|
826
|
+
The `http` test is a filter and not a grammar. Which substrings are
|
|
827
|
+
URLs is `tui/markdown.py`'s question and is asked there against one
|
|
828
|
+
definition; this only declines to carry values that cannot possibly
|
|
829
|
+
contain one, so a `write` call's file content never reaches the
|
|
830
|
+
renderer.
|
|
831
|
+
"""
|
|
832
|
+
if not isinstance(params, dict) or not params:
|
|
833
|
+
return ()
|
|
834
|
+
values = []
|
|
835
|
+
for key, value in params.items():
|
|
836
|
+
if key in omit:
|
|
837
|
+
continue
|
|
838
|
+
text = value if isinstance(value, str) \
|
|
839
|
+
else json.dumps(value, default=str)
|
|
840
|
+
if "http" not in text:
|
|
841
|
+
continue
|
|
842
|
+
redacted = redact_output_text(text)
|
|
843
|
+
if redacted != text:
|
|
844
|
+
continue
|
|
845
|
+
values.append(redacted)
|
|
846
|
+
return tuple(dict.fromkeys(values))
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
# ---------------------------------------------------------------------------
|
|
850
|
+
# ---- Shared scan traversal ------------------------------------------------
|
|
851
|
+
# ---------------------------------------------------------------------------
|
|
852
|
+
|
|
853
|
+
# THERE IS NO KEY ALLOWLIST ANY MORE (#47). There used to be:
|
|
854
|
+
#
|
|
855
|
+
# _SCANNED_KEYS = ("content", "result", "stdout", "stderr", "error")
|
|
856
|
+
#
|
|
857
|
+
# and `check_output_policy` tested exact membership. Both search tools
|
|
858
|
+
# return their payload under `results` -- plural -- which is not `result`,
|
|
859
|
+
# so the output of the two tools whose content is ENTIRELY third-party
|
|
860
|
+
# text was the output nothing redacted. Verified end to end: a planted
|
|
861
|
+
# key survived into the persisted archive and into what the model was
|
|
862
|
+
# sent the next turn.
|
|
863
|
+
#
|
|
864
|
+
# The allowlist grew correctly each time someone noticed a gap ("error"
|
|
865
|
+
# was added because an MCP server reports a failed call in band, often
|
|
866
|
+
# quoting the credential that was rejected), and that is the problem: it
|
|
867
|
+
# re-created the defect for every tool added after it, and it did so
|
|
868
|
+
# SILENTLY. mcp_client/client.py's _normalize was written to satisfy this
|
|
869
|
+
# set deliberately, treating membership as a guarantee -- so the contract
|
|
870
|
+
# was real and known, and the two oldest built-ins never satisfied it.
|
|
871
|
+
#
|
|
872
|
+
# Scanning every key is the producer-side fix this project's own
|
|
873
|
+
# convention names ("a per-case branch added to a normalizer usually
|
|
874
|
+
# means the fix belongs one layer down"). A tool that returns text now
|
|
875
|
+
# has that text scanned because it is text, not because someone
|
|
876
|
+
# remembered to name its key.
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
# How deep to descend into nested values. Bounded so a pathological or
|
|
880
|
+
# hostile structure can't turn a scan into a stack overflow -- this walks
|
|
881
|
+
# output from code the user didn't write, and arguments chosen by a model
|
|
882
|
+
# that has been reading attacker-controlled web pages.
|
|
883
|
+
_MAX_SCAN_DEPTH = 12
|
|
884
|
+
|
|
885
|
+
|
|
886
|
+
class _PolicyRefusal(Exception):
|
|
887
|
+
"""Internal control flow: a leaf handler rejecting what it was given.
|
|
888
|
+
|
|
889
|
+
Never escapes this module -- check_input_policy() converts it to a
|
|
890
|
+
reason string, and the redaction path's leaf handler never raises.
|
|
891
|
+
"""
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def _walk(value, leaf, on_cap, depth: int = 0, scan_keys: bool = False):
|
|
895
|
+
"""Depth-bounded traversal of a JSON-shaped value.
|
|
896
|
+
|
|
897
|
+
ONE traversal, two callers. `leaf` maps a str to its replacement (and
|
|
898
|
+
may raise `_PolicyRefusal`); `on_cap` decides what happens when the
|
|
899
|
+
depth bound is reached. Redaction and argument scanning differ ONLY in
|
|
900
|
+
those two behaviours, and writing the walk twice is how the two drift
|
|
901
|
+
-- the §17 hole existed because `structured_content` was nested and
|
|
902
|
+
the scan was not, and a second copy would reintroduce exactly that
|
|
903
|
+
class of gap on the input side.
|
|
904
|
+
|
|
905
|
+
ADDED IN §17 (as `_redact_value`), and it closed a real hole rather
|
|
906
|
+
than generalising for its own sake. The original scanned only
|
|
907
|
+
TOP-LEVEL strings, which was sufficient while every tool returned
|
|
908
|
+
`{"result": "<some string>"}`. MCP breaks that assumption:
|
|
909
|
+
`structured_content` is arbitrary JSON from a third-party server, and
|
|
910
|
+
it arrives under `result` as a dict -- so the output most likely to
|
|
911
|
+
contain a leaked credential was the one output that was never
|
|
912
|
+
scanned. Caught by an end-to-end test, not by reading the code.
|
|
913
|
+
|
|
914
|
+
Fixed here rather than in mcp_client/, because the producer of the
|
|
915
|
+
unscanned shape isn't MCP -- it's any tool returning nested data under
|
|
916
|
+
a scanned key, which `discrete_math` and `probability_stats` already
|
|
917
|
+
can. A special case at the MCP boundary would have left those.
|
|
918
|
+
|
|
919
|
+
scan_keys is False for redaction and True for argument scanning. The
|
|
920
|
+
asymmetry is deliberate: rewriting a result's dict KEYS would change
|
|
921
|
+
the shape a tool's consumer sees, while a secret sitting in an
|
|
922
|
+
argument key (`{"sk-ant-...": "x"}`) is a plausible way to smuggle one
|
|
923
|
+
past a values-only scan.
|
|
924
|
+
"""
|
|
925
|
+
if isinstance(value, str):
|
|
926
|
+
return leaf(value)
|
|
927
|
+
if depth >= _MAX_SCAN_DEPTH:
|
|
928
|
+
# Fail CLOSED. Returning the container untouched made the depth
|
|
929
|
+
# bound a deterministic bypass: a server placing a credential 13
|
|
930
|
+
# levels down under a scanned key passed straight through, and
|
|
931
|
+
# third-party structured_content is precisely the threat this
|
|
932
|
+
# function's docstring names. The cap exists to stop a hostile
|
|
933
|
+
# structure exhausting the stack, so refusing to descend must
|
|
934
|
+
# mean refusing to emit, not emitting unchecked.
|
|
935
|
+
return on_cap()
|
|
936
|
+
if isinstance(value, dict):
|
|
937
|
+
return {
|
|
938
|
+
(leaf(k) if scan_keys and isinstance(k, str) else k):
|
|
939
|
+
_walk(v, leaf, on_cap, depth + 1, scan_keys)
|
|
940
|
+
for k, v in value.items()
|
|
941
|
+
}
|
|
942
|
+
if isinstance(value, (list, tuple)):
|
|
943
|
+
return type(value)(
|
|
944
|
+
_walk(v, leaf, on_cap, depth + 1, scan_keys) for v in value
|
|
945
|
+
)
|
|
946
|
+
return value
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
# ---------------------------------------------------------------------------
|
|
950
|
+
# ---- Output policy check --------------------------------------------------
|
|
951
|
+
# ---------------------------------------------------------------------------
|
|
952
|
+
|
|
953
|
+
def check_output_policy(tool_name: str, result: dict) -> dict:
|
|
954
|
+
"""Apply content-level policy to a tool's output.
|
|
955
|
+
|
|
956
|
+
Called once, centrally, by registry.dispatch() after every tool
|
|
957
|
+
call — not something individual tool files call themselves.
|
|
958
|
+
Mutates *result* in place and returns it.
|
|
959
|
+
|
|
960
|
+
EVERY value is scanned, not an allowlist of keys (#47) — see the
|
|
961
|
+
note above `_walk`. `scan_keys=False` still, so a result's dict KEYS
|
|
962
|
+
are left alone: rewriting them would change the shape a tool's
|
|
963
|
+
consumer sees, and a tool result's keys are harness- or
|
|
964
|
+
server-declared names rather than free text. That asymmetry with
|
|
965
|
+
`check_input_policy` is deliberate and predates this change.
|
|
966
|
+
|
|
967
|
+
NOTHING THIS FUNCTION DOES IS SILENT ANY MORE (#49). Every
|
|
968
|
+
alteration — a redacted value, a depth-cap substitution — produces
|
|
969
|
+
exactly ONE warning naming the tool, what was altered and how many
|
|
970
|
+
values were touched (and WHICH top-level key sat over a capped
|
|
971
|
+
container). The matched content is never echoed: this function
|
|
972
|
+
exists because something credential-shaped was found, so quoting it
|
|
973
|
+
to say so would leak it into app.log, the TUI transcript and every
|
|
974
|
+
place a WARNING travels — the same reasoning _input_leaf's refusal
|
|
975
|
+
already states. Before #49 the depth cap replaced a whole nested
|
|
976
|
+
container with a sentence and logged nothing, leaving truncated
|
|
977
|
+
output indistinguishable from complete output; `registry.schemas`
|
|
978
|
+
states the rule this was violating ("no quiet invisibility").
|
|
979
|
+
"""
|
|
980
|
+
redacted_values = 0
|
|
981
|
+
capped_keys = []
|
|
982
|
+
|
|
983
|
+
def _leaf(text: str) -> str:
|
|
984
|
+
nonlocal redacted_values
|
|
985
|
+
replacement = redact_output_text(text)
|
|
986
|
+
if replacement != text:
|
|
987
|
+
redacted_values += 1
|
|
988
|
+
return replacement
|
|
989
|
+
|
|
990
|
+
for key in list(result):
|
|
991
|
+
hit_cap = []
|
|
992
|
+
result[key] = _walk(
|
|
993
|
+
result[key],
|
|
994
|
+
_leaf,
|
|
995
|
+
lambda: (hit_cap.append(key),
|
|
996
|
+
"[REDACTED: nested beyond scan depth]")[1],
|
|
997
|
+
)
|
|
998
|
+
if hit_cap:
|
|
999
|
+
capped_keys.append(key)
|
|
1000
|
+
|
|
1001
|
+
if redacted_values or capped_keys:
|
|
1002
|
+
parts = []
|
|
1003
|
+
if redacted_values:
|
|
1004
|
+
parts.append(f"{redacted_values} value(s) had credentials "
|
|
1005
|
+
f"replaced")
|
|
1006
|
+
if capped_keys:
|
|
1007
|
+
parts.append("nested content beyond scan depth was substituted "
|
|
1008
|
+
"under key(s): " + ", ".join(map(repr, capped_keys)))
|
|
1009
|
+
logger.warning("Output policy altered %s result: %s.",
|
|
1010
|
+
tool_name, "; ".join(parts))
|
|
1011
|
+
return result
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
# ---------------------------------------------------------------------------
|
|
1015
|
+
# ---- Input policy check ---------------------------------------------------
|
|
1016
|
+
# ---------------------------------------------------------------------------
|
|
1017
|
+
#
|
|
1018
|
+
# ROADMAP_v2 §25 (R5). The symmetric half of check_output_policy, and it
|
|
1019
|
+
# closes two holes that predate the research-pipeline work:
|
|
1020
|
+
#
|
|
1021
|
+
# 1. Results were scanned for secrets; ARGUMENTS never were. A tool the
|
|
1022
|
+
# user approved can be called with text drawn from context, so the
|
|
1023
|
+
# argument list is an egress path and nothing was watching it.
|
|
1024
|
+
# 2. BLOCKED_DOMAINS was opt-in PER TOOL -- only web_search.py and
|
|
1025
|
+
# fetch_url.py import is_domain_blocked(). Any other tool taking a
|
|
1026
|
+
# URL (every MCP tool that fetches anything) bypassed the list
|
|
1027
|
+
# entirely, which is the `fetch_url` defect shape again: a control
|
|
1028
|
+
# that exists, is documented, and does not cover what it claims to.
|
|
1029
|
+
#
|
|
1030
|
+
# Enforced at dispatch() so it covers every tool in every mode, rather
|
|
1031
|
+
# than at the pipeline boundary that prompted it.
|
|
1032
|
+
|
|
1033
|
+
# A URL is recognised by its scheme, not by looking like a hostname. A
|
|
1034
|
+
# BARE domain is deliberately NOT refused here: this harness researches
|
|
1035
|
+
# security topics, so "exploit-db.com" appears in legitimate queries,
|
|
1036
|
+
# claims and reports constantly, and refusing every mention would make the
|
|
1037
|
+
# blocklist unusable while blocking nothing an attacker couldn't rephrase.
|
|
1038
|
+
# fetch_url.py keeps its own bare-domain check for the value it is given.
|
|
1039
|
+
_URL_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s\"'<>]+")
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def _input_leaf(text: str) -> str:
|
|
1043
|
+
"""Refuse a string carrying a credential or a blocked URL."""
|
|
1044
|
+
for pattern in _SECRET_PATTERNS:
|
|
1045
|
+
if pattern.search(text):
|
|
1046
|
+
# The matched text is NOT echoed. A refusal reason travels
|
|
1047
|
+
# into model context, the TUI transcript and the persisted
|
|
1048
|
+
# MessageLog -- quoting the credential to explain that a
|
|
1049
|
+
# credential leaked would leak it three more places.
|
|
1050
|
+
raise _PolicyRefusal(
|
|
1051
|
+
"an argument contains what looks like a credential")
|
|
1052
|
+
for url in _URL_RE.findall(text):
|
|
1053
|
+
if is_domain_blocked(url):
|
|
1054
|
+
raise _PolicyRefusal(f"an argument targets a blocked domain: {url}")
|
|
1055
|
+
return text
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def check_input_policy(tool_name: str, params: dict) -> str | None:
|
|
1059
|
+
"""Content-level policy on a tool's ARGUMENTS. Returns a refusal
|
|
1060
|
+
reason, or None to proceed.
|
|
1061
|
+
|
|
1062
|
+
Called by registry.dispatch() before the handler runs. REFUSES rather
|
|
1063
|
+
than redacting: silently rewriting an argument changes what the call
|
|
1064
|
+
does, so a tool would run with parameters neither the model nor the
|
|
1065
|
+
user chose and report success. A denial is legible to both -- the same
|
|
1066
|
+
reasoning D24 settled for uncallable tools.
|
|
1067
|
+
"""
|
|
1068
|
+
try:
|
|
1069
|
+
_walk(
|
|
1070
|
+
params,
|
|
1071
|
+
_input_leaf,
|
|
1072
|
+
lambda: _raise_refusal("arguments nested beyond the scan depth"),
|
|
1073
|
+
scan_keys=True,
|
|
1074
|
+
)
|
|
1075
|
+
except _PolicyRefusal as e:
|
|
1076
|
+
return f"{tool_name} call refused by policy: {e}"
|
|
1077
|
+
return None
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _raise_refusal(reason: str):
|
|
1081
|
+
"""`lambda: raise` is not valid syntax; this is the on_cap callback."""
|
|
1082
|
+
raise _PolicyRefusal(reason)
|